Fixing connector-persistente to EDC 0.7.2 (#78)

* feat: migrating to edc 0.7.3

* feat: migrating to edc 0.7.2

* feat: migrating to edc 0.7.2

* feat: migrating to edc 0.7.2

* feat: migrating to edc 0.7.2

* feat: migrating to edc 0.7.2

* feat: migrating to edc 0.7.2

deployment/terraform/clean-state.sh

@@ -26,5 +26,17 @@ rm -f ./ionos-postgresqlaas/terraform.tfstate
 rm -f ./ionos-postgresqlaas/.terraform.lock.hcl
 rm -f ./ionos-postgresqlaas/terraform.tfstate.backup
 
+rm -rf ./postgresql-deploy/.terraform
+rm -f ./postgresql-deploy/terraform.tfstate
+rm -f ./postgresql-deploy/.terraform.lock.hcl
+rm -f ./postgresql-deploy/terraform.tfstate.backup
+
+rm -rf ./db-scripts/.terraform
+rm -f ./db-scripts/terraform.tfstate
+rm -f ./db-scripts/.terraform.lock.hcl
+rm -f ./db-scripts/terraform.tfstate.backup
+
 rm -f vault-init/vault-keys.json
 rm -f vault-init/vault-tokens.json
+
+echo "Terraform state cleanup complete"

deployment/terraform/db-scripts/accesstokendata-store/schema.sql

@@ -0,0 +1,27 @@
+/*
+ *  Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
+ *
+ */
+
+-- Statements are designed for and tested with Postgres only!
+
+CREATE TABLE IF NOT EXISTS edc_accesstokendata
+(
+    id           VARCHAR NOT NULL PRIMARY KEY,
+    claim_token  JSON    NOT NULL,
+    data_address JSON    NOT NULL,
+    additional_properties JSON DEFAULT '{}'
+);
+
+COMMENT ON COLUMN edc_accesstokendata.claim_token IS 'ClaimToken serialized as JSON map';
+COMMENT ON COLUMN edc_accesstokendata.data_address IS 'DataAddress serialized as JSON map';
+COMMENT ON COLUMN edc_accesstokendata.additional_properties IS 'Optional Additional properties serialized as JSON map';

deployment/terraform/db-scripts/asset-index/schema.sql

@@ -0,0 +1,30 @@
+--
+--  Copyright (c) 2022 - 2023 Daimler TSS GmbH
+--
+--  This program and the accompanying materials are made available under the
+--  terms of the Apache License, Version 2.0 which is available at
+--  https://www.apache.org/licenses/LICENSE-2.0
+--
+--  SPDX-License-Identifier: Apache-2.0
+--
+--  Contributors:
+--       Daimler TSS GmbH - Initial SQL Query
+--       Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - improvements
+--
+
+-- THIS SCHEMA HAS BEEN WRITTEN AND TESTED ONLY FOR POSTGRES
+
+-- table: edc_asset
+CREATE TABLE IF NOT EXISTS edc_asset
+(
+    asset_id           VARCHAR NOT NULL,
+    created_at         BIGINT  NOT NULL,
+    properties         JSON    DEFAULT '{}',
+    private_properties JSON    DEFAULT '{}',
+    data_address       JSON    DEFAULT '{}',
+    PRIMARY KEY (asset_id)
+);
+
+COMMENT ON COLUMN edc_asset.properties IS 'Asset properties serialized as JSON';
+COMMENT ON COLUMN edc_asset.private_properties IS 'Asset private properties serialized as JSON';
+COMMENT ON COLUMN edc_asset.data_address IS 'Asset DataAddress serialized as JSON';

deployment/terraform/db-scripts/contract-definition-store/schema.sql

@@ -0,0 +1,27 @@
+--
+--  Copyright (c) 2022 Daimler TSS GmbH
+--
+--  This program and the accompanying materials are made available under the
+--  terms of the Apache License, Version 2.0 which is available at
+--  https://www.apache.org/licenses/LICENSE-2.0
+--
+--  SPDX-License-Identifier: Apache-2.0
+--
+--  Contributors:
+--       Daimler TSS GmbH - Initial SQL Query
+--       Microsoft Corporation - refactoring
+--       SAP SE - add private properties to contract definition
+--
+
+-- table: edc_contract_definitions
+-- only intended for and tested with H2 and Postgres!
+CREATE TABLE IF NOT EXISTS edc_contract_definitions
+(
+    created_at             BIGINT  NOT NULL,
+    contract_definition_id VARCHAR NOT NULL,
+    access_policy_id       VARCHAR NOT NULL,
+    contract_policy_id     VARCHAR NOT NULL,
+    assets_selector        JSON    NOT NULL,
+    private_properties     JSON,
+    PRIMARY KEY (contract_definition_id)
+);

deployment/terraform/db-scripts/contract-negotiation-store/schema.sql

@@ -0,0 +1,86 @@
+-- Statements are designed for and tested with Postgres only!
+
+CREATE TABLE IF NOT EXISTS edc_lease
+(
+    leased_by      VARCHAR               NOT NULL,
+    leased_at      BIGINT,
+    lease_duration INTEGER DEFAULT 60000 NOT NULL,
+    lease_id       VARCHAR               NOT NULL
+        CONSTRAINT lease_pk
+            PRIMARY KEY
+);
+
+COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease';
+
+COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds';
+
+
+CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex
+    ON edc_lease (lease_id);
+
+
+
+CREATE TABLE IF NOT EXISTS edc_contract_agreement
+(
+    agr_id            VARCHAR NOT NULL
+        CONSTRAINT contract_agreement_pk
+            PRIMARY KEY,
+    provider_agent_id VARCHAR,
+    consumer_agent_id VARCHAR,
+    signing_date      BIGINT,
+    start_date        BIGINT,
+    end_date          INTEGER,
+    asset_id          VARCHAR NOT NULL,
+    policy            JSON
+);
+
+
+CREATE TABLE IF NOT EXISTS edc_contract_negotiation
+(
+    id                   VARCHAR           NOT NULL
+        CONSTRAINT contract_negotiation_pk
+            PRIMARY KEY,
+    created_at           BIGINT            NOT NULL,
+    updated_at           BIGINT            NOT NULL,
+    correlation_id       VARCHAR,
+    counterparty_id      VARCHAR           NOT NULL,
+    counterparty_address VARCHAR           NOT NULL,
+    protocol             VARCHAR           NOT NULL,
+    type                 VARCHAR           NOT NULL,
+    state                INTEGER DEFAULT 0 NOT NULL,
+    state_count          INTEGER DEFAULT 0,
+    state_timestamp      BIGINT,
+    error_detail         VARCHAR,
+    agreement_id         VARCHAR
+        CONSTRAINT contract_negotiation_contract_agreement_id_fk
+            REFERENCES edc_contract_agreement,
+    contract_offers      JSON,
+    callback_addresses   JSON,
+    trace_context        JSON,
+    pending              BOOLEAN DEFAULT FALSE,
+    protocol_messages    JSON,
+    lease_id             VARCHAR
+        CONSTRAINT contract_negotiation_lease_lease_id_fk
+            REFERENCES edc_lease
+            ON DELETE SET NULL
+);
+
+COMMENT ON COLUMN edc_contract_negotiation.agreement_id IS 'ContractAgreement serialized as JSON';
+
+COMMENT ON COLUMN edc_contract_negotiation.contract_offers IS 'List<ContractOffer> serialized as JSON';
+
+COMMENT ON COLUMN edc_contract_negotiation.trace_context IS 'Map<String,String> serialized as JSON';
+
+
+CREATE INDEX IF NOT EXISTS contract_negotiation_correlationid_index
+    ON edc_contract_negotiation (correlation_id);
+
+CREATE UNIQUE INDEX IF NOT EXISTS contract_negotiation_id_uindex
+    ON edc_contract_negotiation (id);
+
+CREATE UNIQUE INDEX IF NOT EXISTS contract_agreement_id_uindex
+    ON edc_contract_agreement (agr_id);
+
+
+-- This will help to identify states that need to be transitioned without a table scan when the entries grow
+CREATE INDEX IF NOT EXISTS contract_negotiation_state ON edc_contract_negotiation (state,state_timestamp);

deployment/terraform/db-scripts/data-plane-instance-store/schema.sql

@@ -0,0 +1,20 @@
+CREATE TABLE IF NOT EXISTS edc_lease
+(
+    leased_by      VARCHAR NOT NULL,
+    leased_at      BIGINT,
+    lease_duration INTEGER NOT NULL,
+    lease_id       VARCHAR NOT NULL
+        CONSTRAINT lease_pk
+            PRIMARY KEY
+);
+
+
+CREATE TABLE IF NOT EXISTS edc_data_plane_instance
+(
+    id                   VARCHAR NOT NULL PRIMARY KEY,
+    data                 JSON,
+    lease_id             VARCHAR
+        CONSTRAINT data_plane_instance_lease_id_fk
+                    REFERENCES edc_lease
+                    ON DELETE SET NULL
+);

deployment/terraform/db-scripts/data-plane-store/schema.sql

@@ -0,0 +1,43 @@
+-- Statements are designed for and tested with Postgres only!
+
+CREATE TABLE IF NOT EXISTS edc_lease
+(
+    leased_by      VARCHAR NOT NULL,
+    leased_at      BIGINT,
+    lease_duration INTEGER NOT NULL,
+    lease_id       VARCHAR NOT NULL
+        CONSTRAINT lease_pk
+            PRIMARY KEY
+);
+
+COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease';
+COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds';
+
+CREATE TABLE IF NOT EXISTS edc_data_plane
+(
+    process_id           VARCHAR NOT NULL PRIMARY KEY,
+    state                INTEGER NOT NULL            ,
+    created_at           BIGINT  NOT NULL            ,
+    updated_at           BIGINT  NOT NULL            ,
+    state_count          INTEGER DEFAULT 0 NOT NULL,
+    state_time_stamp     BIGINT,
+    trace_context        JSON,
+    error_detail         VARCHAR,
+    callback_address     VARCHAR,
+    lease_id             VARCHAR
+        CONSTRAINT data_plane_lease_lease_id_fk
+                    REFERENCES edc_lease
+                    ON DELETE SET NULL,
+    source               JSON,
+    destination          JSON,
+    properties           JSON,
+    flow_type            VARCHAR
+);
+
+COMMENT ON COLUMN edc_data_plane.trace_context IS 'Java Map serialized as JSON';
+COMMENT ON COLUMN edc_data_plane.source IS 'DataAddress serialized as JSON';
+COMMENT ON COLUMN edc_data_plane.destination IS 'DataAddress serialized as JSON';
+COMMENT ON COLUMN edc_data_plane.properties IS 'Java Map serialized as JSON';
+
+-- This will help to identify states that need to be transitioned without a table scan when the entries grow
+CREATE INDEX IF NOT EXISTS data_plane_state ON edc_data_plane (state,state_time_stamp);

deployment/terraform/db-scripts/db-scripts.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+
+echo "Creating database $TF_VAR_pg_database"
+set +e
+kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-database --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="postgres" --command="CREATE DATABASE $TF_VAR_pg_database;"
+set -e
+
+echo "Creating accesstokendata-store tables"
+kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-accesstokendata --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./accesstokendata-store/schema.sql
+
+echo "Creating asset-index tables"
+kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-asset-index --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./asset-index/schema.sql
+
+echo "Creating contract-definition-store tables"
+kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-contract-definition --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./contract-definition-store/schema.sql
+
+echo "Creating contract-negotiation-store tables"
+kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-contract-negotiation --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./contract-negotiation-store/schema.sql
+
+echo "Creating data-plane-instance-store tables"
+kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-data-plane-instance --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./data-plane-instance-store/schema.sql
+
+echo "Creating data-plane-store tables"
+kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-data-plane --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./data-plane-store/schema.sql
+
+echo "Creating edr-index tables"
+kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-edr --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./edr-index/schema.sql
+
+echo "Creating policy-definition-store tables"
+kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-policy-definition --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./policy-definition-store/schema.sql
+
+echo "Creating transfer-process-store tables"
+kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-transfer-process --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./transfer-process-store/schema.sql

deployment/terraform/db-scripts/edr-index/schema.sql

@@ -0,0 +1,11 @@
+
+CREATE TABLE IF NOT EXISTS edc_edr_entry
+(
+   transfer_process_id           VARCHAR NOT NULL PRIMARY KEY,
+   agreement_id                  VARCHAR NOT NULL,
+   asset_id                      VARCHAR NOT NULL,
+   provider_id                   VARCHAR NOT NULL,
+   contract_negotiation_id       VARCHAR,
+   created_at                    BIGINT  NOT NULL
+);
+