From 43ec59287fd99bfecb6303e74c5c2bcfe42c0081 Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti <111659831+jannotti-glaucio@users.noreply.github.com> Date: Mon, 30 Sep 2024 12:55:48 -0300 Subject: [PATCH] Fixing connector-persistente to EDC 0.7.2 (#78) * feat: migrating to edc 0.7.3 * feat: migrating to edc 0.7.2 * feat: migrating to edc 0.7.2 * feat: migrating to edc 0.7.2 * feat: migrating to edc 0.7.2 * feat: migrating to edc 0.7.2 * feat: migrating to edc 0.7.2 --- deployment/terraform/clean-state.sh | 12 + .../accesstokendata-store/schema.sql | 27 ++ .../db-scripts/asset-index/schema.sql | 30 ++ .../contract-definition-store/schema.sql | 27 ++ .../contract-negotiation-store/schema.sql | 86 ++++++ .../data-plane-instance-store/schema.sql | 20 ++ .../db-scripts/data-plane-store/schema.sql | 43 +++ deployment/terraform/db-scripts/db-scripts.sh | 35 +++ .../terraform/db-scripts/edr-index/schema.sql | 11 + deployment/terraform/db-scripts/init.sql | 273 ------------------ deployment/terraform/db-scripts/main.tf | 6 + .../policy-definition-store/schema.sql | 41 +++ .../transfer-process-store/schema.sql | 70 +++++ deployment/terraform/deploy-services.sh | 26 +- deployment/terraform/ionos-s3-deploy/main.tf | 7 - .../terraform/postgresql-deploy/main.tf | 51 ++++ deployment/terraform/undeploy-services.sh | 25 +- .../prod/connector-persistence/README.md | 12 +- .../connector-persistence/build.gradle.kts | 7 +- 19 files changed, 508 insertions(+), 301 deletions(-) create mode 100644 deployment/terraform/db-scripts/accesstokendata-store/schema.sql create mode 100644 deployment/terraform/db-scripts/asset-index/schema.sql create mode 100644 deployment/terraform/db-scripts/contract-definition-store/schema.sql create mode 100644 deployment/terraform/db-scripts/contract-negotiation-store/schema.sql create mode 100644 deployment/terraform/db-scripts/data-plane-instance-store/schema.sql create mode 100644 deployment/terraform/db-scripts/data-plane-store/schema.sql create mode 100755 deployment/terraform/db-scripts/db-scripts.sh create mode 100644 deployment/terraform/db-scripts/edr-index/schema.sql delete mode 100644 deployment/terraform/db-scripts/init.sql create mode 100644 deployment/terraform/db-scripts/main.tf create mode 100644 deployment/terraform/db-scripts/policy-definition-store/schema.sql create mode 100644 deployment/terraform/db-scripts/transfer-process-store/schema.sql create mode 100644 deployment/terraform/postgresql-deploy/main.tf diff --git a/deployment/terraform/clean-state.sh b/deployment/terraform/clean-state.sh index 3f92a6e..6d8d4a1 100755 --- a/deployment/terraform/clean-state.sh +++ b/deployment/terraform/clean-state.sh @@ -26,5 +26,17 @@ rm -f ./ionos-postgresqlaas/terraform.tfstate rm -f ./ionos-postgresqlaas/.terraform.lock.hcl rm -f ./ionos-postgresqlaas/terraform.tfstate.backup +rm -rf ./postgresql-deploy/.terraform +rm -f ./postgresql-deploy/terraform.tfstate +rm -f ./postgresql-deploy/.terraform.lock.hcl +rm -f ./postgresql-deploy/terraform.tfstate.backup + +rm -rf ./db-scripts/.terraform +rm -f ./db-scripts/terraform.tfstate +rm -f ./db-scripts/.terraform.lock.hcl +rm -f ./db-scripts/terraform.tfstate.backup + rm -f vault-init/vault-keys.json rm -f vault-init/vault-tokens.json + +echo "Terraform state cleanup complete" \ No newline at end of file diff --git a/deployment/terraform/db-scripts/accesstokendata-store/schema.sql b/deployment/terraform/db-scripts/accesstokendata-store/schema.sql new file mode 100644 index 0000000..de43bc7 --- /dev/null +++ b/deployment/terraform/db-scripts/accesstokendata-store/schema.sql @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +-- Statements are designed for and tested with Postgres only! + +CREATE TABLE IF NOT EXISTS edc_accesstokendata +( + id VARCHAR NOT NULL PRIMARY KEY, + claim_token JSON NOT NULL, + data_address JSON NOT NULL, + additional_properties JSON DEFAULT '{}' +); + +COMMENT ON COLUMN edc_accesstokendata.claim_token IS 'ClaimToken serialized as JSON map'; +COMMENT ON COLUMN edc_accesstokendata.data_address IS 'DataAddress serialized as JSON map'; +COMMENT ON COLUMN edc_accesstokendata.additional_properties IS 'Optional Additional properties serialized as JSON map'; diff --git a/deployment/terraform/db-scripts/asset-index/schema.sql b/deployment/terraform/db-scripts/asset-index/schema.sql new file mode 100644 index 0000000..6274b5f --- /dev/null +++ b/deployment/terraform/db-scripts/asset-index/schema.sql @@ -0,0 +1,30 @@ +-- +-- Copyright (c) 2022 - 2023 Daimler TSS GmbH +-- +-- This program and the accompanying materials are made available under the +-- terms of the Apache License, Version 2.0 which is available at +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Contributors: +-- Daimler TSS GmbH - Initial SQL Query +-- Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - improvements +-- + +-- THIS SCHEMA HAS BEEN WRITTEN AND TESTED ONLY FOR POSTGRES + +-- table: edc_asset +CREATE TABLE IF NOT EXISTS edc_asset +( + asset_id VARCHAR NOT NULL, + created_at BIGINT NOT NULL, + properties JSON DEFAULT '{}', + private_properties JSON DEFAULT '{}', + data_address JSON DEFAULT '{}', + PRIMARY KEY (asset_id) +); + +COMMENT ON COLUMN edc_asset.properties IS 'Asset properties serialized as JSON'; +COMMENT ON COLUMN edc_asset.private_properties IS 'Asset private properties serialized as JSON'; +COMMENT ON COLUMN edc_asset.data_address IS 'Asset DataAddress serialized as JSON'; diff --git a/deployment/terraform/db-scripts/contract-definition-store/schema.sql b/deployment/terraform/db-scripts/contract-definition-store/schema.sql new file mode 100644 index 0000000..98a30b7 --- /dev/null +++ b/deployment/terraform/db-scripts/contract-definition-store/schema.sql @@ -0,0 +1,27 @@ +-- +-- Copyright (c) 2022 Daimler TSS GmbH +-- +-- This program and the accompanying materials are made available under the +-- terms of the Apache License, Version 2.0 which is available at +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Contributors: +-- Daimler TSS GmbH - Initial SQL Query +-- Microsoft Corporation - refactoring +-- SAP SE - add private properties to contract definition +-- + +-- table: edc_contract_definitions +-- only intended for and tested with H2 and Postgres! +CREATE TABLE IF NOT EXISTS edc_contract_definitions +( + created_at BIGINT NOT NULL, + contract_definition_id VARCHAR NOT NULL, + access_policy_id VARCHAR NOT NULL, + contract_policy_id VARCHAR NOT NULL, + assets_selector JSON NOT NULL, + private_properties JSON, + PRIMARY KEY (contract_definition_id) +); diff --git a/deployment/terraform/db-scripts/contract-negotiation-store/schema.sql b/deployment/terraform/db-scripts/contract-negotiation-store/schema.sql new file mode 100644 index 0000000..02d64c4 --- /dev/null +++ b/deployment/terraform/db-scripts/contract-negotiation-store/schema.sql @@ -0,0 +1,86 @@ +-- Statements are designed for and tested with Postgres only! + +CREATE TABLE IF NOT EXISTS edc_lease +( + leased_by VARCHAR NOT NULL, + leased_at BIGINT, + lease_duration INTEGER DEFAULT 60000 NOT NULL, + lease_id VARCHAR NOT NULL + CONSTRAINT lease_pk + PRIMARY KEY +); + +COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; + +COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; + + +CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex + ON edc_lease (lease_id); + + + +CREATE TABLE IF NOT EXISTS edc_contract_agreement +( + agr_id VARCHAR NOT NULL + CONSTRAINT contract_agreement_pk + PRIMARY KEY, + provider_agent_id VARCHAR, + consumer_agent_id VARCHAR, + signing_date BIGINT, + start_date BIGINT, + end_date INTEGER, + asset_id VARCHAR NOT NULL, + policy JSON +); + + +CREATE TABLE IF NOT EXISTS edc_contract_negotiation +( + id VARCHAR NOT NULL + CONSTRAINT contract_negotiation_pk + PRIMARY KEY, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + correlation_id VARCHAR, + counterparty_id VARCHAR NOT NULL, + counterparty_address VARCHAR NOT NULL, + protocol VARCHAR NOT NULL, + type VARCHAR NOT NULL, + state INTEGER DEFAULT 0 NOT NULL, + state_count INTEGER DEFAULT 0, + state_timestamp BIGINT, + error_detail VARCHAR, + agreement_id VARCHAR + CONSTRAINT contract_negotiation_contract_agreement_id_fk + REFERENCES edc_contract_agreement, + contract_offers JSON, + callback_addresses JSON, + trace_context JSON, + pending BOOLEAN DEFAULT FALSE, + protocol_messages JSON, + lease_id VARCHAR + CONSTRAINT contract_negotiation_lease_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL +); + +COMMENT ON COLUMN edc_contract_negotiation.agreement_id IS 'ContractAgreement serialized as JSON'; + +COMMENT ON COLUMN edc_contract_negotiation.contract_offers IS 'List serialized as JSON'; + +COMMENT ON COLUMN edc_contract_negotiation.trace_context IS 'Map serialized as JSON'; + + +CREATE INDEX IF NOT EXISTS contract_negotiation_correlationid_index + ON edc_contract_negotiation (correlation_id); + +CREATE UNIQUE INDEX IF NOT EXISTS contract_negotiation_id_uindex + ON edc_contract_negotiation (id); + +CREATE UNIQUE INDEX IF NOT EXISTS contract_agreement_id_uindex + ON edc_contract_agreement (agr_id); + + +-- This will help to identify states that need to be transitioned without a table scan when the entries grow +CREATE INDEX IF NOT EXISTS contract_negotiation_state ON edc_contract_negotiation (state,state_timestamp); \ No newline at end of file diff --git a/deployment/terraform/db-scripts/data-plane-instance-store/schema.sql b/deployment/terraform/db-scripts/data-plane-instance-store/schema.sql new file mode 100644 index 0000000..0f2f324 --- /dev/null +++ b/deployment/terraform/db-scripts/data-plane-instance-store/schema.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS edc_lease +( + leased_by VARCHAR NOT NULL, + leased_at BIGINT, + lease_duration INTEGER NOT NULL, + lease_id VARCHAR NOT NULL + CONSTRAINT lease_pk + PRIMARY KEY +); + + +CREATE TABLE IF NOT EXISTS edc_data_plane_instance +( + id VARCHAR NOT NULL PRIMARY KEY, + data JSON, + lease_id VARCHAR + CONSTRAINT data_plane_instance_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL +); diff --git a/deployment/terraform/db-scripts/data-plane-store/schema.sql b/deployment/terraform/db-scripts/data-plane-store/schema.sql new file mode 100644 index 0000000..768320a --- /dev/null +++ b/deployment/terraform/db-scripts/data-plane-store/schema.sql @@ -0,0 +1,43 @@ +-- Statements are designed for and tested with Postgres only! + +CREATE TABLE IF NOT EXISTS edc_lease +( + leased_by VARCHAR NOT NULL, + leased_at BIGINT, + lease_duration INTEGER NOT NULL, + lease_id VARCHAR NOT NULL + CONSTRAINT lease_pk + PRIMARY KEY +); + +COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; +COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; + +CREATE TABLE IF NOT EXISTS edc_data_plane +( + process_id VARCHAR NOT NULL PRIMARY KEY, + state INTEGER NOT NULL , + created_at BIGINT NOT NULL , + updated_at BIGINT NOT NULL , + state_count INTEGER DEFAULT 0 NOT NULL, + state_time_stamp BIGINT, + trace_context JSON, + error_detail VARCHAR, + callback_address VARCHAR, + lease_id VARCHAR + CONSTRAINT data_plane_lease_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL, + source JSON, + destination JSON, + properties JSON, + flow_type VARCHAR +); + +COMMENT ON COLUMN edc_data_plane.trace_context IS 'Java Map serialized as JSON'; +COMMENT ON COLUMN edc_data_plane.source IS 'DataAddress serialized as JSON'; +COMMENT ON COLUMN edc_data_plane.destination IS 'DataAddress serialized as JSON'; +COMMENT ON COLUMN edc_data_plane.properties IS 'Java Map serialized as JSON'; + +-- This will help to identify states that need to be transitioned without a table scan when the entries grow +CREATE INDEX IF NOT EXISTS data_plane_state ON edc_data_plane (state,state_time_stamp); diff --git a/deployment/terraform/db-scripts/db-scripts.sh b/deployment/terraform/db-scripts/db-scripts.sh new file mode 100755 index 0000000..5dbdfe9 --- /dev/null +++ b/deployment/terraform/db-scripts/db-scripts.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e + +echo "Creating database $TF_VAR_pg_database" +set +e +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-database --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="postgres" --command="CREATE DATABASE $TF_VAR_pg_database;" +set -e + +echo "Creating accesstokendata-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-accesstokendata --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./accesstokendata-store/schema.sql + +echo "Creating asset-index tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-asset-index --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./asset-index/schema.sql + +echo "Creating contract-definition-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-contract-definition --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./contract-definition-store/schema.sql + +echo "Creating contract-negotiation-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-contract-negotiation --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./contract-negotiation-store/schema.sql + +echo "Creating data-plane-instance-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-data-plane-instance --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./data-plane-instance-store/schema.sql + +echo "Creating data-plane-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-data-plane --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./data-plane-store/schema.sql + +echo "Creating edr-index tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-edr --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./edr-index/schema.sql + +echo "Creating policy-definition-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-policy-definition --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./policy-definition-store/schema.sql + +echo "Creating transfer-process-store tables" +kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-transfer-process --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ./transfer-process-store/schema.sql diff --git a/deployment/terraform/db-scripts/edr-index/schema.sql b/deployment/terraform/db-scripts/edr-index/schema.sql new file mode 100644 index 0000000..3f30585 --- /dev/null +++ b/deployment/terraform/db-scripts/edr-index/schema.sql @@ -0,0 +1,11 @@ + +CREATE TABLE IF NOT EXISTS edc_edr_entry +( + transfer_process_id VARCHAR NOT NULL PRIMARY KEY, + agreement_id VARCHAR NOT NULL, + asset_id VARCHAR NOT NULL, + provider_id VARCHAR NOT NULL, + contract_negotiation_id VARCHAR, + created_at BIGINT NOT NULL +); + diff --git a/deployment/terraform/db-scripts/init.sql b/deployment/terraform/db-scripts/init.sql deleted file mode 100644 index 7508e98..0000000 --- a/deployment/terraform/db-scripts/init.sql +++ /dev/null @@ -1,273 +0,0 @@ --- --- Copyright (c) 2022 Daimler TSS GmbH --- --- This program and the accompanying materials are made available under the --- terms of the Apache License, Version 2.0 which is available at --- https://www.apache.org/licenses/LICENSE-2.0 --- --- SPDX-License-Identifier: Apache-2.0 --- --- Contributors: --- Daimler TSS GmbH - Initial SQL Query --- - --- THIS SCHEMA HAS BEEN WRITTEN AND TESTED ONLY FOR POSTGRES - --- table: edc_asset -CREATE TABLE IF NOT EXISTS edc_asset -( - asset_id VARCHAR NOT NULL, - created_at BIGINT NOT NULL, - properties JSON DEFAULT '{}', - private_properties JSON DEFAULT '{}', - data_address JSON DEFAULT '{}', - PRIMARY KEY (asset_id) -); - -COMMENT ON COLUMN edc_asset.properties IS 'Asset properties serialized as JSON'; -COMMENT ON COLUMN edc_asset.private_properties IS 'Asset private properties serialized as JSON'; -COMMENT ON COLUMN edc_asset.data_address IS 'Asset DataAddress serialized as JSON'; - - - --- --- Copyright (c) 2022 Daimler TSS GmbH --- --- This program and the accompanying materials are made available under the --- terms of the Apache License, Version 2.0 which is available at --- https://www.apache.org/licenses/LICENSE-2.0 --- --- SPDX-License-Identifier: Apache-2.0 --- --- Contributors: --- Daimler TSS GmbH - Initial SQL Query --- Microsoft Corporation - refactoring --- - --- table: edc_contract_definitions --- only intended for and tested with H2 and Postgres! -CREATE TABLE IF NOT EXISTS edc_contract_definitions -( - created_at BIGINT NOT NULL, - contract_definition_id VARCHAR NOT NULL, - access_policy_id VARCHAR NOT NULL, - contract_policy_id VARCHAR NOT NULL, - assets_selector JSON NOT NULL, - private_properties JSON, - PRIMARY KEY (contract_definition_id) -); - - --- Statements are designed for and tested with Postgres only! - -CREATE TABLE IF NOT EXISTS edc_lease -( - leased_by VARCHAR NOT NULL, - leased_at BIGINT, - lease_duration INTEGER DEFAULT 60000 NOT NULL, - lease_id VARCHAR NOT NULL - CONSTRAINT lease_pk - PRIMARY KEY -); - -COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; - -COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; - - -CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex - ON edc_lease (lease_id); - - - -CREATE TABLE IF NOT EXISTS edc_contract_agreement -( - agr_id VARCHAR NOT NULL - CONSTRAINT contract_agreement_pk - PRIMARY KEY, - provider_agent_id VARCHAR, - consumer_agent_id VARCHAR, - signing_date BIGINT, - start_date BIGINT, - end_date INTEGER, - asset_id VARCHAR NOT NULL, - policy JSON -); - - -CREATE TABLE IF NOT EXISTS edc_contract_negotiation -( - id VARCHAR NOT NULL - CONSTRAINT contract_negotiation_pk - PRIMARY KEY, - created_at BIGINT NOT NULL, - updated_at BIGINT NOT NULL, - correlation_id VARCHAR, - counterparty_id VARCHAR NOT NULL, - counterparty_address VARCHAR NOT NULL, - protocol VARCHAR NOT NULL, - type VARCHAR NOT NULL, - state INTEGER DEFAULT 0 NOT NULL, - state_count INTEGER DEFAULT 0, - state_timestamp BIGINT, - error_detail VARCHAR, - agreement_id VARCHAR - CONSTRAINT contract_negotiation_contract_agreement_id_fk - REFERENCES edc_contract_agreement, - contract_offers JSON, - callback_addresses JSON, - trace_context JSON, - pending BOOLEAN DEFAULT FALSE, - protocol_messages JSON, - lease_id VARCHAR - CONSTRAINT contract_negotiation_lease_lease_id_fk - REFERENCES edc_lease - ON DELETE SET NULL -); - -COMMENT ON COLUMN edc_contract_negotiation.agreement_id IS 'ContractAgreement serialized as JSON'; - -COMMENT ON COLUMN edc_contract_negotiation.contract_offers IS 'List serialized as JSON'; - -COMMENT ON COLUMN edc_contract_negotiation.trace_context IS 'Map serialized as JSON'; - - -CREATE INDEX IF NOT EXISTS contract_negotiation_correlationid_index - ON edc_contract_negotiation (correlation_id); - -CREATE UNIQUE INDEX IF NOT EXISTS contract_negotiation_id_uindex - ON edc_contract_negotiation (id); - -CREATE UNIQUE INDEX IF NOT EXISTS contract_agreement_id_uindex - ON edc_contract_agreement (agr_id); - - - - --- --- Copyright (c) 2022 ZF Friedrichshafen AG --- --- This program and the accompanying materials are made available under the --- terms of the Apache License, Version 2.0 which is available at --- https://www.apache.org/licenses/LICENSE-2.0 --- --- SPDX-License-Identifier: Apache-2.0 --- --- Contributors: --- ZF Friedrichshafen AG - Initial SQL Query --- - --- Statements are designed for and tested with Postgres only! - --- table: edc_policydefinitions -CREATE TABLE IF NOT EXISTS edc_policydefinitions -( - policy_id VARCHAR NOT NULL, - created_at BIGINT NOT NULL, - permissions JSON, - prohibitions JSON, - duties JSON, - extensible_properties JSON, - inherits_from VARCHAR, - assigner VARCHAR, - assignee VARCHAR, - target VARCHAR, - policy_type VARCHAR NOT NULL, - private_properties JSON, - PRIMARY KEY (policy_id) -); - -COMMENT ON COLUMN edc_policydefinitions.permissions IS 'Java List serialized as JSON'; -COMMENT ON COLUMN edc_policydefinitions.prohibitions IS 'Java List serialized as JSON'; -COMMENT ON COLUMN edc_policydefinitions.duties IS 'Java List serialized as JSON'; -COMMENT ON COLUMN edc_policydefinitions.extensible_properties IS 'Java Map serialized as JSON'; -COMMENT ON COLUMN edc_policydefinitions.policy_type IS 'Java PolicyType serialized as JSON'; - -CREATE UNIQUE INDEX IF NOT EXISTS edc_policydefinitions_id_uindex - ON edc_policydefinitions (policy_id); - - --- Statements are designed for and tested with Postgres only! - -CREATE TABLE IF NOT EXISTS edc_lease -( - leased_by VARCHAR NOT NULL, - leased_at BIGINT, - lease_duration INTEGER NOT NULL, - lease_id VARCHAR NOT NULL - CONSTRAINT lease_pk - PRIMARY KEY -); - -COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; - -COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; - -CREATE TABLE IF NOT EXISTS edc_transfer_process -( - transferprocess_id VARCHAR NOT NULL - CONSTRAINT transfer_process_pk - PRIMARY KEY, - type VARCHAR NOT NULL, - state INTEGER NOT NULL, - state_count INTEGER DEFAULT 0 NOT NULL, - state_time_stamp BIGINT, - created_at BIGINT NOT NULL, - updated_at BIGINT NOT NULL, - trace_context JSON, - error_detail VARCHAR, - resource_manifest JSON, - provisioned_resource_set JSON, - content_data_address JSON, - deprovisioned_resources JSON, - private_properties JSON, - callback_addresses JSON, - pending BOOLEAN DEFAULT FALSE, - transfer_type VARCHAR, - protocol_messages JSON, - lease_id VARCHAR - CONSTRAINT transfer_process_lease_lease_id_fk - REFERENCES edc_lease - ON DELETE SET NULL -); - -COMMENT ON COLUMN edc_transfer_process.trace_context IS 'Java Map serialized as JSON'; - -COMMENT ON COLUMN edc_transfer_process.resource_manifest IS 'java ResourceManifest serialized as JSON'; - -COMMENT ON COLUMN edc_transfer_process.provisioned_resource_set IS 'ProvisionedResourceSet serialized as JSON'; - -COMMENT ON COLUMN edc_transfer_process.content_data_address IS 'DataAddress serialized as JSON'; - -COMMENT ON COLUMN edc_transfer_process.deprovisioned_resources IS 'List of deprovisioned resources, serialized as JSON'; - - -CREATE UNIQUE INDEX IF NOT EXISTS transfer_process_id_uindex - ON edc_transfer_process (transferprocess_id); - -CREATE TABLE IF NOT EXISTS edc_data_request -( - datarequest_id VARCHAR NOT NULL - CONSTRAINT data_request_pk - PRIMARY KEY, - process_id VARCHAR NOT NULL, - connector_address VARCHAR NOT NULL, - protocol VARCHAR NOT NULL, - connector_id VARCHAR, - asset_id VARCHAR NOT NULL, - contract_id VARCHAR NOT NULL, - data_destination JSON NOT NULL, - transfer_process_id VARCHAR NOT NULL - CONSTRAINT data_request_transfer_process_id_fk - REFERENCES edc_transfer_process - ON UPDATE RESTRICT ON DELETE CASCADE -); - - -COMMENT ON COLUMN edc_data_request.data_destination IS 'DataAddress serialized as JSON'; - -CREATE UNIQUE INDEX IF NOT EXISTS data_request_id_uindex - ON edc_data_request (datarequest_id); - -CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex - ON edc_lease (lease_id); \ No newline at end of file diff --git a/deployment/terraform/db-scripts/main.tf b/deployment/terraform/db-scripts/main.tf new file mode 100644 index 0000000..830a26c --- /dev/null +++ b/deployment/terraform/db-scripts/main.tf @@ -0,0 +1,6 @@ +resource "null_resource" "db-scripts" { + provisioner "local-exec" { + command = "${path.module}/db-scripts.sh" + interpreter = ["bash", "-c"] + } +} \ No newline at end of file diff --git a/deployment/terraform/db-scripts/policy-definition-store/schema.sql b/deployment/terraform/db-scripts/policy-definition-store/schema.sql new file mode 100644 index 0000000..d4ef812 --- /dev/null +++ b/deployment/terraform/db-scripts/policy-definition-store/schema.sql @@ -0,0 +1,41 @@ +-- +-- Copyright (c) 2022 ZF Friedrichshafen AG +-- +-- This program and the accompanying materials are made available under the +-- terms of the Apache License, Version 2.0 which is available at +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Contributors: +-- ZF Friedrichshafen AG - Initial SQL Query +-- + +-- Statements are designed for and tested with Postgres only! + +-- table: edc_policydefinitions +CREATE TABLE IF NOT EXISTS edc_policydefinitions +( + policy_id VARCHAR NOT NULL, + created_at BIGINT NOT NULL, + permissions JSON, + prohibitions JSON, + duties JSON, + extensible_properties JSON, + inherits_from VARCHAR, + assigner VARCHAR, + assignee VARCHAR, + target VARCHAR, + policy_type VARCHAR NOT NULL, + private_properties JSON, + PRIMARY KEY (policy_id) +); + +COMMENT ON COLUMN edc_policydefinitions.permissions IS 'Java List serialized as JSON'; +COMMENT ON COLUMN edc_policydefinitions.prohibitions IS 'Java List serialized as JSON'; +COMMENT ON COLUMN edc_policydefinitions.duties IS 'Java List serialized as JSON'; +COMMENT ON COLUMN edc_policydefinitions.extensible_properties IS 'Java Map serialized as JSON'; +COMMENT ON COLUMN edc_policydefinitions.policy_type IS 'Java PolicyType serialized as JSON'; + +CREATE UNIQUE INDEX IF NOT EXISTS edc_policydefinitions_id_uindex + ON edc_policydefinitions (policy_id); diff --git a/deployment/terraform/db-scripts/transfer-process-store/schema.sql b/deployment/terraform/db-scripts/transfer-process-store/schema.sql new file mode 100644 index 0000000..ab16436 --- /dev/null +++ b/deployment/terraform/db-scripts/transfer-process-store/schema.sql @@ -0,0 +1,70 @@ +-- Statements are designed for and tested with Postgres only! + +CREATE TABLE IF NOT EXISTS edc_lease +( + leased_by VARCHAR NOT NULL, + leased_at BIGINT, + lease_duration INTEGER NOT NULL, + lease_id VARCHAR NOT NULL + CONSTRAINT lease_pk + PRIMARY KEY +); + +COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; + +COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; + +CREATE TABLE IF NOT EXISTS edc_transfer_process +( + transferprocess_id VARCHAR NOT NULL + CONSTRAINT transfer_process_pk + PRIMARY KEY, + type VARCHAR NOT NULL, + state INTEGER NOT NULL, + state_count INTEGER DEFAULT 0 NOT NULL, + state_time_stamp BIGINT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + trace_context JSON, + error_detail VARCHAR, + resource_manifest JSON, + provisioned_resource_set JSON, + content_data_address JSON, + deprovisioned_resources JSON, + private_properties JSON, + callback_addresses JSON, + pending BOOLEAN DEFAULT FALSE, + transfer_type VARCHAR, + protocol_messages JSON, + data_plane_id VARCHAR, + correlation_id VARCHAR, + counter_party_address VARCHAR, + protocol VARCHAR, + asset_id VARCHAR, + contract_id VARCHAR, + data_destination JSON, + lease_id VARCHAR + CONSTRAINT transfer_process_lease_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL +); + +COMMENT ON COLUMN edc_transfer_process.trace_context IS 'Java Map serialized as JSON'; + +COMMENT ON COLUMN edc_transfer_process.resource_manifest IS 'java ResourceManifest serialized as JSON'; + +COMMENT ON COLUMN edc_transfer_process.provisioned_resource_set IS 'ProvisionedResourceSet serialized as JSON'; + +COMMENT ON COLUMN edc_transfer_process.content_data_address IS 'DataAddress serialized as JSON'; + +COMMENT ON COLUMN edc_transfer_process.deprovisioned_resources IS 'List of deprovisioned resources, serialized as JSON'; + + +CREATE UNIQUE INDEX IF NOT EXISTS transfer_process_id_uindex + ON edc_transfer_process (transferprocess_id); + +CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex + ON edc_lease (lease_id); + +-- This will help to identify states that need to be transitioned without a table scan when the entries grow +CREATE INDEX IF NOT EXISTS transfer_process_state ON edc_transfer_process (state,state_time_stamp); \ No newline at end of file diff --git a/deployment/terraform/deploy-services.sh b/deployment/terraform/deploy-services.sh index b858286..1d789f7 100755 --- a/deployment/terraform/deploy-services.sh +++ b/deployment/terraform/deploy-services.sh @@ -145,28 +145,22 @@ if [ "$TF_VAR_persistence_type" == "PostgreSQLaaS" ]; then fi if [ "$TF_VAR_persistence_type" == "PostgreSQL" ]; then - echo "Deploying postgres" - helm repo add bitnami https://charts.bitnami.com/bitnami - set +e - helm --kubeconfig=$TF_VAR_kubeconfig install postgres bitnami/postgresql -n $TF_VAR_namespace --set global.postgresql.auth.username=$TF_VAR_pg_username --set global.postgresql.auth.password=$TF_VAR_pg_password --set global.postgresql.auth.database=$TF_VAR_pg_database - set -e - - kubectl --kubeconfig=$TF_VAR_kubeconfig wait --for=condition=Ready=True pod -l app.kubernetes.io/name=postgresql -n $TF_VAR_namespace --timeout=600s + echo "Deploying postgresql" + # Create PostgreSQL instance + cd ../postgresql-deploy + terraform init + terraform apply -auto-approve - export TF_VAR_pg_host="postgres-postgresql" + export TF_VAR_pg_host="postgresql."$TF_VAR_namespace fi # Create the database if [ "$TF_VAR_persistence_type" == "PostgreSQLaaS" ] || [ "$TF_VAR_persistence_type" == "PostgreSQL" ]; then + echo "Running database scripts" + # Run scripts to create database schemas cd ../db-scripts - echo "Creating database $TF_VAR_pg_database" - set +e - kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-create-database --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="postgres" --command="CREATE DATABASE $TF_VAR_pg_database;" - set -e - - kubectl --kubeconfig=$TF_VAR_kubeconfig run -n $TF_VAR_namespace --timeout=120s -i postgres-restore-database --rm --image=postgres:latest --env="PGUSER=$TF_VAR_pg_username" --env="PGPASSWORD=$TF_VAR_pg_password" --env="PGHOST=$TF_VAR_pg_host" -- psql --dbname="$TF_VAR_pg_database" < ../db-scripts/init.sql -else - echo "WARNING: No persistence, the data will be lost if container pods are restarted" + terraform init + terraform apply -auto-approve fi echo "Deploying ionos s3" diff --git a/deployment/terraform/ionos-s3-deploy/main.tf b/deployment/terraform/ionos-s3-deploy/main.tf index 5e7ab83..4d06a3d 100644 --- a/deployment/terraform/ionos-s3-deploy/main.tf +++ b/deployment/terraform/ionos-s3-deploy/main.tf @@ -16,11 +16,6 @@ variable "ids_webhook_address" { default = "http://localhost:8282" } -variable "persistence_type" { - type = string - default = "None" -} - variable "image_repository" { type = string default = "ghcr.io/digital-ecosystems/connector" @@ -56,8 +51,6 @@ variable "pg_password" { default = "postgres" } -variable "s3_access_key" {} -variable "s3_secret_key" {} variable "s3_endpoint" {} variable "ionos_token" {} diff --git a/deployment/terraform/postgresql-deploy/main.tf b/deployment/terraform/postgresql-deploy/main.tf new file mode 100644 index 0000000..57c9a9c --- /dev/null +++ b/deployment/terraform/postgresql-deploy/main.tf @@ -0,0 +1,51 @@ +provider "helm" { + kubernetes { + config_path = "${var.kubeconfig}" + } +} + +variable "kubeconfig" { + type = string +} + +variable "namespace" { + default = "edc-ionos-s3" +} + +variable "pg_username" { + type = string + default = "postgres" +} + +variable "pg_password" { + type = string + default = "postgres" +} + +variable "pg_database" { + type = string + default = "postgres" +} + +resource "helm_release" "postgresql" { + name = "postgresql" + repository = "https://charts.bitnami.com/bitnami" + chart = "postgresql" + + namespace = var.namespace + + set { + name = "global.postgresql.auth.username" + value = var.pg_username + } + + set { + name = "global.postgresql.auth.password" + value = var.pg_password + } + + set { + name = "global.postgresql.auth.database" + value = var.pg_database + } +} \ No newline at end of file diff --git a/deployment/terraform/undeploy-services.sh b/deployment/terraform/undeploy-services.sh index 98ca353..ba093c8 100755 --- a/deployment/terraform/undeploy-services.sh +++ b/deployment/terraform/undeploy-services.sh @@ -21,11 +21,21 @@ cd ../ionos-s3-deploy terraform init terraform destroy -auto-approve -# Destroy Ionos Postgres Cluister +# Destroy ionos postgresql cluster cd ../ionos-postgresqlaas terraform init terraform destroy -auto-approve +# Destroy postgresql +cd ../postgresql-deploy +terraform init +terraform destroy -auto-approve + +# Destroy db-scripts +cd ../db-scripts +terraform init +terraform destroy -auto-approve + cd ../ # remove terraform state @@ -54,7 +64,18 @@ rm -f ./ionos-postgresqlaas/terraform.tfstate rm -f ./ionos-postgresqlaas/.terraform.lock.hcl rm -f ./ionos-postgresqlaas/terraform.tfstate.backup +rm -rf ./postgresql-deploy/.terraform +rm -f ./postgresql-deploy/terraform.tfstate +rm -f ./postgresql-deploy/.terraform.lock.hcl +rm -f ./postgresql-deploy/terraform.tfstate.backup + +rm -rf ./db-scripts/.terraform +rm -f ./db-scripts/terraform.tfstate +rm -f ./db-scripts/.terraform.lock.hcl +rm -f ./db-scripts/terraform.tfstate.backup + rm -f vault-init/vault-keys.json rm -f vault-init/vault-tokens.json -helm uninstall postgres -n $TF_VAR_namespace kubectl --kubeconfig $TF_VAR_kubeconfig delete namespace $TF_VAR_namespace + +echo "Undeployment complete" \ No newline at end of file diff --git a/launchers/prod/connector-persistence/README.md b/launchers/prod/connector-persistence/README.md index 866ce70..14e902d 100644 --- a/launchers/prod/connector-persistence/README.md +++ b/launchers/prod/connector-persistence/README.md @@ -25,9 +25,17 @@ Just check the `Configuration` section of the example [readme](../example/README Open the `resources/config.properties` file and insert the key and the secret of your IONOS S3 storage and the token. -### Import the initial database +### Create the initial database schemas ``` -psql -h -p -U < ../deployment/terraform/db-scripts/init.sql +psql -h -p -U < ../deployment/terraform/db-scripts/accesstokendata-store/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/asset-index/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/contract-definition-store/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/contract-negotiation-store/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/data-plane-instance-store/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/data-plane-store/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/edr-index/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/policy-definition-store/schema.sql +psql -h -p -U < ../deployment/terraform/db-scripts/transfer-process-store/schema.sql ``` ## Building and running the docker diff --git a/launchers/prod/connector-persistence/build.gradle.kts b/launchers/prod/connector-persistence/build.gradle.kts index 6cc09e9..0ee4d12 100644 --- a/launchers/prod/connector-persistence/build.gradle.kts +++ b/launchers/prod/connector-persistence/build.gradle.kts @@ -34,10 +34,15 @@ dependencies { implementation("${edcGroup}:transaction-local:$edcVersion") implementation("${edcGroup}:transaction-datasource-spi:$edcVersion") + implementation("${edcGroup}:accesstokendata-store-sql:$edcVersion") implementation("${edcGroup}:asset-index-sql:$edcVersion") - implementation("${edcGroup}:policy-definition-store-sql:$edcVersion") implementation("${edcGroup}:contract-definition-store-sql:$edcVersion") implementation("${edcGroup}:contract-negotiation-store-sql:$edcVersion") + implementation("${edcGroup}:control-plane-sql:$edcVersion") + implementation("${edcGroup}:data-plane-instance-store-sql:$edcVersion") + implementation("${edcGroup}:data-plane-store-sql:$edcVersion") + implementation("${edcGroup}:edr-index-sql:$edcVersion") + implementation("${edcGroup}:policy-definition-store-sql:$edcVersion") implementation("${edcGroup}:transfer-process-store-sql:$edcVersion") }