Migrating to EDC 0.7.2 (#77)

* feat: migrating to edc 0.7.3 * feat: migrating to edc 0.7.2 * feat: migrating to edc 0.7.2 * feat: migrating to edc 0.7.2 * feat: migrating to edc 0.7.2 * feat: migrating to edc 0.7.2
Digital-Ecosystems · Sep 25, 2024 · 861bee7 · 861bee7
1 parent 6f982f5
commit 861bee7
Show file tree

Hide file tree

Showing 57 changed files with 248 additions and 1,002 deletions.
diff --git a/.gitignore b/.gitignore
@@ -51,6 +51,7 @@ launchers/demo-e2e/edc-config.properties
 *.hprof
 
 **/vault-keys.json
+**/vault-tokens.json
 runtime_settings.properties
 generated_backend.tf
 

diff --git a/deployment/README.md b/deployment/README.md
@@ -110,21 +110,19 @@ This will allocate a public IP address to the Connector. You can then access it
 
 All commands paths are relative to the current directory where this readme is located.
 
-### 1. Install the EDC Ionos S3 services
+### 1. Deploy the services
 
-To install the services run the script ```deploy-services.sh``` in ```terraform``` directory.
+To deploy the services run the script ```deploy-services.sh``` in ```terraform``` directory.
 
 ```sh
 cd terraform
 ./deploy-services.sh
 ```
+### 2. Undeploy the services
 
-### 2. Vault keys
-After the services are installed you will have ```vault-keys.json``` file containing the vault keys in ```terraform``` directory.
-
-### 3. Destroy the services
+To undeploy the services run the script ```undeploy-services.sh``` in ```terraform``` directory.
 
 ```sh
 cd terraform
-./destroy-services.sh
+./undeploy-services.sh
 ```
diff --git a/deployment/helm/edc-ionos-s3/templates/configmap.yaml b/deployment/helm/edc-ionos-s3/templates/configmap.yaml
@@ -4,6 +4,7 @@ metadata:
   name: {{ include "edc-ionos-s3.fullname" . }}-config
 data:
   config.properties: |
+    edc.participant.id={{ .Values.edc.participant.id }}
     web.http.port={{ .Values.web.http.port }}
     web.http.path={{ .Values.web.http.path }}
     web.http.management.port={{ .Values.web.http.management.port }}
@@ -14,20 +15,19 @@ data:
     web.http.public.path={{ .Values.web.http.public.path }}
     web.http.control.port={{ .Values.web.http.control.port }}
     web.http.control.path={{ .Values.web.http.control.path }}
+    edc.dsp.callback.address={{ .Values.edc.dsp.callback.address }}
+    edc.dataplane.token.validation.endpoint={{ .Values.edc.dataplane.token.validation.endpoint }}
+    edc.dataplane.api.public.baseurl={{ .Values.edc.dataplane.api.public.baseurl }}
     edc.api.auth.key={{ .Values.edc.api.auth.key }}
-    edc.participant.id={{ .Values.edc.participant.id }}
+    edc.transfer.proxy.token.signer.privatekey.alias={{ .Values.edc.vault.certificates.privateKey.alias }}
+    edc.transfer.proxy.token.verifier.publickey.alias={{ .Values.edc.vault.certificates.publicKey.alias }}
+    edc.vault.hashicorp.url={{ .Values.edc.vault.hashicorp.url }}
+    edc.vault.hashicorp.token={{ .Values.edc.vault.hashicorp.token }}
+    edc.vault.hashicorp.timeout.seconds={{ .Values.edc.vault.hashicorp.timeout.seconds }}
     edc.ionos.access.key={{ .Values.edc.ionos.accessKey }}
     edc.ionos.secret.key={{ .Values.edc.ionos.secretKey }}
     edc.ionos.endpoint={{ .Values.edc.ionos.endpoint }}
     edc.ionos.token={{ .Values.edc.ionos.token }}
-    edc.vault.hashicorp.url={{ .Values.edc.vault.hashicorp.url }}
-    edc.vault.hashicorp.token={{ .Values.edc.vault.hashicorp.token }}
-    edc.vault.hashicorp.timeout.seconds={{ .Values.edc.vault.hashicorp.timeout.seconds }}
-    edc.ids.id={{ .Values.edc.ids.id }}
-    edc.dsp.callback.address={{ .Values.edc.dsp.callback.address }}:{{ .Values.web.http.protocol.port }}{{ .Values.web.http.protocol.path }}
-    edc.receiver.http.endpoint={{ .Values.edc.receiver.http.endpoint }}/receiver/{{ .Values.edc.ids.id }}/callback
-    edc.public.key.alias={{ .Values.edc.public.key.alias }}
-    edc.dataplane.token.validation.endpoint={{ .Values.edc.dataplane.token.validation.endpoint }}:{{ .Values.web.http.control.port }}{{ .Values.web.http.control.path }}/token
 
     {{- if eq .Values.edc.persistenceType "PostgreSQLaaS" }}
     edc.datasource.asset.name=asset

diff --git a/deployment/helm/edc-ionos-s3/values.yaml b/deployment/helm/edc-ionos-s3/values.yaml
@@ -113,34 +113,31 @@ edc:
     auth:
       key: password
   vault:
-    clientid: company1
-    tenantid: 1
-    certificate: /resources/
+    certificates:
+      publicKey:
+        alias: edc.connector.public.key
+      privateKey:
+        alias: edc.connector.private.key
     hashicorp:
       url: http://vault:8200
       token:
       timeout:
         seconds: 30
-  ids:
-    id: urn:connector:provider
   ionos:
     endpoint: s3-eu-central-1.ionoscloud.com
     accessKey: notnull
     secretKey: notnull
     token: notnull
   dsp:
     callback:
-      address: http://localhost
-  receiver:
-    http:
-      endpoint: http://localhost:4000
-  public:
-    key:
-      alias: alias
+      address: http://localhost:8281/protocol
   dataplane:
+    api:
+      public:
+        baseurl: http://localhost:8282/public
     token:
       validation:
-        endpoint: http://localhost
+        endpoint: http://localhost:8283/control/token
   persistenceType: PostgreSQLaaS # 'PostgreSQLaaS', 'PostgreSQL' or 'None'
   postgresql: # Only used if persistenceType is 'PostgreSQLaaS' or 'PostgreSQL'
     host: postgresql

diff --git a/deployment/terraform/clean-state.sh b/deployment/terraform/clean-state.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# remove terraform state
+rm -rf ./configure-public-address/.terraform
+rm -f ./configure-public-address/terraform.tfstate
+rm -f ./configure-public-address/.terraform.lock.hcl
+rm -f ./configure-public-address/terraform.tfstate.backup
+
+rm -rf ./ionos-s3-deploy/.terraform
+rm -f ./ionos-s3-deploy/terraform.tfstate
+rm -f ./ionos-s3-deploy/.terraform.lock.hcl
+rm -f ./ionos-s3-deploy/terraform.tfstate.backup
+
+rm -rf ./vault-init/.terraform
+rm -f ./vault-init/terraform.tfstate
+rm -f ./vault-init/.terraform.lock.hcl
+rm -f ./vault-init/terraform.tfstate.backup
+
+rm -rf ./vault-deploy/.terraform
+rm -f ./vault-deploy/terraform.tfstate
+rm -f ./vault-deploy/.terraform.lock.hcl
+rm -f ./vault-deploy/terraform.tfstate.backup
+
+rm -rf ./ionos-postgresqlaas/.terraform
+rm -f ./ionos-postgresqlaas/terraform.tfstate
+rm -f ./ionos-postgresqlaas/.terraform.lock.hcl
+rm -f ./ionos-postgresqlaas/terraform.tfstate.backup
+
+rm -f vault-init/vault-keys.json
+rm -f vault-init/vault-tokens.json
diff --git a/deployment/terraform/configure-public-address/public-addresses.sh b/deployment/terraform/configure-public-address/public-addresses.sh
@@ -12,9 +12,9 @@ fi
 # Change public address in the config.properties in the configmap
 kubectl --kubeconfig=$TF_VAR_kubeconfig -n $TF_VAR_namespace get configmap edc-ionos-s3-config -o yaml | sed "s/edc.dsp.callback.address=.*/edc.dsp.callback.address=http:\/\/$CONNECTOR_ADDRESS:8281\/protocol/g" | kubectl --kubeconfig=$TF_VAR_kubeconfig apply -f -
 
-kubectl --kubeconfig=$TF_VAR_kubeconfig -n $TF_VAR_namespace get configmap edc-ionos-s3-config -o yaml | sed "s/edc.receiver.http.endpoint=.*/edc.receiver.http.endpoint=http:\/\/$CONNECTOR_ADDRESS:4000\/receiver\/urn:connector:provider\/callback/g" | kubectl --kubeconfig=$TF_VAR_kubeconfig apply -f -
-
 kubectl --kubeconfig=$TF_VAR_kubeconfig -n $TF_VAR_namespace get configmap edc-ionos-s3-config -o yaml | sed "s/edc.dataplane.token.validation.endpoint=.*/edc.dataplane.token.validation.endpoint=http:\/\/$CONNECTOR_ADDRESS:8283\/control\/token/g" | kubectl --kubeconfig=$TF_VAR_kubeconfig apply -f -
 
+kubectl --kubeconfig=$TF_VAR_kubeconfig -n $TF_VAR_namespace get configmap edc-ionos-s3-config -o yaml | sed "s/edc.dataplane.api.public.baseurl=.*/edc.dataplane.api.public.baseurl=http:\/\/$CONNECTOR_ADDRESS:8282\/public/g" | kubectl --kubeconfig=$TF_VAR_kubeconfig apply -f -
+
 # Restart the pods
 kubectl --kubeconfig=$TF_VAR_kubeconfig -n $TF_VAR_namespace delete pod -l app.kubernetes.io/name=edc-ionos-s3
diff --git a/deployment/terraform/ionos-s3-deploy/main.tf b/deployment/terraform/ionos-s3-deploy/main.tf
@@ -66,7 +66,7 @@ variable "vaultname" {
 }
 
 locals {
-  root_token = fileexists("../vault-init/vault-keys.json") ? "${jsondecode(file("../vault-init/vault-keys.json")).root_token}" : ""
+  vault_token = fileexists("../vault-init/vault-tokens.json") ? "${jsondecode(file("../vault-init/vault-tokens.json")).auth.client_token}" : ""
 }
 
 resource "helm_release" "edc-ionos-s3" {
@@ -80,7 +80,7 @@ resource "helm_release" "edc-ionos-s3" {
 
   set {
     name  = "edc.vault.hashicorp.token"
-    value = "${jsondecode(file("../vault-init/vault-keys.json")).root_token}"
+    value = local.vault_token
   }
 
   values = [
@@ -92,11 +92,6 @@ resource "helm_release" "edc-ionos-s3" {
     value = "http://${var.vaultname}:8200"
   }
 
-  set {
-    name  = "edc.vault.hashicorp.token"
-    value = local.root_token
-  }
-
   set {
     name  = "edc.ionos.endpoint"
     value = var.s3_endpoint

diff --git a/deployment/terraform/destroy-services.sh → deployment/terraform/undeploy-services.sh b/deployment/terraform/destroy-services.sh → deployment/terraform/undeploy-services.sh
@@ -55,5 +55,6 @@ rm -f ./ionos-postgresqlaas/.terraform.lock.hcl
 rm -f ./ionos-postgresqlaas/terraform.tfstate.backup
 
 rm -f vault-init/vault-keys.json
+rm -f vault-init/vault-tokens.json
 helm uninstall postgres -n $TF_VAR_namespace
 kubectl --kubeconfig $TF_VAR_kubeconfig delete namespace $TF_VAR_namespace
diff --git a/deployment/terraform/vault-deploy/main.tf b/deployment/terraform/vault-deploy/main.tf
@@ -32,7 +32,7 @@ resource "helm_release" "vault" {
 
   repository = "https://helm.releases.hashicorp.com"
   chart      = "vault"
-  version = "v0.19.0"
+  version    = "v0.28.1"
 
   namespace = var.namespace
   create_namespace = true

diff --git a/deployment/terraform/vault-init/certs/private.pem b/deployment/terraform/vault-init/certs/private.pem
@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIARDUGJgKy1yzxkueIJ1k3MPUWQ/tbQWQNqW6TjyHpdcoAoGCCqGSM49
+AwEHoUQDQgAE1l0Lof0a1yBc8KXhesAnoBvxZw5roYnkAXuqCYfNK3ex+hMWFuiX
+GUxHlzShAehR6wvwzV23bbC0tcFcVgW//A==
+-----END EC PRIVATE KEY-----
diff --git a/deployment/terraform/vault-init/certs/public.pem b/deployment/terraform/vault-init/certs/public.pem
@@ -0,0 +1,4 @@
+-----BEGIN PUBLIC KEY-----
+MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1l0Lof0a1yBc8KXhesAnoBvxZw5r
+oYnkAXuqCYfNK3ex+hMWFuiXGUxHlzShAehR6wvwzV23bbC0tcFcVgW//A==
+-----END PUBLIC KEY-----
diff --git a/deployment/terraform/vault-init/vault-init.sh b/deployment/terraform/vault-init/vault-init.sh
@@ -30,14 +30,19 @@ kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR
 # Login to Vault
 kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault login $(jq -r ".root_token" vault-keys.json)
 
-
 if [[ "$INITIALIZED" == "false" ]]; then
    # Enable KV secrets engine
    kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault secrets enable -version=2 -path=secret kv
 fi
 
+## Create connector token
+kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault token create -policy=root -renewable=true -ttl=300s -format=json > vault-tokens.json
+
 # Add secrets to Vault
 kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.access.key content=$TF_VAR_s3_access_key
 kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.secret.key content=$TF_VAR_s3_secret_key
 kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.endpoint content=$TF_VAR_s3_endpoint
 kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.token content=$TF_VAR_ionos_token
+
+kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.connector.private.key content="$(cat ./certs/private.pem)"
+kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.connector.public.key content="$(cat ./certs/public.pem)"
diff --git a/deployment/terraform/vault-keys.json b/deployment/terraform/vault-keys.json
diff --git a/extensions/build.gradle.kts b/extensions/build.gradle.kts
@@ -3,12 +3,6 @@ plugins {
   `maven-publish`
 }
 
-repositories {
-  mavenLocal()
-  mavenCentral()
-
-}
-
 configure<PublishingExtension> {
   publications {
     withType(MavenPublication::class.java) {
@@ -21,6 +15,11 @@ configure<PublishingExtension> {
           }
         }
         developers {
+          developer {
+            id.set("jannotti-glaucio")
+            name.set("Glaucio Jannotti")
+            email.set("[email protected]")
+          }
           developer {
             id.set("paulolory-ionos")
             name.set("Paulo Lory")

diff --git a/extensions/core-ionos-s3/build.gradle.kts b/extensions/core-ionos-s3/build.gradle.kts
@@ -22,6 +22,7 @@ dependencies {
 	implementation("${edcGroup}:transfer-spi:${edcVersion}")
 	implementation("io.minio:minio:${minIOVersion}")
 
+	testImplementation("${edcGroup}:junit:${edcVersion}")
 	testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}")
 	testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}")
 }

diff --git a/...ions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/IonosToken.java b/...ions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/IonosToken.java
@@ -17,7 +17,7 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 
-import org.eclipse.edc.connector.transfer.spi.types.SecretToken;
+import org.eclipse.edc.connector.controlplane.transfer.spi.types.SecretToken;
 
 @JsonTypeName("dataspaceconnector:ionostoken")
 public class IonosToken implements SecretToken {

diff --git a/...ions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java b/...ions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java
@@ -14,7 +14,9 @@
 
 package com.ionos.edc.extension.s3.schema;
 
-import static org.eclipse.edc.spi.CoreConstants.EDC_NAMESPACE;
+import org.eclipse.edc.spi.types.domain.transfer.FlowType;
+
+import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE;
 
 public interface IonosBucketSchema {
     String TYPE = "IonosS3";
@@ -27,5 +29,7 @@ public interface IonosBucketSchema {
     String ACCESS_KEY_ID = EDC_NAMESPACE + "accessKey";
     String SECRET_ACCESS_KEY = EDC_NAMESPACE + "secretKey";
 
+    String PUSH_TRANSFER_TYPE = TYPE + "-" + FlowType.PUSH;
+
     String STORAGE_NAME_DEFAULT = "https://s3-eu-central-1.ionoscloud.com";
 }
diff --git a/extensions/data-plane-ionos-s3/build.gradle.kts b/extensions/data-plane-ionos-s3/build.gradle.kts
@@ -3,9 +3,9 @@ plugins {
     `maven-publish`
 }
 
-val javaVersion: String by project
 val edcGroup: String by project
 val edcVersion: String by project
+val metaModelVersion: String by project
 val extensionsGroup: String by project
 val extensionsVersion: String by project
 val junitVersion: String by project
@@ -17,16 +17,16 @@ val gitHubUser: String? by project
 val gitHubToken: String? by project
 
 dependencies {
-	api("${edcGroup}:data-plane-spi:${edcVersion}")
+	api("${edcGroup}:runtime-metamodel:${metaModelVersion}")
 
-	implementation(project(":extensions:core-ionos-s3"))
-	implementation("${edcGroup}:util:${edcVersion}")
+	implementation("${edcGroup}:util-lib:${edcVersion}")
 	implementation("${edcGroup}:transfer-spi:${edcVersion}")
-	implementation("${edcGroup}:data-plane-util:${edcVersion}")
-    implementation("${edcGroup}:data-plane-core:${edcVersion}")
-	implementation("${edcGroup}:http:${edcVersion}")
 	implementation("${edcGroup}:validator-spi:${edcVersion}")
-
+	implementation("${edcGroup}:data-plane-util:${edcVersion}")
+
+	implementation(project(":extensions:core-ionos-s3"))
+
+	testImplementation("${edcGroup}:junit:${edcVersion}")
 	testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}")
 	testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}")
 	testImplementation("org.mockito:mockito-core:${mockitoVersion}")

diff --git a/...a-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java b/...a-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java
@@ -27,7 +27,7 @@
 import org.eclipse.edc.spi.security.Vault;
 import org.eclipse.edc.spi.types.TypeManager;
 import org.eclipse.edc.spi.types.domain.DataAddress;
-import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest;
+import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage;
 import org.eclipse.edc.validator.spi.Validator;
 import org.eclipse.edc.validator.spi.ValidationResult;
 import org.jetbrains.annotations.NotNull;
@@ -57,18 +57,18 @@ public IonosDataSinkFactory(S3ConnectorApi s3Api, ExecutorService executorServic
     }
 
     @Override
-    public boolean canHandle(DataFlowRequest request) {
-        return IonosBucketSchema.TYPE.equals(request.getDestinationDataAddress().getType());
+    public String supportedType() {
+        return IonosBucketSchema.TYPE;
     }
 
     @Override
-    public @NotNull Result<Void> validateRequest(DataFlowRequest request) {
+    public @NotNull Result<Void> validateRequest(DataFlowStartMessage request) {
         var destination = request.getDestinationDataAddress();
         return validator.validate(destination).flatMap(ValidationResult::toResult);
     }
 
     @Override
-    public DataSink createSink(DataFlowRequest request) {
+    public DataSink createSink(DataFlowStartMessage request) {
 
         var validationResult = validateRequest(request);
         if (validationResult.failed()) {