S3 Endpoint Regions Validation (#79)

* feat: s3 endpoint regions validation * chore: fixing assets doc * fix: launchers config
Digital-Ecosystems · Oct 8, 2024 · e62bf09 · e62bf09
1 parent 43ec592
commit e62bf09
Show file tree

Hide file tree

Showing 43 changed files with 396 additions and 444 deletions.
diff --git a/README.md b/README.md
@@ -59,15 +59,15 @@ The credentials can be found/configured in one of the following:
 
 It is required to configure those parameters:
 
-| Parameter name                       | Description                                                                                                                             | Mandatory                                            |
-|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------|
-| `edc.ionos.access.key`               | IONOS Access Key Id to access S3                                                                                                        | Yes if the context is accessing file                 |
-| `edc.ionos.secret.access.key`        | IONOS Secret Access Key to access S3                                                                                                    | Yes if the context is accessing file                 |
-| `edc.ionos.token`                    | IONOS token to allow S3 provisioning                                                                                                    | Yes if the context is provisioning access for others |
-| `edc.ionos.endpoint`                 | IONOS S3 endpoint address. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/endpoints) for further information.                                                                | Yes, if the context is accessing file                | No, the default value is                             |
-| `edc.ionos.max.files`                | Maximum number of files retrieved by list files function.                                                                               | No, the default value is 5,000 files                 |
-| `edc.ionos.key.validation.attempts`  | Maximum number of attemps to validate a temporary key after its creation.                                                               | No, the default values is 10 attempts                |
-| `edc.ionos.key.validation.delay`     | Time to wait (in milisseconds) before each key validation attempt. In each new attempt the delay is multiplied by the attempt number.   | No, the default value is 3,000 (3 seconds)           |
+| Parameter name                      | Description                                                                                                                                   | Mandatory                                            |
+|-------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------|
+| `edc.ionos.access.key`              | IONOS Access Key Id to access S3                                                                                                              | Yes if the context is accessing file                 |
+| `edc.ionos.secret.access.key`       | IONOS Secret Access Key to access S3                                                                                                          | Yes if the context is accessing file                 |
+| `edc.ionos.token`                   | IONOS token to allow S3 provisioning                                                                                                          | Yes if the context is provisioning access for others |
+| `edc.ionos.endpoint.region`         | IONOS S3 endpoint region. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/endpoints) for further information. | No, the default value is "de"                        |
+| `edc.ionos.max.files`               | Maximum number of files retrieved by list files function.                                                                                     | No, the default value is 5,000 files                 |
+| `edc.ionos.key.validation.attempts` | Maximum number of attemps to validate a temporary key after its creation.                                                                     | No, the default values is 10 attempts                |
+| `edc.ionos.key.validation.delay`    | Time to wait (in milisseconds) before each key validation attempt. In each new attempt the delay is multiplied by the attempt number.         | No, the default value is 3,000 (3 seconds)           |
 
 To create the token please take a look at the following [documentation](./ionos_token.md).
 

diff --git a/assets.md b/assets.md
@@ -9,13 +9,13 @@ The asset registration aims to specify which file/folder we want to share. We ca
 ### Requirements
 
 
-| Parameter        | Description                                                                                                                                                                                                                                                                                                                                                                         | Mandatory |
-|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|
-| `storage`        | IONOS S3 endpoint address. Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints)  for further information.                                                                                                                                                                                                                                 | yes       |
-| `bucketName`     | IONOS S3 bucket name.   Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information.                                                                                                                                                                                                                                 | yes       |
-| `blobName`       | File name or path to folder                                                                                                                                                                                                                                                                                                                                                         | yes       |
-| `filterIncludes` | `filterIncludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will be copied during the transfer <br/> * do not consider the blobName in the expression, but the path from it. example: blobName = folder1, filterIncludes=file1.csv, the file foloder1/file1.csv will be copied                                         | no        |
-| `filterExcludes` | `filterExcludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will NOT be copied during the transfer <br/> | no        |
+| Parameter        | Description                                                                                                                                                                                                                                                                                                                                 | Mandatory              |
+|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|
+| `region`         | IONOS S3 endpoint region. Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints)  for further information.| no, default value = de |
+| `bucketName`     | IONOS S3 bucket name.   Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information.| yes                    |
+| `blobName`       | File name or path to folder| yes                    |
+| `filterIncludes` | `filterIncludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will be copied during the transfer <br/> * do not consider the blobName in the expression, but the path from it. example: blobName = folder1, filterIncludes=file1.csv, the file foloder1/file1.csv will be copied| no                     |
+| `filterExcludes` | `filterExcludes` use regular expression that will be used to select the file name pattern from the asset's blobName that will NOT be copied during the transfer <br/>| no                     |
 
 Note:  if `filterIncludes` and  `filterExcludes` parameters are satisfied, the files to be copied will be selected using the `filterIncludes` and after that selected list, the files that have the pattern defined in the `filterExcludes` will be ignored.
 
@@ -25,7 +25,7 @@ Note:  if `filterIncludes` and  `filterExcludes` parameters are satisfied, the f
 ```json
 "dataAddress":{
   "type": "IonosS3", //from EDC
-  "storage": "s3-eu-central-1.ionoscloud.com",
+  "region": "de",
   "bucketName": "mybucket",
   "blobName": "folder1/",
   "filterIncludes": "file1.csv",
@@ -40,19 +40,19 @@ The transfer of assets aims to transfer the files/folders from one connector to
 ### Requirements
 
 
-| Parameter    | Description                                                                                                                                         | Mandatory |
-|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------------|-----------|
-| `storage`    | IONOS S3 endpoint address. Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints)  for further information. | yes       |
-| `bucketName` | IONOS S3 bucket name.   Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information. | yes       |
-| `path`       | Path of destination where the file/folder will be placed. </br>  *if the path not filled, the file will be placed in the root of the bucket.        | no        |
+| Parameter     | Description                                                                                                                                  | Mandatory              |
+|---------------|----------------------------------------------------------------------------------------------------------------------------------------------|------------------------|
+| `region`      | IONOS S3 endpoint region. Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/s3-endpoints)  for further information.| no, default value = de |
+| `bucketName`  | IONOS S3 bucket name.   Refer to  [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/concepts/buckets) for further information| yes                    |
+| `path`        | Path of destination where the file/folder will be placed. </br>  *if the path not filled, the file will be placed in the root of the bucket. | no                     |
 
 
 ## Example
 
 ```json
 "dataDestination":{
   "type": "IonosS3", //from EDC
-  "storage": "s3-eu-central-1.ionoscloud.com",
+  "region": "de",
   "bucketName": "mybucket",
   "path": "folder2/",
   "keyName": "mykey" //from EDC

diff --git a/deployment/README.md b/deployment/README.md
@@ -48,7 +48,7 @@ export TF_VAR_persistence_type='PostgreSQLaaS' # 'PostgreSQLaaS', 'PostgreSQL' o
 export TF_VAR_vaultname='vault'  # optional if only 1 connector per cluster
 export TF_VAR_s3_access_key='' # S3 access key
 export TF_VAR_s3_secret_key='' # S3 secret key
-export TF_VAR_s3_endpoint='' # s3 endpoint (e.g. s3-eu-central-1.ionoscloud.com)
+export TF_VAR_s3_endpoint_region='' # s3 endpoint region (e.g. de)
 export TF_VAR_ionos_token='' # IONOS Cloud token, for further information: https://docs.ionos.com/cloud/managed-services/s3-object-storage/endpoints
 
 # Required only if persistence_type is PostgreSQLaaS
@@ -86,13 +86,13 @@ In case you want to configure this Connector without Hashicorp Vault, you need t
 
 ```yaml
   ionos:
-    endpoint: <YOUR-S3-ENDPOINT>
+    region: <YOUR-S3-ENDPOINT-REGION>
     accessKey: <YOUR-KEY>
     secretKey: <YOUR-SECRET-KEY>
     token: <IONOS-TOKEN>
 ```
 
-They should be the same as the ones set in the environment variables. The **ionos.endpoint** is set to the default S3 location, but it can be changed to any other location.
+They should be the same as the ones set in the environment variables. The **ionos.region** is set to the default S3 endpoint region, but it can be changed to any other location.
 
 
 If you don't want the Connector to be externally accessible, you need to set the following parameters in the helm [values.yaml](deployment/helm/edc-ionos-s3/values.yaml):

diff --git a/deployment/helm/README.md b/deployment/helm/README.md
@@ -35,7 +35,7 @@ The IONOS S3 Extension can be deployed to a Kubernetes cluster using the Helm ch
     # Add secrets to Vault
     kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.access.key content=
     kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.secret.key content=
-    kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.endpoint content=
+    kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.endpoint.region content=
     kubectl exec --namespace edc-ionos-s3 -it vault-0 -- vault kv put secret/edc.ionos.token content=
     ```
 

diff --git a/deployment/helm/edc-ionos-s3/templates/configmap.yaml b/deployment/helm/edc-ionos-s3/templates/configmap.yaml
@@ -26,7 +26,7 @@ data:
     edc.vault.hashicorp.timeout.seconds={{ .Values.edc.vault.hashicorp.timeout.seconds }}
     edc.ionos.access.key={{ .Values.edc.ionos.accessKey }}
     edc.ionos.secret.key={{ .Values.edc.ionos.secretKey }}
-    edc.ionos.endpoint={{ .Values.edc.ionos.endpoint }}
+    edc.ionos.endpoint.region={{ .Values.edc.ionos.region }}
     edc.ionos.token={{ .Values.edc.ionos.token }}
 
     {{- if eq .Values.edc.persistenceType "PostgreSQLaaS" }}

diff --git a/deployment/helm/edc-ionos-s3/values.yaml b/deployment/helm/edc-ionos-s3/values.yaml
@@ -124,7 +124,7 @@ edc:
       timeout:
         seconds: 30
   ionos:
-    endpoint: s3-eu-central-1.ionoscloud.com
+    region: de
     accessKey: notnull
     secretKey: notnull
     token: notnull

diff --git a/deployment/kind/README.md b/deployment/kind/README.md
@@ -26,7 +26,7 @@ Set environment variables
 export KUBECONFIG=path/to/kubeconfig
 export S3_ACCESS_KEY=''
 export S3_SECRET_KEY=''
-export S3_ENDPOINT=''
+export S3_ENDPOINT_REGION=''
 export IONOS_TOKEN=''
 ```
 

diff --git a/deployment/kind/scripts/deploy.sh b/deployment/kind/scripts/deploy.sh
@@ -34,8 +34,8 @@ if [ -z `printenv S3_SECRET_KEY` ]; then
     echo "Stopping because S3_SECRET_KEY is undefined"
     exit 1
 fi 
-if [ -z `printenv S3_ENDPOINT` ]; then
-    echo "Stopping because S3_ENDPOINT is undefined"
+if [ -z `printenv S3_ENDPOINT_REGION` ]; then
+    echo "Stopping because S3_ENDPOINT_REGION is undefined"
     exit 1
 fi
 if [ -z `printenv IONOS_TOKEN` ]; then
@@ -72,7 +72,7 @@ helm install -n edc-ionos-s3 --wait vault hashicorp/vault \
 export TF_VAR_kubeconfig=$KUBECONFIG
 export TF_VAR_s3_access_key=$S3_ACCESS_KEY
 export TF_VAR_s3_secret_key=$S3_SECRET_KEY
-export TF_VAR_s3_endpoint=$S3_ENDPOINT
+export TF_VAR_s3_endpoint_region=$S3_ENDPOINT_REGION
 export TF_VAR_ionos_token=$IONOS_TOKEN
 ../terraform/vault-init/vault-init.sh
 

diff --git a/deployment/terraform/deploy-services.sh b/deployment/terraform/deploy-services.sh
@@ -19,8 +19,8 @@ if [[ -z `printenv TF_VAR_s3_secret_key` ]]; then
     exit 1
 fi
 
-if [[ -z `printenv TF_VAR_s3_endpoint` ]]; then
-    echo "Stopping because TF_VAR_s3_endpoint is undefined"
+if [[ -z `printenv TF_VAR_s3_endpoint_region` ]]; then
+    echo "Stopping because TF_VAR_s3_endpoint_region is undefined"
     exit 1
 fi
 

diff --git a/deployment/terraform/ionos-s3-deploy/main.tf b/deployment/terraform/ionos-s3-deploy/main.tf
@@ -51,7 +51,7 @@ variable "pg_password" {
   default = "postgres"
 }
 
-variable "s3_endpoint" {}
+variable "s3_endpoint_region" {}
 variable "ionos_token" {}
 
 variable "vaultname" {
@@ -86,8 +86,8 @@ resource "helm_release" "edc-ionos-s3" {
   }
 
   set {
-    name  = "edc.ionos.endpoint"
-    value = var.s3_endpoint
+    name  = "edc.ionos.endpoint.region"
+    value = var.s3_endpoint_region
   }
 
   set {

diff --git a/deployment/terraform/vault-init/vault-init.sh b/deployment/terraform/vault-init/vault-init.sh
@@ -41,7 +41,7 @@ kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR
 # Add secrets to Vault
 kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.access.key content=$TF_VAR_s3_access_key
 kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.secret.key content=$TF_VAR_s3_secret_key
-kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.endpoint content=$TF_VAR_s3_endpoint
+kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.endpoint.region content=$TF_VAR_s3_endpoint_region
 kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.ionos.token content=$TF_VAR_ionos_token
 
 kubectl --kubeconfig=$TF_VAR_kubeconfig exec --namespace $NAMESPACE -it "$TF_VAR_vaultname-0" -- vault kv put secret/edc.connector.private.key content="$(cat ./certs/private.pem)"

diff --git a/extensions/core-ionos-s3/build.gradle.kts b/extensions/core-ionos-s3/build.gradle.kts
@@ -9,7 +9,6 @@ val metaModelVersion: String by project
 val minIOVersion: String by project
 val extensionsGroup: String by project
 val extensionsVersion: String by project
-val junitVersion: String by project
 
 val gitHubPkgsName: String by project
 val gitHubPkgsUrl: String by project
@@ -21,21 +20,13 @@ dependencies {
 
 	implementation("${edcGroup}:transfer-spi:${edcVersion}")
 	implementation("io.minio:minio:${minIOVersion}")
-
-	testImplementation("${edcGroup}:junit:${edcVersion}")
-	testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}")
-	testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}")
 }
 
 java {
 	withJavadocJar()
 	withSourcesJar()
 }
 
-tasks.test {
-	useJUnitPlatform()
-}
-
 publishing {
 	publications {
 		create<MavenPublication>("maven") {