diff --git a/README.md b/README.md index ba9c9d0b..4fef6bf0 100644 --- a/README.md +++ b/README.md @@ -59,12 +59,15 @@ The credentials can be found/configured in one of the following: It is required to configure those parameters: -| Parameter name | Description | Mandatory | -|-----------------------------------------|----------------------------------------| ---------- | -| `edc.ionos.access.key` | IONOS Access Key Id to access S3 | Yes if the context is accessing file | -| `edc.ionos.secret.access.key` | IONOS Secret Access Key to access S3 | Yes if the context is accessing file | -| `edc.ionos.token` | IONOS token to allow S3 provisioning | Yes if the context is provisioning access for others | -| `edc.ionos.endpoint` | IONOS S3 endpoint address. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/endpoints) for further information. | Yes, if the context is accessing file | +| Parameter name | Description | Mandatory | +|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------| +| `edc.ionos.access.key` | IONOS Access Key Id to access S3 | Yes if the context is accessing file | +| `edc.ionos.secret.access.key` | IONOS Secret Access Key to access S3 | Yes if the context is accessing file | +| `edc.ionos.token` | IONOS token to allow S3 provisioning | Yes if the context is provisioning access for others | +| `edc.ionos.endpoint` | IONOS S3 endpoint address. Refer to [docs](https://docs.ionos.com/cloud/managed-services/s3-object-storage/endpoints) for further information. | Yes, if the context is accessing file | No, the default value is | +| `edc.ionos.max.files` | Maximum number of files retrieved by list files function. | No, the default value is 5,000 files | +| `edc.ionos.key.validation.attempts` | Maximum number of attemps to validate a temporary key after its creation. | No, the default values is 10 attempts | +| `edc.ionos.key.validation.delay` | Time to wait (in milisseconds) before each key validation attempt. In each new attempt the delay is multiplied by the attempt number. | No, the default value is 3,000 (3 seconds) | To create the token please take a look at the following [documentation](./ionos_token.md). diff --git a/extensions/core-ionos-s3/build.gradle.kts b/extensions/core-ionos-s3/build.gradle.kts index 13d735e3..3051ff77 100644 --- a/extensions/core-ionos-s3/build.gradle.kts +++ b/extensions/core-ionos-s3/build.gradle.kts @@ -9,6 +9,7 @@ val metaModelVersion: String by project val minIOVersion: String by project val extensionsGroup: String by project val extensionsVersion: String by project +val junitVersion: String by project val gitHubPkgsName: String by project val gitHubPkgsUrl: String by project @@ -20,6 +21,9 @@ dependencies { implementation("${edcGroup}:transfer-spi:${edcVersion}") implementation("io.minio:minio:${minIOVersion}") + + testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}") + testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}") } java { @@ -27,6 +31,10 @@ java { withSourcesJar() } +tasks.test { + useJUnitPlatform() +} + publishing { publications { create("maven") { diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java index 97644a88..9b7d3674 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java @@ -14,10 +14,9 @@ package com.ionos.edc.extension.s3.api; +import com.ionos.edc.extension.s3.connector.ionosapi.S3AccessKey; import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint; -import com.ionos.edc.extension.s3.connector.ionosapi.TemporaryKey; - import java.io.ByteArrayInputStream; import java.util.List; @@ -36,8 +35,10 @@ public interface S3ConnectorApi { List listObjects(String bucketName, String objectName); - TemporaryKey createTemporaryKey(); - - void deleteTemporaryKey(String accessKey); + S3AccessKey createAccessKey(); + + S3AccessKey retrieveAccessKey(String keyID); + + void deleteAccessKey(String keyID); } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java index 5591583a..4f45c7c7 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java @@ -15,8 +15,8 @@ package com.ionos.edc.extension.s3.api; import com.ionos.edc.extension.s3.connector.MinioConnector; -import com.ionos.edc.extension.s3.connector.ionosapi.HttpConnector; -import com.ionos.edc.extension.s3.connector.ionosapi.TemporaryKey; +import com.ionos.edc.extension.s3.connector.ionosapi.S3AccessKey; +import com.ionos.edc.extension.s3.connector.ionosapi.S3ApiConnector; import io.minio.BucketExistsArgs; import io.minio.GetObjectArgs; @@ -34,16 +34,15 @@ public class S3ConnectorApiImpl implements S3ConnectorApi { MinioConnector miniConnector = new MinioConnector(); - HttpConnector ionosApi = new HttpConnector(); + S3ApiConnector ionoss3Api = new S3ApiConnector(); - private MinioClient minioClient; + private final MinioClient minioClient; private final String region; private String token; private final Integer maxFiles; public S3ConnectorApiImpl(String endpoint, String accessKey, String secretKey, int maxFiles) { - if(accessKey != null && secretKey != null && endpoint != null) - this.minioClient = miniConnector.connect(endpoint, accessKey, secretKey); + this.minioClient = miniConnector.connect(endpoint, accessKey, secretKey); this.region = getRegion(endpoint); this.token = ""; this.maxFiles = maxFiles; @@ -54,7 +53,6 @@ public S3ConnectorApiImpl(String endpoint, String accessKey, String secretKey, S this.token = token; } - @Override public void createBucket(String bucketName) { if (!bucketExists(bucketName.toLowerCase())) { @@ -163,35 +161,49 @@ public List listObjects(String bucketName, String objectName) { } @Override - public TemporaryKey createTemporaryKey() { + public S3AccessKey createAccessKey() { try{ - return ionosApi.createTemporaryKey(token); + return ionoss3Api.createAccessKey(token); } catch (Exception e) { throw new EdcException("Creating temporary key - (Warning: max 5 keys on the storage) - " + e.getMessage()); } } + + @Override + public S3AccessKey retrieveAccessKey(String keyID) { + try{ + return ionoss3Api.retrieveAccessKey(token, keyID); + } catch (Exception e) { + throw new EdcException("Retrieving temporary key: " + e.getMessage()); + } + } @Override - public void deleteTemporaryKey(String accessKey) { + public void deleteAccessKey(String keyID) { try{ - ionosApi.deleteTemporaryAccount(token,accessKey); + ionoss3Api.deleteAccessKey(token, keyID); } catch (Exception e) { throw new EdcException("Deleting temporary key: " + e.getMessage()); } } - private String getRegion(String endpoint) { - if (!endpoint.contains(".ionoscloud.com")) - return endpoint; - - var region = endpoint.substring(0, endpoint.indexOf(".ionoscloud.com")); - - if (region.contains("https://" )) { - return region.substring(region.indexOf("https://") + 8); - } else if (region.contains("http://" )) { - return region.substring(region.indexOf("http://") + 7); - } else { - return region; + static String getRegion(String endpoint) { + + switch (endpoint) { + case "https://s3-eu-central-1.ionoscloud.com": + return "de"; + case "s3-eu-central-1.ionoscloud.com": + return "de"; + case "https://s3-eu-central-2.ionoscloud.com": + return "eu-central-2"; + case "s3-eu-central-2.ionoscloud.com": + return "eu-central-2"; + case "https://s3-eu-south-2.ionoscloud.com": + return "eu-south-2"; + case "s3-eu-south-2.ionoscloud.com": + return "eu-south-2"; + default: + throw new EdcException("Invalid endpoint: " + endpoint); } } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/S3CoreExtension.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/S3CoreExtension.java index 20522f6f..be4140e0 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/S3CoreExtension.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/S3CoreExtension.java @@ -28,6 +28,7 @@ import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_SECRET_KEY; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_ENDPOINT; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_TOKEN; +import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_MAX_FILES; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_MAX_FILES_DEFAULT; @Provides(S3ConnectorApi.class) @@ -62,8 +63,10 @@ public void initialize(ServiceExtensionContext context) { endPoint = context.getSetting(IONOS_ENDPOINT, IONOS_ENDPOINT); token = context.getSetting(IONOS_TOKEN, IONOS_TOKEN); } - - var s3Api = new S3ConnectorApiImpl(endPoint, accessKey, secretKey, token, IONOS_MAX_FILES_DEFAULT); + + var maxFiles = context.getSetting(IONOS_MAX_FILES, IONOS_MAX_FILES_DEFAULT); + + var s3Api = new S3ConnectorApiImpl(endPoint, accessKey, secretKey, token, maxFiles); context.registerService(S3ConnectorApi.class, s3Api); } } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/HttpConnector.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/HttpConnector.java deleted file mode 100644 index be5f05a3..00000000 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/HttpConnector.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.ionos.edc.extension.s3.connector.ionosapi; - -import java.io.IOException; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import org.eclipse.edc.spi.EdcException; - -public class HttpConnector { - OkHttpClient client = new OkHttpClient(); - String basicUrl = "https://api.ionos.com/cloudapi/v6/um/users/"; - - public String retrieveUserID(String token) { - String[] jwtParts = token.split("\\."); - String jwtPayload = new String(java.util.Base64.getDecoder().decode(jwtParts[1])); - String uuid = jwtPayload.split("\"uuid\":\"")[1].split("\"")[0]; - - return uuid; - } - - public TemporaryKey createTemporaryKey(String token) { - String url = basicUrl + retrieveUserID(token) + "/s3keys"; - - Request request = new Request.Builder().url(url) - //This adds the token to the header. - .addHeader("Authorization", "Bearer " + token) - .post(RequestBody.create(null, new byte[0])) - .build(); - try (Response response = client.newCall(request).execute()) { - if (!response.isSuccessful()) { - throw new IOException("Unexpected code " + response); - } - - ObjectMapper objectMapper = new ObjectMapper(); - S3Key resp = objectMapper.readValue(response.body().string(), S3Key.class); - TemporaryKey temp = new TemporaryKey(resp.getId().toString(), resp.getProperties().get("secretKey").toString()); - return temp; - } catch (IOException e) { - throw new EdcException("Error getting S3 temporary key", e); - } - } - - public void deleteTemporaryAccount(String token, String keyID) { - String url = basicUrl + retrieveUserID(token) + "/s3keys/" + keyID; - - Request request = new Request.Builder().url(url) - //This adds the token to the header. - .addHeader("Authorization", "Bearer " + token) - .delete() - .build(); - - try (Response response = client.newCall(request).execute()) { - if (!response.isSuccessful()) { - throw new IOException("Unexpected code " + response + " deleting S3 temporary key"); - } - } catch (IOException e) { - throw new EdcException("Error deleting S3 temporary key", e); - } - } -} diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/ResponseIonosApi.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/ResponseIonosApi.java deleted file mode 100644 index f9c23a99..00000000 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/ResponseIonosApi.java +++ /dev/null @@ -1,96 +0,0 @@ -package com.ionos.edc.extension.s3.connector.ionosapi; - -import java.util.List; - -public class ResponseIonosApi { - private String id; - private String type; - private String href; - private List items; - private int offset; - private int limit; - private Links _links; - public String getId() { - return id; - } - public void setId(String id) { - this.id = id; - } - public String getType() { - return type; - } - public void setType(String type) { - this.type = type; - } - public String getHref() { - return href; - } - public void setHref(String href) { - this.href = href; - } - public List getItems() { - return items; - } - public void setItems(List items) { - this.items = items; - } - public int getOffset() { - return offset; - } - public void setOffset(int offset) { - this.offset = offset; - } - public int getLimit() { - return limit; - } - public void setLimit(int limit) { - this.limit = limit; - } - public Links get_links() { - return _links; - } - public void set_links(Links _links) { - this._links = _links; - } - -} - -class Item { - private String id; - private String type; - private String href; - public String getId() { - return id; - } - public void setId(String id) { - this.id = id; - } - public String getType() { - return type; - } - public void setType(String type) { - this.type = type; - } - public String getHref() { - return href; - } - public void setHref(String href) { - this.href = href; - } - - -} - -class Links { - private String self; - - public String getSelf() { - return self; - } - - public void setSelf(String self) { - this.self = self; - } - - -} diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3AccessKey.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3AccessKey.java new file mode 100644 index 00000000..738e2872 --- /dev/null +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3AccessKey.java @@ -0,0 +1,39 @@ +package com.ionos.edc.extension.s3.connector.ionosapi; + +public class S3AccessKey { + public static final String AVAILABLE_STATUS = "AVAILABLE"; + + private String id; + private Metadata metadata; + private Properties properties; + + public String getId() { + return id; + } + public Metadata getMetadata() { + return metadata; + } + public Properties getProperties() { + return properties; + } + + public static class Metadata { + private String status; + + public String getStatus() { + return status; + } + } + + public static class Properties { + private String accessKey; + private String secretKey; + + public String getAccessKey() { + return accessKey; + } + public String getSecretKey() { + return secretKey; + } + } +} \ No newline at end of file diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3ApiConnector.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3ApiConnector.java new file mode 100644 index 00000000..d5bfc2e4 --- /dev/null +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3ApiConnector.java @@ -0,0 +1,86 @@ +package com.ionos.edc.extension.s3.connector.ionosapi; + +import java.io.IOException; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +import okhttp3.*; +import org.eclipse.edc.spi.EdcException; + +public class S3ApiConnector { + private static final String BASE_URL = "https://s3.ionos.com"; + + private final OkHttpClient client; + private final ObjectMapper objectMapper; + + public S3ApiConnector() { + client = new OkHttpClient(); + objectMapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + public S3AccessKey createAccessKey(String token) { + String url = BASE_URL + "/accesskeys"; + + Request request = new Request.Builder().url(url) + .addHeader("Authorization", "Bearer " + token) + .post(RequestBody.create(MediaType.get("application/json"), new byte[0])) + .build(); + + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("Unexpected code [" + response + "] creating S3 accesskey"); + } + + if (response.body() == null) + throw new IOException("Empty response body creating S3 accesskey"); + else + return objectMapper.readValue(response.body().string(), S3AccessKey.class); + + } catch (IOException e) { + throw new EdcException("Error creating S3 accesskey", e); + } + } + + public S3AccessKey retrieveAccessKey(String token, String keyID) { + String url = BASE_URL + "/accesskeys/" + keyID; + + Request request = new Request.Builder().url(url) + .addHeader("Authorization", "Bearer " + token) + .get() + .build(); + + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("Unexpected code [" + response + "] retrieving S3 accesskey"); + } + + if (response.body() == null) + throw new IOException("Empty response body retrieving S3 accesskey"); + else + return objectMapper.readValue(response.body().string(), S3AccessKey.class); + + } catch (IOException e) { + throw new EdcException("Error retrieving S3 accesskey", e); + } + } + + public void deleteAccessKey(String token, String keyID) { + String url = BASE_URL + "/accesskeys/" + keyID; + + Request request = new Request.Builder().url(url) + //This adds the token to the header. + .addHeader("Authorization", "Bearer " + token) + .delete() + .build(); + + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("Unexpected code [" + response + "] deleting S3 accesskey"); + } + } catch (IOException e) { + throw new EdcException("Error deleting S3 accesskey", e); + } + } +} diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3Key.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3Key.java deleted file mode 100644 index 5381cbac..00000000 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/S3Key.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.ionos.edc.extension.s3.connector.ionosapi; - -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Map; - -public class S3Key { - private String id; - private String type; - private String href; - private Metadata metadata; - private Map properties; - public String getId() { - return id; - } - public void setId(String id) { - this.id = id; - } - public String getType() { - return type; - } - public void setType(String type) { - this.type = type; - } - public String getHref() { - return href; - } - public void setHref(String href) { - this.href = href; - } - public Metadata getMetadata() { - return metadata; - } - public void setMetadata(Metadata metadata) { - this.metadata = metadata; - } - public Map getProperties() { - return properties; - } - public void setProperties(Map properties) { - this.properties = properties; - } - - -} - -class Metadata { - private String etag; - @JsonProperty("createdDate") - private String createdDate; - public String getEtag() { - return etag; - } - public void setEtag(String etag) { - this.etag = etag; - } - public String getCreatedDate() { - return createdDate; - } - public void setCreatedDate(String createdDate) { - this.createdDate = createdDate; - } - - -} \ No newline at end of file diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/TemporaryKey.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/TemporaryKey.java deleted file mode 100644 index 4009fdd1..00000000 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/connector/ionosapi/TemporaryKey.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.ionos.edc.extension.s3.connector.ionosapi; - -public class TemporaryKey { - - public String accessKey; - public String secretKey; - - - - public TemporaryKey(String accessKey, String secretKey) { - super(); - this.accessKey = accessKey; - this.secretKey = secretKey; - } - public String getAccessKey() { - return accessKey; - } - public void setAccessKey(String accessKey) { - this.accessKey = accessKey; - } - public String getSecretKey() { - return secretKey; - } - public void setSecretKey(String secretKey) { - this.secretKey = secretKey; - } - - -} diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java index 39e1af8f..9b3460e2 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java @@ -26,4 +26,6 @@ public interface IonosBucketSchema { String FILTER_EXCLUDES = EDC_NAMESPACE + "filter.excludes"; String ACCESS_KEY_ID = EDC_NAMESPACE + "accessKey"; String SECRET_ACCESS_KEY = EDC_NAMESPACE + "secretKey"; + + String STORAGE_NAME_DEFAULT = "https://s3-eu-central-1.ionoscloud.com"; } diff --git a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosSettingsSchema.java b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosSettingsSchema.java index 9f3a7bde..456ed58a 100644 --- a/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosSettingsSchema.java +++ b/extensions/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosSettingsSchema.java @@ -19,5 +19,11 @@ public interface IonosSettingsSchema { String IONOS_SECRET_KEY = "edc.ionos.secret.key"; String IONOS_ENDPOINT = "edc.ionos.endpoint"; String IONOS_TOKEN = "edc.ionos.token"; + String IONOS_KEY_VALIDATION_ATTEMPTS = "edc.ionos.key.validation.attempts"; + String IONOS_KEY_VALIDATION_DELAY = "edc.ionos.key.validation.delay"; + String IONOS_MAX_FILES = "edc.ionos.max.files"; + + int IONOS_KEY_VALIDATION_ATTEMPTS_DEFAULT = 10; + long IONOS_KEY_VALIDATION_DELAY_DEFAULT = 3000; int IONOS_MAX_FILES_DEFAULT = 1000; } \ No newline at end of file diff --git a/extensions/core-ionos-s3/src/test/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImplTest.java b/extensions/core-ionos-s3/src/test/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImplTest.java new file mode 100644 index 00000000..2b48ff35 --- /dev/null +++ b/extensions/core-ionos-s3/src/test/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImplTest.java @@ -0,0 +1,80 @@ +package com.ionos.edc.extension.s3.api; + +import org.eclipse.edc.spi.EdcException; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class S3ConnectorApiImplTest { + + @Test + public void getRegion_Frankfurt_Full() { + var endpoint = "https://s3-eu-central-1.ionoscloud.com"; + var region = S3ConnectorApiImpl.getRegion(endpoint); + + assertEquals("de", region); + } + + @Test + public void getRegion_Frankfurt_Short() { + var endpoint = "s3-eu-central-1.ionoscloud.com"; + var region = S3ConnectorApiImpl.getRegion(endpoint); + + assertEquals("de", region); + } + + @Test + public void getRegion_Berlin_Full() { + var endpoint = "https://s3-eu-central-2.ionoscloud.com"; + var region = S3ConnectorApiImpl.getRegion(endpoint); + + assertEquals("eu-central-2", region); + } + + @Test + public void getRegion_Berlin_Short() { + var endpoint = "s3-eu-central-2.ionoscloud.com"; + var region = S3ConnectorApiImpl.getRegion(endpoint); + + assertEquals("eu-central-2", region); + } + + @Test + public void getRegion_Logrono_Full() { + var endpoint = "https://s3-eu-south-2.ionoscloud.com"; + var region = S3ConnectorApiImpl.getRegion(endpoint); + + assertEquals("eu-south-2", region); + } + + @Test + public void getRegion_Logrono_Short() { + var endpoint = "s3-eu-south-2.ionoscloud.com"; + var region = S3ConnectorApiImpl.getRegion(endpoint); + + assertEquals("eu-south-2", region); + } + + @Test + public void getRegion_Invalid_Full() { + var endpoint = "https://s3-de-central.profitbricks.com"; + + Exception exception = assertThrows(EdcException.class, () -> { + S3ConnectorApiImpl.getRegion(endpoint); + }); + + assertNotNull(exception); + } + + public void getRegion_Invalid_Short() { + var endpoint = "s3-de-central.profitbricks.com"; + + Exception exception = assertThrows(EdcException.class, () -> { + S3ConnectorApiImpl.getRegion(endpoint); + }); + + assertNotNull(exception); + } +} diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java index 9a244bbc..a8d2cfcf 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/DataPlaneIonosS3Extension.java @@ -24,6 +24,7 @@ import org.eclipse.edc.spi.system.ServiceExtensionContext; import org.eclipse.edc.spi.types.TypeManager; +import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_MAX_FILES; import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_MAX_FILES_DEFAULT; @Extension(value = DataPlaneIonosS3Extension.NAME) @@ -53,11 +54,13 @@ public String name() { @Override public void initialize(ServiceExtensionContext context) { var monitor = context.getMonitor(); - + + var maxFiles = context.getSetting(IONOS_MAX_FILES, IONOS_MAX_FILES_DEFAULT); + var sourceFactory = new IonosDataSourceFactory(s3Api, monitor); pipelineService.registerFactory(sourceFactory); - var sinkFactory = new IonosDataSinkFactory(s3Api, executorContainer.getExecutorService(), monitor, vault, typeManager, IONOS_MAX_FILES_DEFAULT); + var sinkFactory = new IonosDataSinkFactory(s3Api, executorContainer.getExecutorService(), monitor, vault, typeManager, maxFiles); pipelineService.registerFactory(sinkFactory); context.getMonitor().info("File Transfer Extension initialized!"); } diff --git a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java index 747a4ec6..1b968dce 100644 --- a/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java +++ b/extensions/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSinkFactory.java @@ -34,10 +34,10 @@ import java.util.concurrent.ExecutorService; +import static com.ionos.edc.extension.s3.schema.IonosBucketSchema.STORAGE_NAME_DEFAULT; + public class IonosDataSinkFactory implements DataSinkFactory { - private static final String DEFAULT_STORAGE = "s3-eu-central-1.ionoscloud.com"; - private final ExecutorService executorService; private final Monitor monitor; private final S3ConnectorApi s3Api; @@ -47,8 +47,7 @@ public class IonosDataSinkFactory implements DataSinkFactory { private final Validator validator = new IonosSinkDataAddressValidationRule(); - public IonosDataSinkFactory(S3ConnectorApi s3Api, ExecutorService executorService, Monitor monitor, Vault vault, - TypeManager typeManager, int maxFiles) { + public IonosDataSinkFactory(S3ConnectorApi s3Api, ExecutorService executorService, Monitor monitor, Vault vault, TypeManager typeManager, int maxFiles) { this.s3Api = s3Api; this.executorService = executorService; this.monitor = monitor; @@ -86,7 +85,6 @@ public DataSink createSink(DataFlowRequest request) { var s3ApiTemp = new S3ConnectorApiImpl(destination.getStringProperty(IonosBucketSchema.STORAGE_NAME), token.getAccessKey(), token.getSecretKey(), - "", maxFiles); return IonosDataSink.Builder.newInstance() .bucketName(destination.getStringProperty(IonosBucketSchema.BUCKET_NAME)) @@ -97,10 +95,9 @@ public DataSink createSink(DataFlowRequest request) { .s3Api(s3ApiTemp) .build(); } else { - var s3ApiTemp = new S3ConnectorApiImpl(DEFAULT_STORAGE, + var s3ApiTemp = new S3ConnectorApiImpl(STORAGE_NAME_DEFAULT, token.getAccessKey(), token.getSecretKey(), - "", maxFiles); return IonosDataSink.Builder.newInstance() .bucketName(destination.getStringProperty(IonosBucketSchema.BUCKET_NAME)) diff --git a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/IonosProvisionExtension.java b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/IonosProvisionExtension.java index bc01d2ce..b59afd54 100644 --- a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/IonosProvisionExtension.java +++ b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/IonosProvisionExtension.java @@ -31,6 +31,11 @@ import org.eclipse.edc.spi.system.ServiceExtensionContext; import org.eclipse.edc.spi.types.TypeManager; +import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_KEY_VALIDATION_ATTEMPTS; +import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_KEY_VALIDATION_ATTEMPTS_DEFAULT; +import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_KEY_VALIDATION_DELAY; +import static com.ionos.edc.extension.s3.schema.IonosSettingsSchema.IONOS_KEY_VALIDATION_DELAY_DEFAULT; + @Extension(value = IonosProvisionExtension.NAME) public class IonosProvisionExtension implements ServiceExtension { @@ -54,18 +59,24 @@ public String name() { @Override public void initialize(ServiceExtensionContext context) { monitor = context.getMonitor(); + + var keyValidationAttempts = context.getSetting(IONOS_KEY_VALIDATION_ATTEMPTS, IONOS_KEY_VALIDATION_ATTEMPTS_DEFAULT); + var keyValidationDelay = context.getSetting(IONOS_KEY_VALIDATION_DELAY, IONOS_KEY_VALIDATION_DELAY_DEFAULT); + monitor.debug("IonosProvisionExtension" + "provisionManager"); var provisionManager = context.getService(ProvisionManager.class); + monitor.debug("IonosProvisionExtension" + "retryPolicy"); var retryPolicy = (RetryPolicy) context.getService(RetryPolicy.class); + monitor.debug("IonosProvisionExtension" + "s3BucketProvisioner"); - var s3BucketProvisioner = new IonosS3Provisioner(retryPolicy, clientApi); + var s3BucketProvisioner = new IonosS3Provisioner(monitor, retryPolicy, clientApi, keyValidationAttempts, keyValidationDelay); provisionManager.register(s3BucketProvisioner); - // register the generator monitor.debug("IonosProvisionExtension" + "manifestGenerator"); var manifestGenerator = context.getService(ResourceManifestGenerator.class); manifestGenerator.registerGenerator(new IonosS3ConsumerResourceDefinitionGenerator()); + monitor.debug("IonosProvisionExtension" + "registerTypes"); registerTypes(typeManager); } diff --git a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ProvisionedResource.java b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ProvisionedResource.java index 7edef32a..9117ddcb 100644 --- a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ProvisionedResource.java +++ b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3ProvisionedResource.java @@ -27,10 +27,10 @@ @JsonTypeName("dataspaceconnector:ionoss3provisionedresource") public class IonosS3ProvisionedResource extends ProvisionedDataDestinationResource { - private String accessKey; + private String accessKeyID; - public String getAccessKey() { - return accessKey; + public String getAccessKeyID() { + return accessKeyID; } private IonosS3ProvisionedResource() { @@ -65,8 +65,8 @@ public Builder path(String path) { return this; } - public Builder accessKey(String accessKey) { - provisionedResource.accessKey = accessKey; + public Builder accessKeyID(String accessKeyID) { + provisionedResource.accessKeyID = accessKeyID; return this; } } diff --git a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3Provisioner.java b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3Provisioner.java index e4fd8b8c..2663305c 100644 --- a/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3Provisioner.java +++ b/extensions/provision-ionos-s3/src/main/java/com/ionos/edc/provision/s3/bucket/IonosS3Provisioner.java @@ -17,12 +17,15 @@ import com.ionos.edc.extension.s3.api.S3ConnectorApi; import com.ionos.edc.extension.s3.configuration.IonosToken; +import com.ionos.edc.extension.s3.connector.ionosapi.S3AccessKey; import dev.failsafe.RetryPolicy; import org.eclipse.edc.connector.transfer.spi.provision.Provisioner; import org.eclipse.edc.connector.transfer.spi.types.DeprovisionedResource; import org.eclipse.edc.connector.transfer.spi.types.ProvisionResponse; import org.eclipse.edc.connector.transfer.spi.types.ProvisionedResource; import org.eclipse.edc.connector.transfer.spi.types.ResourceDefinition; +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.response.StatusResult; import java.time.OffsetDateTime; @@ -30,13 +33,20 @@ import static dev.failsafe.Failsafe.with; public class IonosS3Provisioner implements Provisioner { + + private final Monitor monitor; private final RetryPolicy retryPolicy; private final S3ConnectorApi s3Api; + private final Integer keyValidationAttempts; + private final Long keyValidationDelay; - public IonosS3Provisioner(RetryPolicy retryPolicy, S3ConnectorApi s3Api) { + public IonosS3Provisioner(Monitor monitor, RetryPolicy retryPolicy, S3ConnectorApi s3Api, int keyValidationAttempts, long keyValidationDelay) { + this.monitor = monitor; this.retryPolicy = retryPolicy; this.s3Api = s3Api; + this.keyValidationAttempts = keyValidationAttempts; + this.keyValidationDelay = keyValidationDelay; } @Override @@ -58,7 +68,7 @@ public CompletableFuture> provision(IonosS3Resou createBucket(bucketName); } - var serviceAccount = s3Api.createTemporaryKey(); + var temporaryKey = createTemporaryKey(); String resourceName = resourceDefinition.getKeyName(); @@ -67,7 +77,7 @@ public CompletableFuture> provision(IonosS3Resou .resourceName(resourceName) .bucketName(resourceDefinition.getBucketName()) .resourceDefinitionId(resourceDefinition.getId()) - .accessKey(serviceAccount.getAccessKey()) + .accessKeyID(temporaryKey.getId()) .transferProcessId(resourceDefinition.getTransferProcessId()) .hasToken(true); if (resourceDefinition.getStorage() != null) { @@ -79,7 +89,9 @@ public CompletableFuture> provision(IonosS3Resou var resource = resourceBuilder.build(); var expiryTime = OffsetDateTime.now().plusHours(1); - var secretToken = new IonosToken(serviceAccount.getAccessKey(), serviceAccount.getSecretKey(), expiryTime.toInstant().toEpochMilli() ); + var secretToken = new IonosToken(temporaryKey.getProperties().getAccessKey(), + temporaryKey.getProperties().getSecretKey(), + expiryTime.toInstant().toEpochMilli()); var response = ProvisionResponse.Builder.newInstance().resource(resource).secretToken(secretToken).build(); return CompletableFuture.completedFuture(StatusResult.success(response)); @@ -88,12 +100,49 @@ public CompletableFuture> provision(IonosS3Resou @Override public CompletableFuture> deprovision( IonosS3ProvisionedResource provisionedResource, org.eclipse.edc.policy.model.Policy policy) { - return with(retryPolicy).runAsync(() -> s3Api.deleteTemporaryKey(provisionedResource.getAccessKey())) + return with(retryPolicy).runAsync(() -> s3Api.deleteAccessKey(provisionedResource.getAccessKeyID())) .thenApply(empty -> StatusResult.success(DeprovisionedResource.Builder.newInstance().provisionedResourceId(provisionedResource.getId()).build()) ); } + private S3AccessKey createTemporaryKey() { + var accessKey = s3Api.createAccessKey(); + + // Validate the temporary key + var validated = false; + int attempts = 0; + while(attempts <= keyValidationAttempts) { + attempts++; + if (validateKey(accessKey)) { + validated = true; + break; + } + } + + if (validated) { + monitor.debug("[IonosS3Provisioner] Temporary key validated after " + attempts + " attempts of " + keyValidationDelay + " ms"); + return accessKey; + } else { + // Delete the not validated temporary key + s3Api.deleteAccessKey(accessKey.getId()); + throw new EdcException("Temporary key not validated after " + attempts + " attempts of " + keyValidationDelay + " ms"); + } + } + + private boolean validateKey(S3AccessKey accessKey) { + try { + // Wait the validation delay + Thread.sleep(keyValidationDelay); + } catch (InterruptedException e) { + throw new EdcException("Error waiting delay to validate temporary key", e); + } + + // Validate the key status + var retrievedAccessKey = s3Api.retrieveAccessKey(accessKey.getId()); + return (retrievedAccessKey.getMetadata().getStatus().equals(S3AccessKey.AVAILABLE_STATUS)); + } + private void createBucket(String bucketName) { s3Api.createBucket(bucketName); }