From 342f911b49fffe357b8f0f32074242e44b00cff7 Mon Sep 17 00:00:00 2001 From: lefebsy Date: Sat, 14 Dec 2024 00:54:04 +0100 Subject: [PATCH] adding roleArn, camelCase refactoring, typo, cleaning --- .../polaris/core/entity/CatalogEntity.java | 2 ++ ...mpatibleCredentialsStorageIntegration.java | 4 +-- .../S3CompatibleStorageConfigurationInfo.java | 27 ++++++++++--------- regtests/run_spark_sql_s3compatible.sh | 23 ++++++++-------- spec/polaris-management-service.yml | 18 ++++++++----- 5 files changed, 41 insertions(+), 33 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index c58a5bf9a..c2e370bdd 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -159,6 +159,7 @@ private StorageConfigInfo getStorageInfo(Map internalProperties) .setS3CredentialsClientAccessKeyId(s3Config.getS3CredentialsClientSecretAccessKey()) .setS3CredentialsClientSecretAccessKey(s3Config.getS3CredentialsClientAccessKeyId()) .setS3Region(s3Config.getS3Region()) + .setS3RoleArn(s3Config.getS3RoleArn()) .build(); } if (configInfo instanceof AzureStorageConfigurationInfo) { @@ -285,6 +286,7 @@ public Builder setStorageConfigurationInfo( s3ConfigModel.getS3CredentialsClientSecretAccessKey(), s3ConfigModel.getS3PathStyleAccess(), s3ConfigModel.getS3Region(), + s3ConfigModel.getS3RoleArn(), new ArrayList<>(allowedLocations)); break; case AZURE: diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java index 893cc71f0..ef087ecde 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java @@ -100,11 +100,11 @@ public EnumMap getSubscopedCreds( stsClient.assumeRole( AssumeRoleRequest.builder() .roleSessionName("PolarisCredentialsSTS") + .roleArn( + (storageConfig.getS3RoleArn() == null) ? "" : storageConfig.getS3RoleArn()) .policy( policyString(allowListOperation, allowedReadLocations, allowedWriteLocations) .toJson()) - // @TODO customize duration - // .durationSeconds(1800) .build()); propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, response.credentials().accessKeyId()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java index 1a56599f8..e4d1c7805 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java @@ -35,14 +35,15 @@ public class S3CompatibleStorageConfigurationInfo extends PolarisStorageConfigur // 5 is the approximate max allowed locations for the size of AccessPolicy when LIST is required // for allowed read and write locations for subscoping creds. @JsonIgnore private static final int MAX_ALLOWED_LOCATIONS = 5; - private @NotNull String s3endpoint; + private @NotNull String s3Endpoint; private @Nullable String s3CredentialsCatalogAccessKeyId; private @Nullable String s3CredentialsCatalogSecretAccessKey; - private @Nullable Boolean s3pathStyleAccess; + private @Nullable Boolean s3PathStyleAccess; private @NotNull Boolean skipCredentialSubscopingIndirection; private @Nullable String s3CredentialsClientAccessKeyId; private @Nullable String s3CredentialsClientSecretAccessKey; private @Nullable String s3Region; + private @Nullable String s3RoleArn; // Constructor @JsonCreator @@ -61,29 +62,31 @@ public S3CompatibleStorageConfigurationInfo( String s3CredentialsClientSecretAccessKey, @JsonProperty(value = "s3PathStyleAccess", required = false) @Nullable Boolean s3PathStyleAccess, - @JsonProperty(value = "S3Region", required = false) @Nullable String region, + @JsonProperty(value = "s3Region", required = false) @Nullable String s3Region, + @JsonProperty(value = "s3RoleArn", required = false) @Nullable String s3RoleArn, @JsonProperty(value = "allowedLocations", required = true) @Nullable List allowedLocations) { // storing properties super(storageType, allowedLocations); validateMaxAllowedLocations(MAX_ALLOWED_LOCATIONS); - this.s3pathStyleAccess = s3PathStyleAccess; - this.s3endpoint = s3Endpoint; + this.s3PathStyleAccess = s3PathStyleAccess; + this.s3Endpoint = s3Endpoint; this.s3CredentialsCatalogAccessKeyId = s3CredentialsCatalogAccessKeyId; this.s3CredentialsCatalogSecretAccessKey = s3CredentialsCatalogSecretAccessKey; this.s3CredentialsClientAccessKeyId = s3CredentialsClientAccessKeyId; this.s3CredentialsClientSecretAccessKey = s3CredentialsClientSecretAccessKey; this.skipCredentialSubscopingIndirection = skipCredentialSubscopingIndirection; - this.s3Region = region; + this.s3Region = s3Region; + this.s3RoleArn = s3RoleArn; } public @NotNull String getS3Endpoint() { - return this.s3endpoint; + return this.s3Endpoint; } public @NotNull Boolean getS3PathStyleAccess() { - return this.s3pathStyleAccess; + return this.s3PathStyleAccess; } public @NotNull String getS3CredentialsCatalogAccessKeyId() { @@ -108,12 +111,12 @@ public S3CompatibleStorageConfigurationInfo( : this.s3CredentialsClientSecretAccessKey; } - public @Nullable String getS3Region() { - return s3Region; + public @NotNull String getS3RoleArn() { + return this.s3RoleArn; } - public void setS3Region(@Nullable String region) { - this.s3Region = region; + public @Nullable String getS3Region() { + return this.s3Region; } public @Nullable Boolean getSkipCredentialSubscopingIndirection() { diff --git a/regtests/run_spark_sql_s3compatible.sh b/regtests/run_spark_sql_s3compatible.sh index 1d842ba4b..31e489f9a 100755 --- a/regtests/run_spark_sql_s3compatible.sh +++ b/regtests/run_spark_sql_s3compatible.sh @@ -165,9 +165,10 @@ curl -s -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" \ \"storageType\": \"S3_COMPATIBLE\", \"allowedLocations\": [\"${S3_LOCATION}/\",\"${S3_LOCATION_2}/\"], \"s3.endpoint\": \"https://localhost:9000\", - \"s3.path-style-access\": true, - \"s3.credentials.catalog.access-key-id\": \"CATALOG_ID\", - \"s3.credentials.catalog.secret-access-key\": \"CATALOG_SECRET\" + \"s3.pathStyleAccess\": true, + \"s3.credentials.catalog.accessKeyId\": \"CATALOG_ID\", + \"s3.credentials.catalog.secretAccessKey\": \"CATALOG_SECRET\", + \"s3.roleArn\": \"arn:xxx:xxx:xxx:xxxx\" } }" @@ -194,8 +195,8 @@ ${SPARK_HOME}/bin/spark-sql --verbose \ -f "minio/queries-for-spark.sql" -# skip-credential-subscoping-indirection = true -echo -e "\n----\nUPDATE the catalog v2, - skip-credential-subscoping-indirection = true \n" +# skipCredentialSubscopingIndirection = true +echo -e "\n----\nUPDATE the catalog v2, - skipCredentialSubscopingIndirection = true \n" curl -s -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" \ -H 'Accept: application/json' \ -H 'Content-Type: application/json' \ @@ -209,15 +210,15 @@ curl -s -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" \ \"storageType\": \"S3_COMPATIBLE\", \"allowedLocations\": [\"${S3_LOCATION}/\",\"${S3_LOCATION_2}/\"], \"s3.endpoint\": \"https://localhost:9000\", - \"s3.path-style-access\": true, - \"s3.credentials.catalog.access-key-id\": \"CATALOG_ID\", - \"s3.credentials.catalog.secret-access-key\": \"CATALOG_SECRET\", - \"skip-credential-subscoping-indirection\": true, + \"s3.pathStyleAccess\": true, + \"s3.credentials.catalog.accessKeyId\": \"CATALOG_ID\", + \"s3.credentials.catalog.secretAccessKey\": \"CATALOG_SECRET\", + \"skipCredentialSubscopingIndirection\": true, \"s3.region\": \"rack-1\" } }" -echo -e "\n\n----\nStart again Spark-sql to test Polaris catalog with queries and skip-credential-subscoping-indirection = true \n" +echo -e "\n\n----\nStart again Spark-sql to test Polaris catalog with queries and skipCredentialSubscopingIndirection = true \n" ${SPARK_HOME}/bin/spark-sql --verbose \ --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ --conf spark.sql.catalog.polaris.token="${SPARK_BEARER_TOKEN}" \ @@ -227,10 +228,8 @@ ${SPARK_HOME}/bin/spark-sql --verbose \ --conf spark.hadoop.fs.s3a.aws.region=rack-1 \ -f "minio/queries-for-spark.sql" - echo -e "\n\n\nEnd of tests, a table and a view data with displayed should be visible in log above" echo "Minio stopping, bucket browser will be shutdown, volume data of the bucket remains in 'regtests/minio/miniodata'" echo ":-)" echo "" docker-compose --progress quiet --project-name polaris-minio --project-directory minio/ -f minio/docker-compose.yml down - diff --git a/spec/polaris-management-service.yml b/spec/polaris-management-service.yml index 98c328cbb..987eed6c1 100644 --- a/spec/polaris-management-service.yml +++ b/spec/polaris-management-service.yml @@ -917,36 +917,40 @@ components: type: string description: the S3 endpoint, will also be used as STS endpoint example: "http[s]://host:port" - s3.credentials.catalog.access-key-id: + s3.credentials.catalog.accessKeyId: type: string description: The environement variable name for the 'ACCESS_KEY_ID' used by the catalog to communicate with S3, default to aws sdk environement variable name default: "AWS_ACCESS_KEY_ID" example: "CATALOG_1_ACCESS_KEY_ENV_VARIABLE_NAME" - s3.credentials.catalog.secret-access-key: + s3.credentials.catalog.secretAccessKey: type: string description: The environement variable name for the 'SECRET_ACCESS_KEY' used by the catalog to communicate with S3, default to aws sdk environement variable name default: "AWS_SECRET_ACCESS_KEY" example: "CATALOG_1_SECRET_KEY_ENV_VARIABLE_NAME" - s3.path-style-access: + s3.pathStyleAccess: type: boolean description: if true use path style default: false - skip-credential-subscoping-indirection: + skipCredentialSubscopingIndirection: type: boolean default: false description: By default, the catalog sends to the client a temporary token (STS) limited to the requested resource. If set to true (when STS is not available) the catalog will skip subscoping, allowing technical access to all the catalog content - s3.credentials.client.access-key-id: + s3.credentials.client.accessKeyId: type: string description: Optional - if not empty, it will be used when 'skip-credential-subscoping-indirection' is true example: "CLIENT_OF_CATALOG_1_ACCESS_KEY_ENV_VARIABLE_NAME" - s3.credentials.client.secret-access-key: + s3.credentials.client.secretAccessKey: type: string description: Optional - if not empty, it will be used when 'skip-credential-subscoping-indirection' is true example: "CLIENT_OF_CATALOG_1_SECRET_KEY_ENV_VARIABLE_NAME" s3.region: type: string - description: the s3 region where data is stored + description: Optional - the s3 region where data is stored example: "rack-1" + s3.roleArn: + type: string + description: Optional - a s3 role arn + example: "arn:aws:iam::123456789001:principal/abc1-b-self1234" required: - s3.endpoint