diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index b5166f7e7..c58a5bf9a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -158,6 +158,7 @@ private StorageConfigInfo getStorageInfo(Map internalProperties) s3Config.getS3CredentialsCatalogSecretAccessKey()) .setS3CredentialsClientAccessKeyId(s3Config.getS3CredentialsClientSecretAccessKey()) .setS3CredentialsClientSecretAccessKey(s3Config.getS3CredentialsClientAccessKeyId()) + .setS3Region(s3Config.getS3Region()) .build(); } if (configInfo instanceof AzureStorageConfigurationInfo) { @@ -283,6 +284,7 @@ public Builder setStorageConfigurationInfo( s3ConfigModel.getS3CredentialsClientAccessKeyId(), s3ConfigModel.getS3CredentialsClientSecretAccessKey(), s3ConfigModel.getS3PathStyleAccess(), + s3ConfigModel.getS3Region(), new ArrayList<>(allowedLocations)); break; case AZURE: diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java index 94764bbd0..893cc71f0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java @@ -78,6 +78,9 @@ public EnumMap getSubscopedCreds( propertiesMap.put( PolarisCredentialProperty.AWS_PATH_STYLE_ACCESS, storageConfig.getS3PathStyleAccess().toString()); + if (storageConfig.getS3Region() != null) { + propertiesMap.put(PolarisCredentialProperty.CLIENT_REGION, storageConfig.getS3Region()); + } if (storageConfig.getSkipCredentialSubscopingIndirection() == true) { LOGGER.debug("S3Compatible - skipCredentialSubscopingIndirection !"); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java index 5a8dbc19d..1a56599f8 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java @@ -42,6 +42,7 @@ public class S3CompatibleStorageConfigurationInfo extends PolarisStorageConfigur private @NotNull Boolean skipCredentialSubscopingIndirection; private @Nullable String s3CredentialsClientAccessKeyId; private @Nullable String s3CredentialsClientSecretAccessKey; + private @Nullable String s3Region; // Constructor @JsonCreator @@ -60,6 +61,7 @@ public S3CompatibleStorageConfigurationInfo( String s3CredentialsClientSecretAccessKey, @JsonProperty(value = "s3PathStyleAccess", required = false) @Nullable Boolean s3PathStyleAccess, + @JsonProperty(value = "S3Region", required = false) @Nullable String region, @JsonProperty(value = "allowedLocations", required = true) @Nullable List allowedLocations) { @@ -73,6 +75,7 @@ public S3CompatibleStorageConfigurationInfo( this.s3CredentialsClientAccessKeyId = s3CredentialsClientAccessKeyId; this.s3CredentialsClientSecretAccessKey = s3CredentialsClientSecretAccessKey; this.skipCredentialSubscopingIndirection = skipCredentialSubscopingIndirection; + this.s3Region = region; } public @NotNull String getS3Endpoint() { @@ -105,6 +108,14 @@ public S3CompatibleStorageConfigurationInfo( : this.s3CredentialsClientSecretAccessKey; } + public @Nullable String getS3Region() { + return s3Region; + } + + public void setS3Region(@Nullable String region) { + this.s3Region = region; + } + public @Nullable Boolean getSkipCredentialSubscopingIndirection() { return this.skipCredentialSubscopingIndirection; } diff --git a/regtests/run_spark_sql_s3compatible.sh b/regtests/run_spark_sql_s3compatible.sh index 7ccb6c138..1d842ba4b 100755 --- a/regtests/run_spark_sql_s3compatible.sh +++ b/regtests/run_spark_sql_s3compatible.sh @@ -212,7 +212,8 @@ curl -s -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" \ \"s3.path-style-access\": true, \"s3.credentials.catalog.access-key-id\": \"CATALOG_ID\", \"s3.credentials.catalog.secret-access-key\": \"CATALOG_SECRET\", - \"skip-credential-subscoping-indirection\": true + \"skip-credential-subscoping-indirection\": true, + \"s3.region\": \"rack-1\" } }" @@ -223,6 +224,7 @@ ${SPARK_HOME}/bin/spark-sql --verbose \ --conf spark.sql.catalog.polaris.warehouse=manual_spark \ --conf spark.sql.defaultCatalog=polaris \ --conf spark.hadoop.hive.cli.print.header=true \ + --conf spark.hadoop.fs.s3a.aws.region=rack-1 \ -f "minio/queries-for-spark.sql" diff --git a/spec/polaris-management-service.yml b/spec/polaris-management-service.yml index bb5902657..98c328cbb 100644 --- a/spec/polaris-management-service.yml +++ b/spec/polaris-management-service.yml @@ -943,6 +943,10 @@ components: type: string description: Optional - if not empty, it will be used when 'skip-credential-subscoping-indirection' is true example: "CLIENT_OF_CATALOG_1_SECRET_KEY_ENV_VARIABLE_NAME" + s3.region: + type: string + description: the s3 region where data is stored + example: "rack-1" required: - s3.endpoint