adding roleArn, camelCase refactoring, typo, cleaning

polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java

@@ -159,6 +159,7 @@ private StorageConfigInfo getStorageInfo(Map<String, String> internalProperties)
             .setS3CredentialsClientAccessKeyId(s3Config.getS3CredentialsClientSecretAccessKey())
             .setS3CredentialsClientSecretAccessKey(s3Config.getS3CredentialsClientAccessKeyId())
             .setS3Region(s3Config.getS3Region())
+            .setS3RoleArn(s3Config.getS3RoleArn())
             .build();
       }
       if (configInfo instanceof AzureStorageConfigurationInfo) {
@@ -285,6 +286,7 @@ public Builder setStorageConfigurationInfo(
                     s3ConfigModel.getS3CredentialsClientSecretAccessKey(),
                     s3ConfigModel.getS3PathStyleAccess(),
                     s3ConfigModel.getS3Region(),
+                    s3ConfigModel.getS3RoleArn(),
                     new ArrayList<>(allowedLocations));
             break;
           case AZURE:

polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java

@@ -100,11 +100,11 @@ public EnumMap<PolarisCredentialProperty, String> getSubscopedCreds(
           stsClient.assumeRole(
               AssumeRoleRequest.builder()
                   .roleSessionName("PolarisCredentialsSTS")
+                  .roleArn(
+                      (storageConfig.getS3RoleArn() == null) ? "" : storageConfig.getS3RoleArn())
                   .policy(
                       policyString(allowListOperation, allowedReadLocations, allowedWriteLocations)
                           .toJson())
-                  // @TODO customize duration
-                  // .durationSeconds(1800)
                   .build());
 
       propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, response.credentials().accessKeyId());

polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java

@@ -35,14 +35,15 @@ public class S3CompatibleStorageConfigurationInfo extends PolarisStorageConfigur
   // 5 is the approximate max allowed locations for the size of AccessPolicy when LIST is required
   // for allowed read and write locations for subscoping creds.
   @JsonIgnore private static final int MAX_ALLOWED_LOCATIONS = 5;
-  private @NotNull String s3endpoint;
+  private @NotNull String s3Endpoint;
   private @Nullable String s3CredentialsCatalogAccessKeyId;
   private @Nullable String s3CredentialsCatalogSecretAccessKey;
-  private @Nullable Boolean s3pathStyleAccess;
+  private @Nullable Boolean s3PathStyleAccess;
   private @NotNull Boolean skipCredentialSubscopingIndirection;
   private @Nullable String s3CredentialsClientAccessKeyId;
   private @Nullable String s3CredentialsClientSecretAccessKey;
   private @Nullable String s3Region;
+  private @Nullable String s3RoleArn;
 
   // Constructor
   @JsonCreator
@@ -61,29 +62,31 @@ public S3CompatibleStorageConfigurationInfo(
           String s3CredentialsClientSecretAccessKey,
       @JsonProperty(value = "s3PathStyleAccess", required = false) @Nullable
           Boolean s3PathStyleAccess,
-      @JsonProperty(value = "S3Region", required = false) @Nullable String region,
+      @JsonProperty(value = "s3Region", required = false) @Nullable String s3Region,
+      @JsonProperty(value = "s3RoleArn", required = false) @Nullable String s3RoleArn,
       @JsonProperty(value = "allowedLocations", required = true) @Nullable
           List<String> allowedLocations) {
 
     // storing properties
     super(storageType, allowedLocations);
     validateMaxAllowedLocations(MAX_ALLOWED_LOCATIONS);
-    this.s3pathStyleAccess = s3PathStyleAccess;
-    this.s3endpoint = s3Endpoint;
+    this.s3PathStyleAccess = s3PathStyleAccess;
+    this.s3Endpoint = s3Endpoint;
     this.s3CredentialsCatalogAccessKeyId = s3CredentialsCatalogAccessKeyId;
     this.s3CredentialsCatalogSecretAccessKey = s3CredentialsCatalogSecretAccessKey;
     this.s3CredentialsClientAccessKeyId = s3CredentialsClientAccessKeyId;
     this.s3CredentialsClientSecretAccessKey = s3CredentialsClientSecretAccessKey;
     this.skipCredentialSubscopingIndirection = skipCredentialSubscopingIndirection;
-    this.s3Region = region;
+    this.s3Region = s3Region;
+    this.s3RoleArn = s3RoleArn;
   }
 
   public @NotNull String getS3Endpoint() {
-    return this.s3endpoint;
+    return this.s3Endpoint;
   }
 
   public @NotNull Boolean getS3PathStyleAccess() {
-    return this.s3pathStyleAccess;
+    return this.s3PathStyleAccess;
   }
 
   public @NotNull String getS3CredentialsCatalogAccessKeyId() {
@@ -108,12 +111,12 @@ public S3CompatibleStorageConfigurationInfo(
         : this.s3CredentialsClientSecretAccessKey;
   }
 
-  public @Nullable String getS3Region() {
-    return s3Region;
+  public @NotNull String getS3RoleArn() {
+    return this.s3RoleArn;
   }
 
-  public void setS3Region(@Nullable String region) {
-    this.s3Region = region;
+  public @Nullable String getS3Region() {
+    return this.s3Region;
   }
 
   public @Nullable Boolean getSkipCredentialSubscopingIndirection() {

regtests/run_spark_sql_s3compatible.sh

@@ -165,9 +165,10 @@ curl -s -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" \
               \"storageType\": \"S3_COMPATIBLE\",
               \"allowedLocations\": [\"${S3_LOCATION}/\",\"${S3_LOCATION_2}/\"],
               \"s3.endpoint\": \"https://localhost:9000\",
-              \"s3.path-style-access\": true,
-              \"s3.credentials.catalog.access-key-id\": \"CATALOG_ID\",
-              \"s3.credentials.catalog.secret-access-key\": \"CATALOG_SECRET\"
+              \"s3.pathStyleAccess\": true,
+              \"s3.credentials.catalog.accessKeyId\": \"CATALOG_ID\",
+              \"s3.credentials.catalog.secretAccessKey\": \"CATALOG_SECRET\",
+              \"s3.roleArn\": \"arn:xxx:xxx:xxx:xxxx\"
             }
           }"
 
@@ -194,8 +195,8 @@ ${SPARK_HOME}/bin/spark-sql --verbose \
   -f "minio/queries-for-spark.sql"
 
 
-# skip-credential-subscoping-indirection = true
-echo -e "\n----\nUPDATE the catalog v2, - skip-credential-subscoping-indirection = true \n"
+# skipCredentialSubscopingIndirection = true
+echo -e "\n----\nUPDATE the catalog v2, - skipCredentialSubscopingIndirection = true \n"
 curl -s -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" \
       -H 'Accept: application/json' \
       -H 'Content-Type: application/json' \
@@ -209,15 +210,15 @@ curl -s -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" \
               \"storageType\": \"S3_COMPATIBLE\",
               \"allowedLocations\": [\"${S3_LOCATION}/\",\"${S3_LOCATION_2}/\"],
               \"s3.endpoint\": \"https://localhost:9000\",
-              \"s3.path-style-access\": true,
-              \"s3.credentials.catalog.access-key-id\": \"CATALOG_ID\",
-              \"s3.credentials.catalog.secret-access-key\": \"CATALOG_SECRET\",
-              \"skip-credential-subscoping-indirection\": true,
+              \"s3.pathStyleAccess\": true,
+              \"s3.credentials.catalog.accessKeyId\": \"CATALOG_ID\",
+              \"s3.credentials.catalog.secretAccessKey\": \"CATALOG_SECRET\",
+              \"skipCredentialSubscopingIndirection\": true,
               \"s3.region\": \"rack-1\"
             }
           }"
 
-echo -e "\n\n----\nStart again Spark-sql to test Polaris catalog with queries and skip-credential-subscoping-indirection = true \n"
+echo -e "\n\n----\nStart again Spark-sql to test Polaris catalog with queries and skipCredentialSubscopingIndirection = true \n"
 ${SPARK_HOME}/bin/spark-sql --verbose \
   --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
   --conf spark.sql.catalog.polaris.token="${SPARK_BEARER_TOKEN}" \
@@ -227,10 +228,8 @@ ${SPARK_HOME}/bin/spark-sql --verbose \
   --conf spark.hadoop.fs.s3a.aws.region=rack-1 \
   -f "minio/queries-for-spark.sql"
 
-
 echo -e "\n\n\nEnd of tests, a table and a view data with displayed should be visible in log above"
 echo "Minio stopping, bucket browser will be shutdown, volume data of the bucket remains in 'regtests/minio/miniodata'"
 echo ":-)"
 echo ""
 docker-compose --progress quiet --project-name polaris-minio --project-directory minio/ -f minio/docker-compose.yml down
-

spec/polaris-management-service.yml

@@ -917,36 +917,40 @@ components:
           type: string
           description: the S3 endpoint, will also be used as STS endpoint
           example: "http[s]://host:port"
-        s3.credentials.catalog.access-key-id:
+        s3.credentials.catalog.accessKeyId:
           type: string
           description: The environement variable name for the 'ACCESS_KEY_ID' used by the catalog to communicate with S3, default to aws sdk environement variable name
           default: "AWS_ACCESS_KEY_ID"
           example: "CATALOG_1_ACCESS_KEY_ENV_VARIABLE_NAME"
-        s3.credentials.catalog.secret-access-key:
+        s3.credentials.catalog.secretAccessKey:
           type: string
           description: The environement variable name for the 'SECRET_ACCESS_KEY' used by the catalog to communicate with S3, default to aws sdk environement variable name
           default: "AWS_SECRET_ACCESS_KEY"
           example: "CATALOG_1_SECRET_KEY_ENV_VARIABLE_NAME"
-        s3.path-style-access:
+        s3.pathStyleAccess:
           type: boolean
           description: if true use path style
           default: false
-        skip-credential-subscoping-indirection:
+        skipCredentialSubscopingIndirection:
           type: boolean
           default: false
           description: By default, the catalog sends to the client a temporary token (STS) limited to the requested resource. If set to true (when STS is not available) the catalog will skip subscoping, allowing technical access to all the catalog content
-        s3.credentials.client.access-key-id:
+        s3.credentials.client.accessKeyId:
           type: string
           description: Optional - if not empty, it will be used when 'skip-credential-subscoping-indirection' is true
           example: "CLIENT_OF_CATALOG_1_ACCESS_KEY_ENV_VARIABLE_NAME"
-        s3.credentials.client.secret-access-key:
+        s3.credentials.client.secretAccessKey:
           type: string
           description: Optional - if not empty, it will be used when 'skip-credential-subscoping-indirection' is true
           example: "CLIENT_OF_CATALOG_1_SECRET_KEY_ENV_VARIABLE_NAME"
         s3.region:
           type: string
-          description: the s3 region where data is stored
+          description: Optional - the s3 region where data is stored
           example: "rack-1"
+        s3.roleArn:
+          type: string
+          description: Optional - a s3 role arn
+          example: "arn:aws:iam::123456789001:principal/abc1-b-self1234"        
       required:
         - s3.endpoint