From 996cd5b53a667fd0a5e6f58b94beae531fad038e Mon Sep 17 00:00:00 2001 From: Adnan Hemani Date: Thu, 4 Jan 2024 11:38:38 -0800 Subject: [PATCH] AWS: Add S3 Access Grants Integration (#9385) --- .../iceberg/aws/AwsClientFactories.java | 1 + .../s3/DefaultS3FileIOAwsClientFactory.java | 1 + .../S3AccessGrantsPluginConfigurations.java | 50 +++++++++++ .../iceberg/aws/s3/S3FileIOProperties.java | 90 +++++++++++++++++++ .../iceberg/aws/TestS3FileIOProperties.java | 32 +++++++ build.gradle | 2 + gradle/libs.versions.toml | 2 + 7 files changed, 178 insertions(+) create mode 100644 aws/src/main/java/org/apache/iceberg/aws/s3/S3AccessGrantsPluginConfigurations.java diff --git a/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java b/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java index 501c7ca8b3b4..df31f6ab8032 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java +++ b/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java @@ -112,6 +112,7 @@ public S3Client s3() { .applyMutation( b -> s3FileIOProperties.applyCredentialConfigurations(awsClientProperties, b)) .applyMutation(s3FileIOProperties::applySignerConfiguration) + .applyMutation(s3FileIOProperties::applyS3AccessGrantsConfigurations) .build(); } diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/DefaultS3FileIOAwsClientFactory.java b/aws/src/main/java/org/apache/iceberg/aws/s3/DefaultS3FileIOAwsClientFactory.java index 89db96e8e137..a65910612fe6 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/DefaultS3FileIOAwsClientFactory.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/DefaultS3FileIOAwsClientFactory.java @@ -53,6 +53,7 @@ public S3Client s3() { s3FileIOProperties.applyCredentialConfigurations( awsClientProperties, s3ClientBuilder)) .applyMutation(s3FileIOProperties::applySignerConfiguration) + .applyMutation(s3FileIOProperties::applyS3AccessGrantsConfigurations) .build(); } } diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3AccessGrantsPluginConfigurations.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3AccessGrantsPluginConfigurations.java new file mode 100644 index 000000000000..a5f5ea6fa118 --- /dev/null +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3AccessGrantsPluginConfigurations.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.aws.s3; + +import java.util.Map; +import org.apache.iceberg.util.PropertyUtil; +import software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsPlugin; +import software.amazon.awssdk.services.s3.S3ClientBuilder; + +class S3AccessGrantsPluginConfigurations { + private boolean isS3AccessGrantsFallbackToIamEnabled; + + private S3AccessGrantsPluginConfigurations() {} + + public void configureS3ClientBuilder(T builder) { + S3AccessGrantsPlugin s3AccessGrantsPlugin = + S3AccessGrantsPlugin.builder().enableFallback(isS3AccessGrantsFallbackToIamEnabled).build(); + builder.addPlugin(s3AccessGrantsPlugin); + } + + private void initialize(Map properties) { + this.isS3AccessGrantsFallbackToIamEnabled = + PropertyUtil.propertyAsBoolean( + properties, + S3FileIOProperties.S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED, + S3FileIOProperties.S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED_DEFAULT); + } + + public static S3AccessGrantsPluginConfigurations create(Map properties) { + S3AccessGrantsPluginConfigurations configurations = new S3AccessGrantsPluginConfigurations(); + configurations.initialize(properties); + return configurations; + } +} diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java index af3d5a8472dd..9aad784be865 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java @@ -27,6 +27,7 @@ import org.apache.iceberg.aws.AwsClientProperties; import org.apache.iceberg.aws.glue.GlueCatalog; import org.apache.iceberg.aws.s3.signer.S3V4RestSignerClient; +import org.apache.iceberg.common.DynMethods; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Maps; @@ -50,6 +51,28 @@ public class S3FileIOProperties implements Serializable { */ public static final String CLIENT_FACTORY = "s3.client-factory-impl"; + /** + * This property is used to enable using the S3 Access Grants product to control authorization to + * S3 data. More information regarding this feature can be found at: + * https://aws.amazon.com/s3/features/access-grants/. + */ + public static final String S3_ACCESS_GRANTS_ENABLED = "s3.access-grants.enabled"; + + public static final boolean S3_ACCESS_GRANTS_ENABLED_DEFAULT = false; + + /** + * The fallback-to-iam property allows users to customize whether or not they would like their + * jobs fall back to the Job Execution IAM role in case they get an Access Denied from the S3 + * Access Grants call. Further documentation regarding this flag can be found in the S3 Access + * Grants Plugin GitHub: + * + *

For more details, see: https://github.com/aws/aws-s3-accessgrants-plugin-java-v2 + */ + public static final String S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED = + "s3.access-grants.fallback-to-iam"; + + public static final boolean S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED_DEFAULT = false; + /** * Type of S3 Server side encryption used, default to {@link S3FileIOProperties#SSE_TYPE_NONE}. * @@ -358,6 +381,8 @@ public class S3FileIOProperties implements Serializable { private String accessKeyId; private String secretAccessKey; private String sessionToken; + private boolean isS3AccessGrantsEnabled; + private boolean isS3AccessGrantsFallbackToIamEnabled; private int multipartUploadThreads; private int multiPartSize; private int deleteBatchSize; @@ -410,6 +435,8 @@ public S3FileIOProperties() { this.isUseArnRegionEnabled = USE_ARN_REGION_ENABLED_DEFAULT; this.isAccelerationEnabled = ACCELERATION_ENABLED_DEFAULT; this.isRemoteSigningEnabled = REMOTE_SIGNING_ENABLED_DEFAULT; + this.isS3AccessGrantsEnabled = S3_ACCESS_GRANTS_ENABLED_DEFAULT; + this.isS3AccessGrantsFallbackToIamEnabled = S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED_DEFAULT; this.allProperties = Maps.newHashMap(); ValidationException.check( @@ -500,6 +527,14 @@ public S3FileIOProperties(Map properties) { properties, REMOTE_SIGNING_ENABLED, REMOTE_SIGNING_ENABLED_DEFAULT); this.writeStorageClass = properties.get(WRITE_STORAGE_CLASS); this.allProperties = SerializableMap.copyOf(properties); + this.isS3AccessGrantsEnabled = + PropertyUtil.propertyAsBoolean( + properties, S3_ACCESS_GRANTS_ENABLED, S3_ACCESS_GRANTS_ENABLED_DEFAULT); + this.isS3AccessGrantsFallbackToIamEnabled = + PropertyUtil.propertyAsBoolean( + properties, + S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED, + S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED_DEFAULT); ValidationException.check( keyIdAccessKeyBothConfigured(), @@ -684,6 +719,22 @@ private Set toS3Tags(Map properties, String prefix) { .collect(Collectors.toSet()); } + public boolean isS3AccessGrantsEnabled() { + return isS3AccessGrantsEnabled; + } + + public void setS3AccessGrantsEnabled(boolean s3AccessGrantsEnabled) { + this.isS3AccessGrantsEnabled = s3AccessGrantsEnabled; + } + + public boolean isS3AccessGrantsFallbackToIamEnabled() { + return isS3AccessGrantsFallbackToIamEnabled; + } + + public void setS3AccessGrantsFallbackToIamEnabled(boolean s3AccessGrantsFallbackToIamEnabled) { + this.isS3AccessGrantsFallbackToIamEnabled = s3AccessGrantsFallbackToIamEnabled; + } + private boolean keyIdAccessKeyBothConfigured() { return (accessKeyId == null) == (secretAccessKey == null); } @@ -749,4 +800,43 @@ public void applyEndpointConfigurations(T builder) { builder.endpointOverride(URI.create(endpoint)); } } + + /** + * Add the S3 Access Grants Plugin for an S3 client. + * + *

Sample usage: + * + * 

+   *     S3Client.builder().applyMutation(s3FileIOProperties::applyS3AccessGrantsConfigurations)
+   *
+ */ + public void applyS3AccessGrantsConfigurations(T builder) { + if (isS3AccessGrantsEnabled) { + S3AccessGrantsPluginConfigurations s3AccessGrantsPluginConfigurations = + loadSdkPluginConfigurations( + S3AccessGrantsPluginConfigurations.class.getName(), allProperties); + s3AccessGrantsPluginConfigurations.configureS3ClientBuilder(builder); + } + } + + /** + * Dynamically load the http client builder to avoid runtime deps requirements of any optional SDK + * Plugins + */ + private T loadSdkPluginConfigurations(String impl, Map properties) { + Object sdkPluginConfigurations; + try { + sdkPluginConfigurations = + DynMethods.builder("create") + .hiddenImpl(impl, Map.class) + .buildStaticChecked() + .invoke(properties); + return (T) sdkPluginConfigurations; + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException( + String.format( + "Cannot create %s to generate and configure the client SDK Plugin builder", impl), + e); + } + } } diff --git a/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java b/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java index 4f9d879a7e10..83234dc09e6a 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java +++ b/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java @@ -249,4 +249,36 @@ public void testS3RemoteSigningDisabled() { builder.overrideConfiguration().advancedOption(SdkAdvancedClientOption.SIGNER); Assertions.assertThat(signer).isNotPresent(); } + + @Test + public void testS3AccessGrantsEnabled() { + // Explicitly true + Map properties = + ImmutableMap.of(S3FileIOProperties.S3_ACCESS_GRANTS_ENABLED, "true"); + S3FileIOProperties s3Properties = new S3FileIOProperties(properties); + S3ClientBuilder builder = S3Client.builder(); + + s3Properties.applyS3AccessGrantsConfigurations(builder); + Assertions.assertThat(builder.plugins().size()).isEqualTo(1); + } + + @Test + public void testS3AccessGrantsDisabled() { + // Explicitly false + Map properties = + ImmutableMap.of(S3FileIOProperties.S3_ACCESS_GRANTS_ENABLED, "false"); + S3FileIOProperties s3Properties = new S3FileIOProperties(properties); + S3ClientBuilder builder = S3Client.builder(); + + s3Properties.applyS3AccessGrantsConfigurations(builder); + Assertions.assertThat(builder.plugins().size()).isEqualTo(0); + + // Implicitly false + properties = ImmutableMap.of(); + s3Properties = new S3FileIOProperties(properties); + builder = S3Client.builder(); + + s3Properties.applyS3AccessGrantsConfigurations(builder); + Assertions.assertThat(builder.plugins().size()).isEqualTo(0); + } } diff --git a/build.gradle b/build.gradle index 4318d214391c..0f1363db4fcc 100644 --- a/build.gradle +++ b/build.gradle @@ -465,6 +465,7 @@ project(':iceberg-aws') { implementation "com.fasterxml.jackson.core:jackson-core" compileOnly(platform(libs.awssdk.bom)) + compileOnly(libs.awssdk.s3accessgrants) compileOnly("software.amazon.awssdk:url-connection-client") compileOnly("software.amazon.awssdk:apache-client") compileOnly("software.amazon.awssdk:auth") @@ -487,6 +488,7 @@ project(':iceberg-aws') { testImplementation(platform(libs.awssdk.bom)) testImplementation("software.amazon.awssdk:iam") testImplementation("software.amazon.awssdk:s3control") + testImplementation("software.amazon.s3.accessgrants:aws-s3-accessgrants-java-plugin") testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') testImplementation(libs.s3mock.junit5) { exclude module: "spring-boot-starter-logging" diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 02bfe327bad9..ebb105f1d7b0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -29,6 +29,7 @@ assertj-core = "3.24.2" awaitility = "4.2.0" awssdk-bom = "2.21.42" azuresdk-bom = "1.2.18" +awssdk-s3accessgrants = "1.0.1" caffeine = "2.9.3" calcite = "1.10.0" delta-standalone = "0.6.0" @@ -96,6 +97,7 @@ arrow-memory-netty = { module = "org.apache.arrow:arrow-memory-netty", version.r arrow-vector = { module = "org.apache.arrow:arrow-vector", version.ref = "arrow" } avro-avro = { module = "org.apache.avro:avro", version.ref = "avro" } awssdk-bom = { module = "software.amazon.awssdk:bom", version.ref = "awssdk-bom" } +awssdk-s3accessgrants = { module = "software.amazon.s3.accessgrants:aws-s3-accessgrants-java-plugin", version.ref = "awssdk-s3accessgrants" } azuresdk-bom = { module = "com.azure:azure-sdk-bom", version.ref = "azuresdk-bom" } caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version.ref = "caffeine" } calcite-core = { module = "org.apache.calcite:calcite-core", version.ref = "calcite" }