Add SecretsSupplier implementations for AWS, GCP, Vault

projectnessie · Jul 24, 2024 · 7861e8b · 7861e8b
1 parent fcfec08
commit 7861e8b
Show file tree

Hide file tree

Showing 49 changed files with 2,078 additions and 210 deletions.
diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts
@@ -106,6 +106,11 @@ dependencies {
     api(project(":nessie-catalog-service-impl"))
     api(project(":nessie-catalog-service-transfer"))
     api(project(":nessie-catalog-secrets-api"))
+    api(project(":nessie-catalog-secrets-cache"))
+    api(project(":nessie-catalog-secrets-aws"))
+    api(project(":nessie-catalog-secrets-gcs"))
+    api(project(":nessie-catalog-secrets-azure"))
+    api(project(":nessie-catalog-secrets-vault"))
 
     if (!isIncludedInNesQuEIT()) {
       api(project(":nessie-spark-antlr-runtime"))

diff --git a/catalog/files/impl/build.gradle.kts b/catalog/files/impl/build.gradle.kts
@@ -63,11 +63,14 @@ dependencies {
 
   testFixturesApi(project(":nessie-object-storage-mock"))
 
+  testImplementation(testFixtures(project(":nessie-catalog-secrets-api")))
+
   testRuntimeOnly(libs.logback.classic)
 
   jmhImplementation(libs.jmh.core)
   jmhImplementation(project(":nessie-object-storage-mock"))
   jmhAnnotationProcessor(libs.jmh.generator.annprocess)
+  jmhImplementation(testFixtures(project(":nessie-catalog-secrets-api")))
 }
 
 tasks.named("processJmhJandexIndex").configure { enabled = false }

diff --git a/...files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java b/...files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java
@@ -24,8 +24,6 @@
 import com.azure.core.http.HttpClient;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Map;
-import java.util.stream.Collectors;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -40,6 +38,7 @@
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 import org.projectnessie.catalog.secrets.SecretsProvider;
+import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
 
@@ -76,12 +75,7 @@ public void init() {
 
       clientSupplier =
           new AdlsClientSupplier(
-              httpClient,
-              adlsOptions,
-              new SecretsProvider(
-                  (names) ->
-                      names.stream()
-                          .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))));
+              httpClient, adlsOptions, new SecretsProvider(new DummySecretsSupplier()));
     }
 
     @TearDown

diff --git a/...g/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java b/...g/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java
@@ -23,8 +23,6 @@
 import com.google.auth.http.HttpTransportFactory;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Map;
-import java.util.stream.Collectors;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -40,6 +38,7 @@
 import org.openjdk.jmh.infra.Blackhole;
 import org.projectnessie.catalog.secrets.SecretsProvider;
 import org.projectnessie.catalog.secrets.TokenSecret;
+import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
 
@@ -74,12 +73,7 @@ public void init() {
 
       storageSupplier =
           new GcsStorageSupplier(
-              httpTransportFactory,
-              gcsOptions,
-              new SecretsProvider(
-                  (names) ->
-                      names.stream()
-                          .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))));
+              httpTransportFactory, gcsOptions, new SecretsProvider(new DummySecretsSupplier()));
     }
 
     @TearDown

diff --git a/...log/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java b/...log/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java
@@ -22,8 +22,6 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Map;
-import java.util.stream.Collectors;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -38,6 +36,7 @@
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 import org.projectnessie.catalog.secrets.SecretsProvider;
+import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
 import software.amazon.awssdk.http.SdkHttpClient;
@@ -62,7 +61,8 @@ public void init() {
       server = mockServer(mock -> {});
 
       S3Config s3config = S3Config.builder().build();
-      httpClient = S3Clients.apacheHttpClient(s3config, new SecretsProvider(names -> Map.of()));
+      httpClient =
+          S3Clients.apacheHttpClient(s3config, new SecretsProvider(new DummySecretsSupplier()));
 
       S3ProgrammaticOptions s3options =
           ImmutableS3ProgrammaticOptions.builder()
@@ -79,13 +79,7 @@ public void init() {
 
       clientSupplier =
           new S3ClientSupplier(
-              httpClient,
-              s3options,
-              new SecretsProvider(
-                  (names) ->
-                      names.stream()
-                          .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))),
-              sessions);
+              httpClient, s3options, new SecretsProvider(new DummySecretsSupplier()), sessions);
     }
 
     @TearDown

diff --git a/...les/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3SessionCacheResourceBench.java b/...les/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3SessionCacheResourceBench.java
@@ -23,7 +23,6 @@
 
 import java.net.URI;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.stream.Collectors;
@@ -43,6 +42,7 @@
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 import org.projectnessie.catalog.secrets.SecretsProvider;
+import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import software.amazon.awssdk.http.SdkHttpClient;
 import software.amazon.awssdk.regions.Region;
@@ -72,7 +72,8 @@ public void init() {
       server = mockServer(mock -> {});
 
       S3Config s3config = S3Config.builder().build();
-      httpClient = S3Clients.apacheHttpClient(s3config, new SecretsProvider(names -> Map.of()));
+      httpClient =
+          S3Clients.apacheHttpClient(s3config, new SecretsProvider(new DummySecretsSupplier()));
 
       S3Options s3options =
           ImmutableS3ProgrammaticOptions.builder()

diff --git a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsOptions.java b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsOptions.java
@@ -65,7 +65,7 @@ public interface AdlsOptions {
                   AdlsFileSystemOptions::account,
                   ImmutableAdlsNamedFileSystemOptions.Builder::account),
               secretAttribute(
-                  "sasToken",
+                  "sas-token",
                   SecretType.KEY,
                   AdlsFileSystemOptions::sasToken,
                   ImmutableAdlsNamedFileSystemOptions.Builder::sasToken));

diff --git a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/gcs/GcsOptions.java b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/gcs/GcsOptions.java
@@ -95,22 +95,22 @@ public interface GcsOptions {
   List<SecretAttribute<GcsBucketOptions, Builder, ?>> SECRET_ATTRIBUTES =
       ImmutableList.of(
           secretAttribute(
-              "authCredentialsJson",
+              "auth-credentials-json",
               SecretType.KEY,
               GcsBucketOptions::authCredentialsJson,
               ImmutableGcsNamedBucketOptions.Builder::authCredentialsJson),
           secretAttribute(
-              "oauth2Token",
+              "oauth2-token",
               SecretType.EXPIRING_TOKEN,
               GcsBucketOptions::oauth2Token,
               ImmutableGcsNamedBucketOptions.Builder::oauth2Token),
           secretAttribute(
-              "encryptionKey",
+              "encryption-key",
               SecretType.KEY,
               GcsBucketOptions::encryptionKey,
               ImmutableGcsNamedBucketOptions.Builder::encryptionKey),
           secretAttribute(
-              "decryptionKey",
+              "decryption-key",
               SecretType.KEY,
               GcsBucketOptions::decryptionKey,
               ImmutableGcsNamedBucketOptions.Builder::decryptionKey));

diff --git a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/s3/S3Options.java b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/s3/S3Options.java
@@ -87,7 +87,7 @@ default int effectiveStsClientsCacheMaxEntries() {
       SECRET_ATTRIBUTES =
           ImmutableList.of(
               secretAttribute(
-                  "accessKey",
+                  "access-key",
                   SecretType.BASIC,
                   S3BucketOptions::accessKey,
                   ImmutableS3NamedBucketOptions.Builder::accessKey));

diff --git a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/adls/TestAdlsClients.java b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/adls/TestAdlsClients.java
@@ -18,12 +18,11 @@
 import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials;
 
 import com.azure.core.http.HttpClient;
-import java.util.Map;
-import java.util.stream.Collectors;
 import org.projectnessie.catalog.files.AbstractClients;
 import org.projectnessie.catalog.files.api.BackendExceptionMapper;
 import org.projectnessie.catalog.files.api.ObjectIO;
 import org.projectnessie.catalog.secrets.SecretsProvider;
+import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
 
@@ -66,12 +65,7 @@ protected ObjectIO buildObjectIO(
 
     AdlsClientSupplier supplier =
         new AdlsClientSupplier(
-            httpClient,
-            adlsOptions.build(),
-            new SecretsProvider(
-                (names) ->
-                    names.stream()
-                        .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))));
+            httpClient, adlsOptions.build(), new SecretsProvider(new DummySecretsSupplier()));
 
     return new AdlsObjectIO(supplier);
   }

diff --git a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/gcs/TestGcsClients.java b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/gcs/TestGcsClients.java
@@ -18,12 +18,11 @@
 import static org.projectnessie.catalog.files.gcs.GcsClients.buildSharedHttpTransportFactory;
 
 import com.google.auth.http.HttpTransportFactory;
-import java.util.Map;
-import java.util.stream.Collectors;
 import org.projectnessie.catalog.files.AbstractClients;
 import org.projectnessie.catalog.files.api.BackendExceptionMapper;
 import org.projectnessie.catalog.files.api.ObjectIO;
 import org.projectnessie.catalog.secrets.SecretsProvider;
+import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
 
@@ -67,10 +66,7 @@ protected ObjectIO buildObjectIO(
         new GcsStorageSupplier(
             httpTransportFactory,
             gcsOptions.build(),
-            new SecretsProvider(
-                (names) ->
-                    names.stream()
-                        .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))));
+            new SecretsProvider(new DummySecretsSupplier()));
 
     return new GcsObjectIO(supplier);
   }

diff --git a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/s3/TestS3Clients.java b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/s3/TestS3Clients.java
@@ -15,7 +15,6 @@
  */
 package org.projectnessie.catalog.files.s3;
 
-import static java.util.function.Function.identity;
 import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials;
 import static org.projectnessie.catalog.secrets.KeySecret.keySecret;
 
@@ -25,8 +24,6 @@
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.security.KeyStore;
-import java.util.Map;
-import java.util.stream.Collectors;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -35,6 +32,7 @@
 import org.projectnessie.catalog.files.api.BackendExceptionMapper;
 import org.projectnessie.catalog.files.api.ObjectIO;
 import org.projectnessie.catalog.secrets.SecretsProvider;
+import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
 import software.amazon.awssdk.http.SdkHttpClient;
@@ -46,7 +44,8 @@ public class TestS3Clients extends AbstractClients {
   @BeforeAll
   static void createHttpClient() {
     S3Config s3Config = S3Config.builder().build();
-    sdkHttpClient = S3Clients.apacheHttpClient(s3Config, new SecretsProvider(names -> Map.of()));
+    sdkHttpClient =
+        S3Clients.apacheHttpClient(s3Config, new SecretsProvider(new DummySecretsSupplier()));
   }
 
   @AfterAll
@@ -84,10 +83,7 @@ protected ObjectIO buildObjectIO(
         new S3ClientSupplier(
             sdkHttpClient,
             s3options.build(),
-            new SecretsProvider(
-                names ->
-                    names.stream()
-                        .collect(Collectors.toMap(identity(), k -> Map.of("secret", "secret")))),
+            new SecretsProvider(new DummySecretsSupplier()),
             null);
     return new S3ObjectIO(supplier);
   }
@@ -121,7 +117,7 @@ public void invalidTrustStore(@TempDir Path tempDir) throws Exception {
             () ->
                 S3Clients.apacheHttpClient(
                         S3Config.builder().trustStorePath(file).build(),
-                        new SecretsProvider(names -> Map.of()))
+                        new SecretsProvider(new DummySecretsSupplier()))
                     .close())
         .withMessage("No trust store type");
     soft.assertThatThrownBy(
@@ -132,7 +128,7 @@ public void invalidTrustStore(@TempDir Path tempDir) throws Exception {
                             .trustStoreType("jks")
                             .trustStorePassword(keySecret(password))
                             .build(),
-                        new SecretsProvider(names -> Map.of()))
+                        new SecretsProvider(new DummySecretsSupplier()))
                     .close())
         .isInstanceOf(RuntimeException.class)
         .cause()
@@ -145,7 +141,7 @@ public void invalidTrustStore(@TempDir Path tempDir) throws Exception {
                             .trustStoreType("jks")
                             .trustStorePassword(keySecret("wrong_password"))
                             .build(),
-                        new SecretsProvider(names -> Map.of()))
+                        new SecretsProvider(new DummySecretsSupplier()))
                     .close())
         .isInstanceOf(RuntimeException.class)
         .cause()
@@ -158,7 +154,7 @@ public void invalidTrustStore(@TempDir Path tempDir) throws Exception {
                             .trustStoreType("jks")
                             .trustStorePassword(keySecret(password))
                             .build(),
-                        new SecretsProvider(names -> Map.of()))
+                        new SecretsProvider(new DummySecretsSupplier()))
                     .close())
         .doesNotThrowAnyException();
   }
@@ -181,7 +177,7 @@ public void invalidKeyStore(@TempDir Path tempDir) throws Exception {
             () ->
                 S3Clients.apacheHttpClient(
                         S3Config.builder().keyStorePath(file).build(),
-                        new SecretsProvider(names -> Map.of()))
+                        new SecretsProvider(new DummySecretsSupplier()))
                     .close())
         .withMessage("No key store type");
     soft.assertThatThrownBy(
@@ -192,7 +188,7 @@ public void invalidKeyStore(@TempDir Path tempDir) throws Exception {
                             .keyStoreType("jks")
                             .keyStorePassword(keySecret(password))
                             .build(),
-                        new SecretsProvider(names -> Map.of()))
+                        new SecretsProvider(new DummySecretsSupplier()))
                     .close())
         .isInstanceOf(RuntimeException.class)
         .cause()
@@ -205,7 +201,7 @@ public void invalidKeyStore(@TempDir Path tempDir) throws Exception {
                             .keyStoreType("jks")
                             .keyStorePassword(keySecret("wrong_password"))
                             .build(),
-                        new SecretsProvider(names -> Map.of()))
+                        new SecretsProvider(new DummySecretsSupplier()))
                     .close())
         .isInstanceOf(RuntimeException.class)
         .cause()
@@ -218,7 +214,7 @@ public void invalidKeyStore(@TempDir Path tempDir) throws Exception {
                             .keyStoreType("jks")
                             .keyStorePassword(keySecret(password))
                             .build(),
-                        new SecretsProvider(names -> Map.of()))
+                        new SecretsProvider(new DummySecretsSupplier()))
                     .close())
         .doesNotThrowAnyException();
   }

diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java
@@ -15,5 +15,12 @@
  */
 package org.projectnessie.catalog.secrets;
 
-/** Base interface for all secrets. */
+/**
+ * Base interface for all secrets.
+ *
+ * <p>Secrets must not implement (override) any of these functions in a way: {@link
+ * Object#toString()}, {@link Object#hashCode()} or {@link Object#equals(Object)} that would
+ * directly (for example return a secret value from {@code toString()}) or indirectly (compare the
+ * instance itself against another instance) expose the values of a secret.
+ */
 public interface Secret {}