Add SecretsSupplier implementations for AWS, GCP, Vault

projectnessie · Jun 27, 2024 · b6a7594 · b6a7594
1 parent 1c86a86
commit b6a7594
Show file tree

Hide file tree

Showing 46 changed files with 2,067 additions and 182 deletions.
diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts
@@ -103,6 +103,11 @@ dependencies {
     api(project(":nessie-catalog-service-rest"))
     api(project(":nessie-catalog-service-impl"))
     api(project(":nessie-catalog-secrets-api"))
+    api(project(":nessie-catalog-secrets-cache"))
+    api(project(":nessie-catalog-secrets-aws"))
+    api(project(":nessie-catalog-secrets-gcs"))
+    api(project(":nessie-catalog-secrets-azure"))
+    api(project(":nessie-catalog-secrets-vault"))
 
     if (!isIncludedInNesQuEIT()) {
       api(project(":nessie-spark-antlr-runtime"))

diff --git a/catalog/files/impl/build.gradle.kts b/catalog/files/impl/build.gradle.kts
@@ -63,11 +63,14 @@ dependencies {
 
   testFixturesApi(project(":nessie-object-storage-mock"))
 
+  testImplementation(testFixtures(project(":nessie-catalog-secrets-api")))
+
   testRuntimeOnly(libs.logback.classic)
 
   jmhImplementation(libs.jmh.core)
   jmhImplementation(project(":nessie-object-storage-mock"))
   jmhAnnotationProcessor(libs.jmh.generator.annprocess)
+  jmhImplementation(testFixtures(project(":nessie-catalog-secrets-api")))
 }
 
 tasks.named("processJmhJandexIndex").configure { enabled = false }

diff --git a/...files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java b/...files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java
@@ -24,8 +24,6 @@
 import com.azure.core.http.HttpClient;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Map;
-import java.util.stream.Collectors;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -41,6 +39,7 @@
 import org.openjdk.jmh.infra.Blackhole;
 import org.projectnessie.catalog.files.adls.AdlsProgrammaticOptions.AdlsPerFileSystemOptions;
 import org.projectnessie.catalog.secrets.SecretsProvider;
+import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
 
@@ -77,12 +76,7 @@ public void init() {
 
       clientSupplier =
           new AdlsClientSupplier(
-              httpClient,
-              adlsOptions,
-              new SecretsProvider(
-                  (names) ->
-                      names.stream()
-                          .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))));
+              httpClient, adlsOptions, new SecretsProvider(new DummySecretsSupplier()));
     }
 
     @TearDown

diff --git a/...g/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java b/...g/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java
@@ -23,8 +23,6 @@
 import com.google.auth.http.HttpTransportFactory;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Map;
-import java.util.stream.Collectors;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -41,6 +39,7 @@
 import org.projectnessie.catalog.files.gcs.GcsProgrammaticOptions.GcsPerBucketOptions;
 import org.projectnessie.catalog.secrets.SecretsProvider;
 import org.projectnessie.catalog.secrets.TokenSecret;
+import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
 
@@ -75,12 +74,7 @@ public void init() {
 
       storageSupplier =
           new GcsStorageSupplier(
-              httpTransportFactory,
-              gcsOptions,
-              new SecretsProvider(
-                  (names) ->
-                      names.stream()
-                          .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))));
+              httpTransportFactory, gcsOptions, new SecretsProvider(new DummySecretsSupplier()));
     }
 
     @TearDown

diff --git a/...log/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java b/...log/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java
@@ -23,8 +23,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.time.Clock;
-import java.util.Map;
-import java.util.stream.Collectors;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -40,6 +38,7 @@
 import org.openjdk.jmh.infra.Blackhole;
 import org.projectnessie.catalog.files.s3.S3ProgrammaticOptions.S3PerBucketOptions;
 import org.projectnessie.catalog.secrets.SecretsProvider;
+import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
 import software.amazon.awssdk.http.SdkHttpClient;
@@ -84,10 +83,7 @@ public void init() {
               httpClient,
               s3config,
               s3options,
-              new SecretsProvider(
-                  (names) ->
-                      names.stream()
-                          .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))),
+              new SecretsProvider(new DummySecretsSupplier()),
               sessions);
     }
 

diff --git a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsOptions.java b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsOptions.java
@@ -79,7 +79,7 @@ default AdlsFileSystemOptions effectiveOptionsForFileSystem(
                   AdlsFileSystemOptions::account,
                   AdlsPerFileSystemOptions.Builder::account),
               secretAttribute(
-                  "sasToken",
+                  "sas-token",
                   SecretType.KEY,
                   AdlsFileSystemOptions::sasToken,
                   AdlsPerFileSystemOptions.Builder::sasToken));

diff --git a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/gcs/GcsOptions.java b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/gcs/GcsOptions.java
@@ -121,22 +121,22 @@ default GcsBucketOptions effectiveOptionsForBucket(
   List<SecretAttribute<GcsBucketOptions, GcsPerBucketOptions.Builder, ?>> SECRET_ATTRIBUTES =
       ImmutableList.of(
           secretAttribute(
-              "authCredentialsJson",
+              "auth-credentials-json",
               SecretType.KEY,
               GcsBucketOptions::authCredentialsJson,
               GcsPerBucketOptions.Builder::authCredentialsJson),
           secretAttribute(
-              "oauth2Token",
+              "oauth2-token",
               SecretType.EXPIRING_TOKEN,
               GcsBucketOptions::oauth2Token,
               GcsPerBucketOptions.Builder::oauth2Token),
           secretAttribute(
-              "encryptionKey",
+              "encryption-key",
               SecretType.KEY,
               GcsBucketOptions::encryptionKey,
               GcsPerBucketOptions.Builder::encryptionKey),
           secretAttribute(
-              "decryptionKey",
+              "decryption-key",
               SecretType.KEY,
               GcsBucketOptions::decryptionKey,
               GcsPerBucketOptions.Builder::decryptionKey));

diff --git a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/s3/S3Options.java b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/s3/S3Options.java
@@ -101,7 +101,7 @@ default S3BucketOptions effectiveOptionsForBucket(
   List<SecretAttribute<S3BucketOptions, S3PerBucketOptions.Builder, ?>> SECRET_ATTRIBUTES =
       ImmutableList.of(
           secretAttribute(
-              "accessKey",
+              "access-key",
               SecretType.BASIC,
               S3BucketOptions::accessKey,
               S3PerBucketOptions.Builder::accessKey));

diff --git a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/adls/TestAdlsClients.java b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/adls/TestAdlsClients.java
@@ -18,11 +18,10 @@
 import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials;
 
 import com.azure.core.http.HttpClient;
-import java.util.Map;
-import java.util.stream.Collectors;
 import org.projectnessie.catalog.files.AbstractClients;
 import org.projectnessie.catalog.files.api.ObjectIO;
 import org.projectnessie.catalog.secrets.SecretsProvider;
+import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
 
@@ -57,12 +56,7 @@ protected ObjectIO buildObjectIO(
 
     AdlsClientSupplier supplier =
         new AdlsClientSupplier(
-            httpClient,
-            adlsOptions.build(),
-            new SecretsProvider(
-                (names) ->
-                    names.stream()
-                        .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))));
+            httpClient, adlsOptions.build(), new SecretsProvider(new DummySecretsSupplier()));
 
     return new AdlsObjectIO(supplier);
   }

diff --git a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/gcs/TestGcsClients.java b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/gcs/TestGcsClients.java
@@ -18,11 +18,10 @@
 import static org.projectnessie.catalog.files.gcs.GcsClients.buildSharedHttpTransportFactory;
 
 import com.google.auth.http.HttpTransportFactory;
-import java.util.Map;
-import java.util.stream.Collectors;
 import org.projectnessie.catalog.files.AbstractClients;
 import org.projectnessie.catalog.files.api.ObjectIO;
 import org.projectnessie.catalog.secrets.SecretsProvider;
+import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
 
@@ -60,10 +59,7 @@ protected ObjectIO buildObjectIO(
         new GcsStorageSupplier(
             httpTransportFactory,
             gcsOptions.build(),
-            new SecretsProvider(
-                (names) ->
-                    names.stream()
-                        .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))));
+            new SecretsProvider(new DummySecretsSupplier()));
 
     return new GcsObjectIO(supplier);
   }

diff --git a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/s3/TestS3Clients.java b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/s3/TestS3Clients.java
@@ -15,17 +15,15 @@
  */
 package org.projectnessie.catalog.files.s3;
 
-import static java.util.function.Function.identity;
 import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials;
 
 import java.time.Clock;
-import java.util.Map;
-import java.util.stream.Collectors;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.projectnessie.catalog.files.AbstractClients;
 import org.projectnessie.catalog.files.api.ObjectIO;
 import org.projectnessie.catalog.secrets.SecretsProvider;
+import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
 import software.amazon.awssdk.http.SdkHttpClient;
@@ -76,10 +74,7 @@ protected ObjectIO buildObjectIO(
             sdkHttpClient,
             S3Config.builder().build(),
             s3options.build(),
-            new SecretsProvider(
-                names ->
-                    names.stream()
-                        .collect(Collectors.toMap(identity(), k -> Map.of("secret", "secret")))),
+            new SecretsProvider(new DummySecretsSupplier()),
             null);
     return new S3ObjectIO(supplier, Clock.systemUTC());
   }

diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java
@@ -15,5 +15,12 @@
  */
 package org.projectnessie.catalog.secrets;
 
-/** Base interface for all secrets. */
+/**
+ * Base interface for all secrets.
+ *
+ * <p>Secrets must not implement (override) any of these functions in a way: {@link
+ * Object#toString()}, {@link Object#hashCode()} or {@link Object#equals(Object)} that would
+ * directly (for example return a secret value from {@code toString()}) or indirectly (compare the
+ * instance itself against another instance) expose the values of a secret.
+ */
 public interface Secret {}
diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretJsonParser.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretJsonParser.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.catalog.secrets;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Map;
+
+final class SecretJsonParser {
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+
+  private SecretJsonParser() {}
+
+  @SuppressWarnings("unchecked")
+  static Map<String, String> parseOrSingle(String s) {
+    if (!s.trim().startsWith("{")) {
+      return Map.of("value", s);
+    }
+    try {
+      return MAPPER.readValue(s, Map.class);
+    } catch (JsonProcessingException e) {
+      return Map.of("value", s);
+    }
+  }
+}
diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java
@@ -17,6 +17,7 @@
 
 import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials;
 import static org.projectnessie.catalog.secrets.KeySecret.keySecret;
+import static org.projectnessie.catalog.secrets.SecretJsonParser.parseOrSingle;
 import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret;
 
 import java.util.Map;
@@ -33,6 +34,16 @@ public Secret fromValueMap(Map<String, String> value) {
     public Secret fromValueMap(Map<String, String> value) {
       return keySecret(value);
     }
+
+    @Override
+    public Secret parse(String string) {
+      return keySecret(string);
+    }
+
+    @Override
+    public boolean singleValued() {
+      return true;
+    }
   },
   EXPIRING_TOKEN() {
     @Override
@@ -44,4 +55,12 @@ public Secret fromValueMap(Map<String, String> value) {
 
   /** Construct a {@link Secret} instance from its map representation. */
   public abstract Secret fromValueMap(Map<String, String> value);
+
+  public boolean singleValued() {
+    return false;
+  }
+
+  public Secret parse(String string) {
+    return fromValueMap(parseOrSingle(string));
+  }
 }