projectnessie · snazy · Sep 9, 2024 · Aug 14, 2024 · Aug 15, 2024 · Aug 16, 2024
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -823,18 +823,33 @@ jobs:
         run: |
           ct list-changed --target-branch ${{ github.event.repository.default_branch }}
 
+      - name: Run 'helm template' validation
+        run: |
+          cd helm/nessie
+          for f in values.yaml ci/*.yaml; do
+            echo "::group::helm template $f"
+            helm template --debug --namespace nessie-ns --values $f .
+            echo "::endgroup::"
+          done
+
       - name: Run chart-testing (lint)
         run: ct lint --debug --charts ./helm/nessie
 
       - name: Show pods
         run: kubectl get pods -A
 
+      - name: Install secrets
+        run: |
+          kubectl create namespace nessie-ns
+          kubectl apply --namespace nessie-ns $(find helm/nessie/ci/secrets -name "*.yaml" -printf '-f %p ')
+
       - name: Run chart-testing (install)
         run: |
           echo "Using image: ${DOCKER_IMAGE}"
           echo "        tag: ${DOCKER_VERSION}"
-      
+
           ct install \
+            --namespace nessie-ns \
             --helm-extra-set-args "--set=image.repository=${DOCKER_IMAGE} --set=image.tag=${DOCKER_VERSION}" \
             --debug --charts ./helm/nessie
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,6 +25,16 @@ as necessary. Empty sections will not end in the release notes.
 ### Breaking changes
 
 - The deprecated JDBC configuration properties for `catalog` and `schema` have been removed.
+- Catalog/Object store secrets: Secrets are now referenced via a URN as requirement to introduce support
+  for secret managers like Vault or those offered by cloud vendors. All secret reference URNs use the
+  pattern `urn:nessie-secret:<provider>:<secret-name>`.
+  The currently supported provider is `quarkus`, the `<secret-name>` is the name of the Quarkus
+  configuration entry, which can also be an environment variable name.
+  Make sure to use the new helm chart.
+  See [Nessie Docs](https://projectnessie.org/nessie-latest/configuration/#secrets-manager-settings).
+- Catalog/Object store secrets: secrets are now handled as immutable composites, which is important
+  to support secrets rotation with external secrets managers.
+  See [Nessie Docs](https://projectnessie.org/nessie-latest/configuration/#secrets-manager-settings).
 
 ### New Features
 

diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts
@@ -57,6 +57,7 @@ dependencies {
     api(project(":nessie-quarkus-ext-deployment"))
     api(project(":nessie-quarkus-ext"))
     api(project(":nessie-quarkus-rest"))
+    api(project(":nessie-quarkus-secrets"))
     api(project(":nessie-server-admin-tool"))
     api(project(":nessie-quarkus"))
     api(project(":nessie-quarkus-tests"))
@@ -120,6 +121,7 @@ dependencies {
     api(project(":nessie-catalog-service-impl"))
     api(project(":nessie-catalog-service-transfer"))
     api(project(":nessie-catalog-secrets-api"))
+    api(project(":nessie-catalog-secrets-smallrye"))
 
     if (!isIncludedInNesQuEIT()) {
       api(project(":nessie-gc-iceberg"))

diff --git a/...files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java b/...files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java
@@ -20,12 +20,13 @@
 import static org.projectnessie.catalog.files.BenchUtils.mockServer;
 import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials;
 import static org.projectnessie.catalog.secrets.KeySecret.keySecret;
+import static org.projectnessie.catalog.secrets.UnsafePlainTextSecretsManager.unsafePlainTextSecretsProvider;
 
 import com.azure.core.http.HttpClient;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 import java.util.Map;
-import java.util.stream.Collectors;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -39,6 +40,7 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
+import org.projectnessie.catalog.secrets.ResolvingSecretsProvider;
 import org.projectnessie.catalog.secrets.SecretsProvider;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
@@ -64,24 +66,29 @@ public void init() {
       AdlsConfig adlsConfig = AdlsConfig.builder().build();
       HttpClient httpClient = AdlsClients.buildSharedHttpClient(adlsConfig);
 
+      String theAccount = "the-account";
+      String theKey = "the-key";
+      SecretsProvider secretsProvider =
+          ResolvingSecretsProvider.builder()
+              .putSecretsManager(
+                  "plain",
+                  unsafePlainTextSecretsProvider(
+                      Map.of(
+                          theAccount, basicCredentials("foo", "foo").asMap(),
+                          theKey, keySecret("foo").asMap())))
+              .build();
+
       AdlsProgrammaticOptions adlsOptions =
           ImmutableAdlsProgrammaticOptions.builder()
               .defaultOptions(
                   ImmutableAdlsNamedFileSystemOptions.builder()
-                      .account(basicCredentials("foo", "foo"))
-                      .sasToken(keySecret("foo"))
+                      .account(URI.create("urn:nessie-secret:plain:" + theAccount))
+                      .sasToken(URI.create("urn:nessie-secret:plain:" + theKey))
                       .endpoint(server.getAdlsGen2BaseUri().toString())
                       .build())
               .build();
 
-      clientSupplier =
-          new AdlsClientSupplier(
-              httpClient,
-              adlsOptions,
-              new SecretsProvider(
-                  (names) ->
-                      names.stream()
-                          .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))));
+      clientSupplier = new AdlsClientSupplier(httpClient, adlsOptions, secretsProvider);
     }
 
     @TearDown

diff --git a/...g/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java b/...g/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java
@@ -19,12 +19,14 @@
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.projectnessie.catalog.files.BenchUtils.mockServer;
 import static org.projectnessie.catalog.files.gcs.GcsLocation.gcsLocation;
+import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret;
+import static org.projectnessie.catalog.secrets.UnsafePlainTextSecretsManager.unsafePlainTextSecretsProvider;
 
 import com.google.auth.http.HttpTransportFactory;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 import java.util.Map;
-import java.util.stream.Collectors;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -38,8 +40,8 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
+import org.projectnessie.catalog.secrets.ResolvingSecretsProvider;
 import org.projectnessie.catalog.secrets.SecretsProvider;
-import org.projectnessie.catalog.secrets.TokenSecret;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
 
@@ -63,23 +65,25 @@ public void init() {
 
       HttpTransportFactory httpTransportFactory = GcsClients.buildSharedHttpTransportFactory();
 
+      String theToken = "the-token";
+      SecretsProvider secretsProvider =
+          ResolvingSecretsProvider.builder()
+              .putSecretsManager(
+                  "plain",
+                  unsafePlainTextSecretsProvider(
+                      Map.of(theToken, tokenSecret("foo", null).asMap())))
+              .build();
+
       GcsProgrammaticOptions gcsOptions =
           ImmutableGcsProgrammaticOptions.builder()
               .defaultOptions(
                   ImmutableGcsNamedBucketOptions.builder()
-                      .oauth2Token(TokenSecret.tokenSecret("foo", null))
+                      .oauth2Token(URI.create("urn:nessie-secret:plain:" + theToken))
                       .host(server.getGcsBaseUri())
                       .build())
               .build();
 
-      storageSupplier =
-          new GcsStorageSupplier(
-              httpTransportFactory,
-              gcsOptions,
-              new SecretsProvider(
-                  (names) ->
-                      names.stream()
-                          .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))));
+      storageSupplier = new GcsStorageSupplier(httpTransportFactory, gcsOptions, secretsProvider);
     }
 
     @TearDown

diff --git a/...log/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java b/...log/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java
@@ -19,11 +19,12 @@
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.projectnessie.catalog.files.BenchUtils.mockServer;
 import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials;
+import static org.projectnessie.catalog.secrets.UnsafePlainTextSecretsManager.unsafePlainTextSecretsProvider;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 import java.util.Map;
-import java.util.stream.Collectors;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -37,6 +38,7 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
+import org.projectnessie.catalog.secrets.ResolvingSecretsProvider;
 import org.projectnessie.catalog.secrets.SecretsProvider;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
@@ -61,14 +63,23 @@ public static class BenchmarkParam {
     public void init() {
       server = mockServer(mock -> {});
 
+      String theAccessKey = "the-access-key";
+      SecretsProvider secretsProvider =
+          ResolvingSecretsProvider.builder()
+              .putSecretsManager(
+                  "plain",
+                  unsafePlainTextSecretsProvider(
+                      Map.of(theAccessKey, basicCredentials("foo", "bar").asMap())))
+              .build();
+
       S3Config s3config = S3Config.builder().build();
-      httpClient = S3Clients.apacheHttpClient(s3config, new SecretsProvider(names -> Map.of()));
+      httpClient = S3Clients.apacheHttpClient(s3config, secretsProvider);
 
       S3ProgrammaticOptions s3options =
           ImmutableS3ProgrammaticOptions.builder()
               .defaultOptions(
                   ImmutableS3NamedBucketOptions.builder()
-                      .accessKey(basicCredentials("foo", "bar"))
+                      .accessKey(URI.create("urn:nessie-secret:plain:" + theAccessKey))
                       .region("eu-central-1")
                       .endpoint(server.getS3BaseUri())
                       .pathStyleAccess(true)
@@ -77,15 +88,7 @@ public void init() {
 
       S3Sessions sessions = new S3Sessions("foo", null);
 
-      clientSupplier =
-          new S3ClientSupplier(
-              httpClient,
-              s3options,
-              new SecretsProvider(
-                  (names) ->
-                      names.stream()
-                          .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))),
-              sessions);
+      clientSupplier = new S3ClientSupplier(httpClient, s3options, sessions, secretsProvider);
     }
 
     @TearDown

diff --git a/...les/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3SessionCacheResourceBench.java b/...les/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3SessionCacheResourceBench.java
@@ -20,8 +20,10 @@
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.projectnessie.catalog.files.BenchUtils.mockServer;
 import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials;
+import static org.projectnessie.catalog.secrets.UnsafePlainTextSecretsManager.unsafePlainTextSecretsProvider;
 
 import java.net.URI;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ThreadLocalRandom;
@@ -42,6 +44,7 @@
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 import org.projectnessie.catalog.files.api.StorageLocations;
+import org.projectnessie.catalog.secrets.ResolvingSecretsProvider;
 import org.projectnessie.catalog.secrets.SecretsProvider;
 import org.projectnessie.objectstoragemock.ObjectStorageMock;
 import org.projectnessie.storage.uri.StorageUri;
@@ -72,21 +75,32 @@ public static class BenchmarkParam {
     public void init() {
       server = mockServer(mock -> {});
 
+      Map<String, Map<String, String>> secretsMap = new HashMap<>();
+      for (int i = 0; i < numBucketOptions; i++) {
+        secretsMap.put("access-key-" + i, basicCredentials("foo" + i, "bar" + 1).asMap());
+      }
+      secretsMap.put("the-access-key", basicCredentials("foo", "bar").asMap());
+      SecretsProvider secretsProvider =
+          ResolvingSecretsProvider.builder()
+              .putSecretsManager("plain", unsafePlainTextSecretsProvider(secretsMap))
+              .build();
+
       S3Config s3config = S3Config.builder().build();
-      httpClient = S3Clients.apacheHttpClient(s3config, new SecretsProvider(names -> Map.of()));
+      httpClient = S3Clients.apacheHttpClient(s3config, secretsProvider);
 
       S3Options s3options =
           ImmutableS3ProgrammaticOptions.builder()
               .defaultOptions(
                   ImmutableS3NamedBucketOptions.builder()
-                      .accessKey(basicCredentials("foo", "bar"))
+                      .accessKey(URI.create("the-access-key"))
                       .region("eu-central-1")
                       .pathStyleAccess(true)
                       .build())
               .build();
 
       StsClientsPool stsClientsPool = new StsClientsPool(s3options, httpClient, null);
-      stsCredentialsManager = new StsCredentialsManager(s3options, stsClientsPool, null);
+      stsCredentialsManager =
+          new StsCredentialsManager(s3options, stsClientsPool, secretsProvider, null);
 
       List<String> regions =
           Region.regions().stream()
@@ -100,7 +114,7 @@ public void init() {
               .mapToObj(
                   i ->
                       ImmutableS3NamedBucketOptions.builder()
-                          .accessKey(basicCredentials("foo" + i, "bar" + 1))
+                          .accessKey(URI.create("access-key-" + 1))
                           .region(regions.get(i % regions.size()))
                           .stsEndpoint(stsEndpoint)
                           .clientIam(

diff --git a/...log/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsClientSupplier.java b/...log/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsClientSupplier.java
@@ -51,6 +51,8 @@
 import org.projectnessie.catalog.files.adls.AdlsFileSystemOptions.AzureAuthType;
 import org.projectnessie.catalog.files.api.StorageLocations;
 import org.projectnessie.catalog.secrets.BasicCredentials;
+import org.projectnessie.catalog.secrets.KeySecret;
+import org.projectnessie.catalog.secrets.SecretType;
 import org.projectnessie.catalog.secrets.SecretsProvider;
 import org.projectnessie.storage.uri.StorageUri;
 import org.slf4j.Logger;
@@ -94,7 +96,7 @@ DataLakeFileSystemClient fileSystemClient(AdlsLocation location) {
     Configuration clientConfig = buildClientConfiguration();
 
     AdlsFileSystemOptions fileSystemOptions =
-        adlsOptions.effectiveOptionsForFileSystem(location.container(), secretsProvider);
+        adlsOptions.effectiveOptionsForFileSystem(location.container());
 
     String endpoint = endpointForLocation(location, fileSystemOptions);
 
@@ -229,6 +231,12 @@ private boolean applyCredentials(
         BasicCredentials account =
             fileSystemOptions
                 .account()
+                .map(
+                    secretName ->
+                        secretsProvider.getSecret(
+                            secretName, SecretType.BASIC, BasicCredentials.class))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
                 .orElseThrow(() -> new IllegalStateException("storage shared key missing"));
 
         sharedKeyCredentialConsumer.accept(
@@ -238,6 +246,11 @@ private boolean applyCredentials(
         sasTokenConsumer.accept(
             fileSystemOptions
                 .sasToken()
+                .map(
+                    secretName ->
+                        secretsProvider.getSecret(secretName, SecretType.KEY, KeySecret.class))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
                 .orElseThrow(() -> new IllegalStateException("SAS token missing"))
                 .key());
         return true;

diff --git a/.../files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsFileSystemOptions.java b/.../files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsFileSystemOptions.java
@@ -15,11 +15,10 @@
  */
 package org.projectnessie.catalog.files.adls;
 
+import java.net.URI;
 import java.time.Duration;
 import java.time.temporal.ChronoUnit;
 import java.util.Optional;
-import org.projectnessie.catalog.secrets.BasicCredentials;
-import org.projectnessie.catalog.secrets.KeySecret;
 
 public interface AdlsFileSystemOptions {
 
@@ -30,14 +29,14 @@ public interface AdlsFileSystemOptions {
   Optional<AzureAuthType> authType();
 
   /**
-   * Fully-qualified account name, e.g. {@code "myaccount.dfs.core.windows.net"} and account key,
-   * configured using the {@code name} and {@code secret} fields. If not specified, it will be
-   * queried via the configured credentials provider.
+   * Name of the basic-credentials secret containing the fully-qualified account name, e.g. {@code
+   * "myaccount.dfs.core.windows.net"} and account key, configured using the {@code name} and {@code
+   * secret} fields. If not specified, it will be queried via the configured credentials provider.
    */
-  Optional<BasicCredentials> account();
+  Optional<URI> account();
 
-  /** SAS token to access the ADLS file system. */
-  Optional<KeySecret> sasToken();
+  /** Name of the key-secret containing the SAS token to access the ADLS file system. */
+  Optional<URI> sasToken();
 
   /**
    * Enable short-lived user-delegation SAS tokens per file-system.