From 8cc8ab86792fe0da4963aef970b1ed2d51c67d5d Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Mon, 3 Jun 2024 18:06:21 +0200 Subject: [PATCH] Add `SecretsSupplier` implementations for AWS, GCP, Vault --- bom/build.gradle.kts | 5 + catalog/files/impl/build.gradle.kts | 3 + .../files/adls/AdlsClientResourceBench.java | 10 +- .../files/gcs/GcsClientResourceBench.java | 10 +- .../files/s3/S3ClientResourceBench.java | 8 +- .../catalog/files/adls/AdlsOptions.java | 2 +- .../catalog/files/gcs/GcsOptions.java | 8 +- .../catalog/files/s3/S3Options.java | 2 +- .../catalog/files/adls/TestAdlsClients.java | 10 +- .../catalog/files/gcs/TestGcsClients.java | 8 +- .../catalog/files/s3/TestS3Clients.java | 9 +- .../projectnessie/catalog/secrets/Secret.java | 9 +- .../catalog/secrets/SecretJsonParser.java | 38 +++ .../catalog/secrets/SecretType.java | 19 ++ .../catalog/secrets/SecretsProvider.java | 29 +- .../catalog/secrets/spi/SecretsSupplier.java | 14 +- .../spi/SingleValueSecretsSupplier.java | 31 +-- .../catalog/secrets/TestSecretType.java | 115 ++++++++ .../catalog/secrets/TestSecretsProvider.java | 124 ++++----- .../secrets/spi/DummySecretsSupplier.java | 34 +++ catalog/secrets/aws/build.gradle.kts | 54 ++++ .../secrets/aws/ITAwsSecretsSupplier.java | 179 +++++++++++++ .../src/intTest/resources/logback-test.xml | 30 +++ .../secrets/aws/Dockerfile-localstack-version | 3 + .../secrets/aws/AwsSecretsSupplier.java | 64 +++++ catalog/secrets/azure/build.gradle.kts | 54 ++++ .../secrets/azure/ITAzureSecretsSupplier.java | 178 +++++++++++++ .../src/intTest/resources/logback-test.xml | 30 +++ .../azure/Dockerfile-lowkey-vault-version | 3 + .../secrets/azure/AzureSecretsSupplier.java | 87 ++++++ catalog/secrets/cache/build.gradle.kts | 46 ++++ .../catalog/secrets/cache/CachingSecrets.java | 31 +++ .../secrets/cache/CachingSecretsBackend.java | 251 ++++++++++++++++++ .../secrets/cache/SecretsCacheConfig.java | 58 ++++ .../cache/TestCachingSecretsBackend.java | 178 +++++++++++++ catalog/secrets/gcs/build.gradle.kts | 43 +++ .../secrets/gcs/GcsSecretsSupplier.java | 97 +++++++ catalog/secrets/vault/build.gradle.kts | 52 ++++ .../secrets/vault/ITVaultSecretsSupplier.java | 201 ++++++++++++++ .../src/intTest/resources/logback-test.xml | 30 +++ .../secrets/vault/Dockerfile-vault-version | 3 + .../secrets/vault/VaultSecretsSupplier.java | 73 +++++ cli/cli/build.gradle.kts | 1 + .../nessie/cli/commands/WithNessie.java | 7 +- gradle/libs.versions.toml | 3 + gradle/projects.main.properties | 5 + 46 files changed, 2067 insertions(+), 182 deletions(-) create mode 100644 catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretJsonParser.java create mode 100644 catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretType.java create mode 100644 catalog/secrets/api/src/testFixtures/java/org/projectnessie/catalog/secrets/spi/DummySecretsSupplier.java create mode 100644 catalog/secrets/aws/build.gradle.kts create mode 100644 catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsSupplier.java create mode 100644 catalog/secrets/aws/src/intTest/resources/logback-test.xml create mode 100644 catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version create mode 100644 catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsSupplier.java create mode 100644 catalog/secrets/azure/build.gradle.kts create mode 100644 catalog/secrets/azure/src/intTest/java/org/projectnessie/catalog/secrets/azure/ITAzureSecretsSupplier.java create mode 100644 catalog/secrets/azure/src/intTest/resources/logback-test.xml create mode 100644 catalog/secrets/azure/src/intTest/resources/org/projectnessie/catalog/secrets/azure/Dockerfile-lowkey-vault-version create mode 100644 catalog/secrets/azure/src/main/java/org/projectnessie/catalog/secrets/azure/AzureSecretsSupplier.java create mode 100644 catalog/secrets/cache/build.gradle.kts create mode 100644 catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java create mode 100644 catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecretsBackend.java create mode 100644 catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/SecretsCacheConfig.java create mode 100644 catalog/secrets/cache/src/test/java/org/projectnessie/catalog/secrets/cache/TestCachingSecretsBackend.java create mode 100644 catalog/secrets/gcs/build.gradle.kts create mode 100644 catalog/secrets/gcs/src/main/java/org/projectnessie/catalog/secrets/gcs/GcsSecretsSupplier.java create mode 100644 catalog/secrets/vault/build.gradle.kts create mode 100644 catalog/secrets/vault/src/intTest/java/org/projectnessie/catalog/secrets/vault/ITVaultSecretsSupplier.java create mode 100644 catalog/secrets/vault/src/intTest/resources/logback-test.xml create mode 100644 catalog/secrets/vault/src/intTest/resources/org/projectnessie/catalog/secrets/vault/Dockerfile-vault-version create mode 100644 catalog/secrets/vault/src/main/java/org/projectnessie/catalog/secrets/vault/VaultSecretsSupplier.java diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index bb11e55f3e1..b1f5c54d2d2 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -103,6 +103,11 @@ dependencies { api(project(":nessie-catalog-service-rest")) api(project(":nessie-catalog-service-impl")) api(project(":nessie-catalog-secrets-api")) + api(project(":nessie-catalog-secrets-cache")) + api(project(":nessie-catalog-secrets-aws")) + api(project(":nessie-catalog-secrets-gcs")) + api(project(":nessie-catalog-secrets-azure")) + api(project(":nessie-catalog-secrets-vault")) if (!isIncludedInNesQuEIT()) { api(project(":nessie-spark-antlr-runtime")) diff --git a/catalog/files/impl/build.gradle.kts b/catalog/files/impl/build.gradle.kts index cde07dd220e..6f476ac2826 100644 --- a/catalog/files/impl/build.gradle.kts +++ b/catalog/files/impl/build.gradle.kts @@ -63,11 +63,14 @@ dependencies { testFixturesApi(project(":nessie-object-storage-mock")) + testImplementation(testFixtures(project(":nessie-catalog-secrets-api"))) + testRuntimeOnly(libs.logback.classic) jmhImplementation(libs.jmh.core) jmhImplementation(project(":nessie-object-storage-mock")) jmhAnnotationProcessor(libs.jmh.generator.annprocess) + jmhImplementation(testFixtures(project(":nessie-catalog-secrets-api"))) } tasks.named("processJmhJandexIndex").configure { enabled = false } diff --git a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java index 3cdd4ccefdf..3be9de20004 100644 --- a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java +++ b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java @@ -24,8 +24,6 @@ import com.azure.core.http.HttpClient; import java.io.IOException; import java.io.InputStream; -import java.util.Map; -import java.util.stream.Collectors; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -41,6 +39,7 @@ import org.openjdk.jmh.infra.Blackhole; import org.projectnessie.catalog.files.adls.AdlsProgrammaticOptions.AdlsPerFileSystemOptions; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.objectstoragemock.ObjectStorageMock; import org.projectnessie.storage.uri.StorageUri; @@ -77,12 +76,7 @@ public void init() { clientSupplier = new AdlsClientSupplier( - httpClient, - adlsOptions, - new SecretsProvider( - (names) -> - names.stream() - .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret"))))); + httpClient, adlsOptions, new SecretsProvider(new DummySecretsSupplier())); } @TearDown diff --git a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java index ad5ad138c18..76b6ecac2be 100644 --- a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java +++ b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java @@ -23,8 +23,6 @@ import com.google.auth.http.HttpTransportFactory; import java.io.IOException; import java.io.InputStream; -import java.util.Map; -import java.util.stream.Collectors; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -41,6 +39,7 @@ import org.projectnessie.catalog.files.gcs.GcsProgrammaticOptions.GcsPerBucketOptions; import org.projectnessie.catalog.secrets.SecretsProvider; import org.projectnessie.catalog.secrets.TokenSecret; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.objectstoragemock.ObjectStorageMock; import org.projectnessie.storage.uri.StorageUri; @@ -75,12 +74,7 @@ public void init() { storageSupplier = new GcsStorageSupplier( - httpTransportFactory, - gcsOptions, - new SecretsProvider( - (names) -> - names.stream() - .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret"))))); + httpTransportFactory, gcsOptions, new SecretsProvider(new DummySecretsSupplier())); } @TearDown diff --git a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java index 94d7bd1419b..ddc16ac8ef8 100644 --- a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java +++ b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java @@ -23,8 +23,6 @@ import java.io.IOException; import java.io.InputStream; import java.time.Clock; -import java.util.Map; -import java.util.stream.Collectors; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -40,6 +38,7 @@ import org.openjdk.jmh.infra.Blackhole; import org.projectnessie.catalog.files.s3.S3ProgrammaticOptions.S3PerBucketOptions; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.objectstoragemock.ObjectStorageMock; import org.projectnessie.storage.uri.StorageUri; import software.amazon.awssdk.http.SdkHttpClient; @@ -84,10 +83,7 @@ public void init() { httpClient, s3config, s3options, - new SecretsProvider( - (names) -> - names.stream() - .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))), + new SecretsProvider(new DummySecretsSupplier()), sessions); } diff --git a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsOptions.java b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsOptions.java index dc5e46e8e04..3a93d876aa1 100644 --- a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsOptions.java +++ b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsOptions.java @@ -79,7 +79,7 @@ default AdlsFileSystemOptions effectiveOptionsForFileSystem( AdlsFileSystemOptions::account, AdlsPerFileSystemOptions.Builder::account), secretAttribute( - "sasToken", + "sas-token", SecretType.KEY, AdlsFileSystemOptions::sasToken, AdlsPerFileSystemOptions.Builder::sasToken)); diff --git a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/gcs/GcsOptions.java b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/gcs/GcsOptions.java index 693400e011a..a25681cee79 100644 --- a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/gcs/GcsOptions.java +++ b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/gcs/GcsOptions.java @@ -121,22 +121,22 @@ default GcsBucketOptions effectiveOptionsForBucket( List> SECRET_ATTRIBUTES = ImmutableList.of( secretAttribute( - "authCredentialsJson", + "auth-credentials-json", SecretType.KEY, GcsBucketOptions::authCredentialsJson, GcsPerBucketOptions.Builder::authCredentialsJson), secretAttribute( - "oauth2Token", + "oauth2-token", SecretType.EXPIRING_TOKEN, GcsBucketOptions::oauth2Token, GcsPerBucketOptions.Builder::oauth2Token), secretAttribute( - "encryptionKey", + "encryption-key", SecretType.KEY, GcsBucketOptions::encryptionKey, GcsPerBucketOptions.Builder::encryptionKey), secretAttribute( - "decryptionKey", + "decryption-key", SecretType.KEY, GcsBucketOptions::decryptionKey, GcsPerBucketOptions.Builder::decryptionKey)); diff --git a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/s3/S3Options.java b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/s3/S3Options.java index 1e4e76f3f93..7825bbc4986 100644 --- a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/s3/S3Options.java +++ b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/s3/S3Options.java @@ -101,7 +101,7 @@ default S3BucketOptions effectiveOptionsForBucket( List> SECRET_ATTRIBUTES = ImmutableList.of( secretAttribute( - "accessKey", + "access-key", SecretType.BASIC, S3BucketOptions::accessKey, S3PerBucketOptions.Builder::accessKey)); diff --git a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/adls/TestAdlsClients.java b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/adls/TestAdlsClients.java index d8d9b8a9a20..b9a0e1c7e85 100644 --- a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/adls/TestAdlsClients.java +++ b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/adls/TestAdlsClients.java @@ -18,11 +18,10 @@ import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; import com.azure.core.http.HttpClient; -import java.util.Map; -import java.util.stream.Collectors; import org.projectnessie.catalog.files.AbstractClients; import org.projectnessie.catalog.files.api.ObjectIO; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.objectstoragemock.ObjectStorageMock; import org.projectnessie.storage.uri.StorageUri; @@ -57,12 +56,7 @@ protected ObjectIO buildObjectIO( AdlsClientSupplier supplier = new AdlsClientSupplier( - httpClient, - adlsOptions.build(), - new SecretsProvider( - (names) -> - names.stream() - .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret"))))); + httpClient, adlsOptions.build(), new SecretsProvider(new DummySecretsSupplier())); return new AdlsObjectIO(supplier); } diff --git a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/gcs/TestGcsClients.java b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/gcs/TestGcsClients.java index 34cb6763b25..e2d919d23cf 100644 --- a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/gcs/TestGcsClients.java +++ b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/gcs/TestGcsClients.java @@ -18,11 +18,10 @@ import static org.projectnessie.catalog.files.gcs.GcsClients.buildSharedHttpTransportFactory; import com.google.auth.http.HttpTransportFactory; -import java.util.Map; -import java.util.stream.Collectors; import org.projectnessie.catalog.files.AbstractClients; import org.projectnessie.catalog.files.api.ObjectIO; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.objectstoragemock.ObjectStorageMock; import org.projectnessie.storage.uri.StorageUri; @@ -60,10 +59,7 @@ protected ObjectIO buildObjectIO( new GcsStorageSupplier( httpTransportFactory, gcsOptions.build(), - new SecretsProvider( - (names) -> - names.stream() - .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret"))))); + new SecretsProvider(new DummySecretsSupplier())); return new GcsObjectIO(supplier); } diff --git a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/s3/TestS3Clients.java b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/s3/TestS3Clients.java index d53dba60e54..a72ba07a6ea 100644 --- a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/s3/TestS3Clients.java +++ b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/s3/TestS3Clients.java @@ -15,17 +15,15 @@ */ package org.projectnessie.catalog.files.s3; -import static java.util.function.Function.identity; import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; import java.time.Clock; -import java.util.Map; -import java.util.stream.Collectors; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.projectnessie.catalog.files.AbstractClients; import org.projectnessie.catalog.files.api.ObjectIO; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.objectstoragemock.ObjectStorageMock; import org.projectnessie.storage.uri.StorageUri; import software.amazon.awssdk.http.SdkHttpClient; @@ -76,10 +74,7 @@ protected ObjectIO buildObjectIO( sdkHttpClient, S3Config.builder().build(), s3options.build(), - new SecretsProvider( - names -> - names.stream() - .collect(Collectors.toMap(identity(), k -> Map.of("secret", "secret")))), + new SecretsProvider(new DummySecretsSupplier()), null); return new S3ObjectIO(supplier, Clock.systemUTC()); } diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java index 87af479ab8e..fffe0a6f863 100644 --- a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java @@ -15,5 +15,12 @@ */ package org.projectnessie.catalog.secrets; -/** Base interface for all secrets. */ +/** + * Base interface for all secrets. + * + *

Secrets must not implement (override) any of these functions in a way: {@link + * Object#toString()}, {@link Object#hashCode()} or {@link Object#equals(Object)} that would + * directly (for example return a secret value from {@code toString()}) or indirectly (compare the + * instance itself against another instance) expose the values of a secret. + */ public interface Secret {} diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretJsonParser.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretJsonParser.java new file mode 100644 index 00000000000..cf05230c820 --- /dev/null +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretJsonParser.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Map; + +final class SecretJsonParser { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private SecretJsonParser() {} + + @SuppressWarnings("unchecked") + static Map parseOrSingle(String s) { + if (!s.trim().startsWith("{")) { + return Map.of("value", s); + } + try { + return MAPPER.readValue(s, Map.class); + } catch (JsonProcessingException e) { + return Map.of("value", s); + } + } +} diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java index d92db7079d8..2fe802e6e52 100644 --- a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java @@ -17,6 +17,7 @@ import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; import static org.projectnessie.catalog.secrets.KeySecret.keySecret; +import static org.projectnessie.catalog.secrets.SecretJsonParser.parseOrSingle; import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; import java.util.Map; @@ -33,6 +34,16 @@ public Secret fromValueMap(Map value) { public Secret fromValueMap(Map value) { return keySecret(value); } + + @Override + public Secret parse(String string) { + return keySecret(string); + } + + @Override + public boolean singleValued() { + return true; + } }, EXPIRING_TOKEN() { @Override @@ -44,4 +55,12 @@ public Secret fromValueMap(Map value) { /** Construct a {@link Secret} instance from its map representation. */ public abstract Secret fromValueMap(Map value); + + public boolean singleValued() { + return false; + } + + public Secret parse(String string) { + return fromValueMap(parseOrSingle(string)); + } } diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretsProvider.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretsProvider.java index 28c72ee216a..1722df9cf4a 100644 --- a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretsProvider.java +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretsProvider.java @@ -17,11 +17,10 @@ import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; -import java.util.HashSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import org.projectnessie.catalog.secrets.spi.SecretsSupplier; /** @@ -79,7 +78,7 @@ public B applySecrets( @Nullable R specific, @Nonnull List> attributes) { - Set names = new HashSet<>(); + Map toResolve = new HashMap<>(); String basePrefix = baseName + '.'; boolean hasSpecific = specificName != null; @@ -93,35 +92,34 @@ public B applySecrets( specific != null ? attr.current().apply(specific) : Optional.empty(); if (specificSecret.isEmpty()) { if (hasSpecific) { - names.add(specificPrefix + attr.name()); + toResolve.put(specificPrefix + attr.name(), attr.type()); } Optional baseSecret = attr.current().apply(base); if (baseSecret.isEmpty()) { - names.add(basePrefix + attr.name()); + toResolve.put(basePrefix + attr.name(), attr.type()); } } } - if (names.isEmpty()) { + if (toResolve.isEmpty()) { // All secrets present, no need to ask for secrets. return builder; } - Map> secretsMap = secretsSupplier.resolveSecrets(names); + Map secretsMap = secretsSupplier.resolveSecrets(toResolve); for (SecretAttribute attr : casted) { Optional specificSecret = specific != null ? attr.current().apply(specific) : Optional.empty(); if (specificSecret.isEmpty()) { - Map value = - hasSpecific ? secretsMap.get(specificPrefix + attr.name()) : null; + Secret value = hasSpecific ? secretsMap.get(specificPrefix + attr.name()) : null; if (value != null) { - applySecretValue(builder, attr, value); + attr.applicator().accept(builder, value); } else { value = secretsMap.get(basePrefix + attr.name()); if (value != null) { - applySecretValue(builder, attr, value); + attr.applicator().accept(builder, value); } } } @@ -129,13 +127,4 @@ public B applySecrets( return builder; } - - private void applySecretValue( - B builder, SecretAttribute attr, Map value) { - Secret v = attr.type().fromValueMap(value); - if (v == null) { - return; - } - attr.applicator().accept(builder, v); - } } diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SecretsSupplier.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SecretsSupplier.java index c9f3905b998..d4989dcde20 100644 --- a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SecretsSupplier.java +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SecretsSupplier.java @@ -15,21 +15,17 @@ */ package org.projectnessie.catalog.secrets.spi; -import java.util.Collection; import java.util.Map; -import org.projectnessie.catalog.secrets.BasicCredentials; -import org.projectnessie.catalog.secrets.KeySecret; -import org.projectnessie.catalog.secrets.TokenSecret; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; /** SPI interface for actual secrets managers. */ public interface SecretsSupplier { /** * Resolve secrets. * - * @param names names of the secrets to resolve - * @return map of secret names to a map of key-value pairs representing the secret. The keys and - * -values depend on the type of secret. See {@link KeySecret#keySecret(Map)}, {@link - * BasicCredentials#basicCredentials(Map)}, {@link TokenSecret#tokenSecret(Map)} + * @param toResolve names and types of the secrets to resolve + * @return map of secret names to secrets */ - Map> resolveSecrets(Collection names); + Map resolveSecrets(Map toResolve); } diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SingleValueSecretsSupplier.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SingleValueSecretsSupplier.java index 06ad54d62a4..66a874e4fd6 100644 --- a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SingleValueSecretsSupplier.java +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SingleValueSecretsSupplier.java @@ -15,13 +15,12 @@ */ package org.projectnessie.catalog.secrets.spi; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.Collection; import java.util.Map; import java.util.stream.Collectors; import org.projectnessie.catalog.secrets.BasicCredentials; import org.projectnessie.catalog.secrets.KeySecret; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; import org.projectnessie.catalog.secrets.TokenSecret; /** @@ -29,34 +28,24 @@ * parse each secret from JSON as a map. */ public abstract class SingleValueSecretsSupplier implements SecretsSupplier { - private static final ObjectMapper MAPPER = new ObjectMapper(); @Override - public final Map> resolveSecrets(Collection names) { - Map maps = resolveSingleValueSecrets(names); + public final Map resolveSecrets(Map toResolve) { + Map maps = resolveSingleValueSecrets(toResolve); return maps.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> parseOrSingle(e.getValue()))); - } - - @SuppressWarnings("unchecked") - static Map parseOrSingle(String s) { - if (!s.trim().startsWith("{")) { - return Map.of("value", s); - } - try { - return MAPPER.readValue(s, Map.class); - } catch (JsonProcessingException e) { - return Map.of("value", s); - } + .collect( + Collectors.toMap( + Map.Entry::getKey, e -> toResolve.get(e.getKey()).parse(e.getValue()))); } /** * Resolve secrets. * - * @param names names of the secrets to resolve + * @param toResolve names and types of the secrets to resolve * @return map of secret names to either the JSON representations of the secrets or, if not a JSON * document, the single value for the secret. See {@link KeySecret#keySecret(Map)}, {@link * BasicCredentials#basicCredentials(Map)}, {@link TokenSecret#tokenSecret(Map)} */ - protected abstract Map resolveSingleValueSecrets(Collection names); + protected abstract Map resolveSingleValueSecrets( + Map toResolve); } diff --git a/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretType.java b/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretType.java new file mode 100644 index 00000000000..9d59857a226 --- /dev/null +++ b/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretType.java @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets; + +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +public class TestSecretType { + @InjectSoftAssertions protected SoftAssertions soft; + + @Test + public void keySecret() { + soft.assertThat(SecretType.KEY.parse("foo")) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("foo"); + soft.assertThat(SecretType.KEY.parse("{\"key\": \"foo\"}")) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("{\"key\": \"foo\"}"); + soft.assertThat(SecretType.KEY.parse("{\"value\": \"foo\"}")) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("{\"value\": \"foo\"}"); + + soft.assertThat(SecretType.KEY.fromValueMap(Map.of("key", "foo"))) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("foo"); + soft.assertThat(SecretType.KEY.fromValueMap(Map.of("value", "foo"))) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("foo"); + soft.assertThat(SecretType.KEY.fromValueMap(Map.of("blah", "foo"))).isNull(); + soft.assertThat(SecretType.KEY.fromValueMap(Map.of())).isNull(); + } + + @Test + public void tokenSecret() { + String instantStr = "2024-12-24T12:12:12Z"; + Instant instant = Instant.parse(instantStr); + + soft.assertThat(SecretType.EXPIRING_TOKEN.parse("tok")) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.empty()); + soft.assertThat(SecretType.EXPIRING_TOKEN.parse("{\"value\": \"tok\"}")) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.empty()); + soft.assertThat( + SecretType.EXPIRING_TOKEN.parse( + "{\"value\": \"tok\", \"expiresAt\": \"" + instantStr + "\"}")) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.of(instant)); + soft.assertThat(SecretType.EXPIRING_TOKEN.parse("{\"blah\": \"foo\"}")).isNull(); + soft.assertThat(SecretType.EXPIRING_TOKEN.parse("{}")).isNull(); + + soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("token", "tok"))) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.empty()); + soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("value", "tok"))) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.empty()); + soft.assertThat( + SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("token", "tok", "expiresAt", instantStr))) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.of(instant)); + soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("blah", "foo"))).isNull(); + soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of())).isNull(); + } + + @Test + public void basicCredential() { + soft.assertThat(SecretType.BASIC.parse("tok")).isNull(); + soft.assertThat(SecretType.BASIC.parse("{\"name\": \"user\", \"secret\": \"pass\"}")) + .extracting(BasicCredentials.class::cast) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly("user", "pass"); + soft.assertThat(SecretType.BASIC.parse("{\"blah\": \"foo\"}")).isNull(); + soft.assertThat(SecretType.BASIC.parse("{}")).isNull(); + + soft.assertThat(SecretType.BASIC.fromValueMap(Map.of("name", "user", "secret", "pass"))) + .extracting(BasicCredentials.class::cast) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly("user", "pass"); + soft.assertThat(SecretType.BASIC.fromValueMap(Map.of("name", "user"))).isNull(); + soft.assertThat(SecretType.BASIC.fromValueMap(Map.of("blah", "foo"))).isNull(); + soft.assertThat(SecretType.BASIC.fromValueMap(Map.of())).isNull(); + } +} diff --git a/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretsProvider.java b/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretsProvider.java index 4a7d26322cd..c5c802ecca5 100644 --- a/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretsProvider.java +++ b/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretsProvider.java @@ -20,6 +20,7 @@ import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; import static org.projectnessie.catalog.secrets.KeySecret.keySecret; import static org.projectnessie.catalog.secrets.SecretAttribute.secretAttribute; +import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; import java.time.Instant; import java.util.HashMap; @@ -43,8 +44,7 @@ public class TestSecretsProvider { @ParameterizedTest @MethodSource - public void secretsHelper( - Opts base, Opts spec, Opts expected, Map> secrets) { + public void secretsHelper(Opts base, Opts spec, Opts expected, Map secrets) { List> attributes = List.of( secretAttribute("key", SecretType.KEY, Opts::key, Opts.Builder::key), @@ -78,36 +78,6 @@ public static Stream secretsHelper() { arguments(Opts.EMPTY, null, Opts.EMPTY, Map.of()), arguments(Opts.EMPTY, Opts.EMPTY, Opts.EMPTY, Map.of()), // - // invalid "name" + "token" fields - arguments( - Opts.EMPTY, - Opts.EMPTY, - Opts.EMPTY, - Map.of( - "base.spec.basic", - Map.of("namex", "basic-name", "secret", "basic-secret"), - "base.spec.expiringToken", - Map.of("tokenx", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), - // invalid "expiresAt" + "secret" - arguments( - Opts.EMPTY, - Opts.EMPTY, - Opts.builder().expiringToken(TokenSecret.tokenSecret("exp-token", null)).build(), - Map.of( - "base.spec.basic", - Map.of("name", "basic-name", "secretx", "basic-secret"), - "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "ewpfoijkewoipfjewijfo"))), - // invalid "expiresAt" - arguments( - Opts.EMPTY, - Opts.EMPTY, - Opts.builder().expiringToken(TokenSecret.tokenSecret("exp-token", null)).build(), - Map.of( - "base.spec.basic", - Map.of("name", "basic-name", "secretx", "basic-secret"), - "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "42"))), // all secrets via "spec" from secrets-supplier arguments( Opts.EMPTY, @@ -115,30 +85,30 @@ public static Stream secretsHelper() { Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.key", - Map.of("value", "key"), + keySecret("key"), "base.spec.basic", - Map.of("name", "basic-name", "secret", "basic-secret"), + basicCredentials("basic-name", "basic-secret"), "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), arguments( Opts.EMPTY, Opts.EMPTY, Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.key", - Map.of("value", "key"), + keySecret("key"), "base.basic", - Map.of("name", "basic-name", "secret", "basic-secret"), + basicCredentials("basic-name", "basic-secret"), "base.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), // key present arguments( Opts.EMPTY, @@ -146,26 +116,26 @@ public static Stream secretsHelper() { Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.basic", - Map.of("name", "basic-name", "secret", "basic-secret"), + basicCredentials("basic-name", "basic-secret"), "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), arguments( Opts.builder().key(keySecret("key")).build(), Opts.EMPTY, Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.basic", - Map.of("name", "basic-name", "secret", "basic-secret"), + basicCredentials("basic-name", "basic-secret"), "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), // basic present arguments( Opts.builder().basic(basicCredentials("basic-name", "basic-secret")).build(), @@ -173,99 +143,99 @@ public static Stream secretsHelper() { Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.key", - Map.of("value", "key"), + keySecret("key"), "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), arguments( Opts.EMPTY, Opts.builder().basic(basicCredentials("basic-name", "basic-secret")).build(), Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.key", - Map.of("value", "key"), + keySecret("key"), "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), // expiringToken present arguments( - Opts.builder().expiringToken(TokenSecret.tokenSecret("exp-token", instant)).build(), + Opts.builder().expiringToken(tokenSecret("exp-token", instant)).build(), Opts.EMPTY, Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.key", - Map.of("value", "key"), + keySecret("key"), "base.spec.basic", - Map.of("name", "basic-name", "secret", "basic-secret"))), + basicCredentials("basic-name", "basic-secret"))), arguments( Opts.EMPTY, - Opts.builder().expiringToken(TokenSecret.tokenSecret("exp-token", instant)).build(), + Opts.builder().expiringToken(tokenSecret("exp-token", instant)).build(), Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.key", - Map.of("value", "key"), + keySecret("key"), "base.spec.basic", - Map.of("name", "basic-name", "secret", "basic-secret"))), + basicCredentials("basic-name", "basic-secret"))), // override in "base" arguments( Opts.builder() .key(keySecret("nope")) .basic(basicCredentials("nope", "basic-nope")) - .expiringToken(TokenSecret.tokenSecret("nope", instant)) + .expiringToken(tokenSecret("nope", instant)) .build(), Opts.EMPTY, Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.key", - Map.of("value", "key"), + keySecret("key"), "base.spec.basic", - Map.of("name", "basic-name", "secret", "basic-secret"), + basicCredentials("basic-name", "basic-secret"), "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), // NO override in "base" arguments( Opts.builder() .key(keySecret("nope")) .basic(basicCredentials("nope", "basic-nope")) - .expiringToken(TokenSecret.tokenSecret("nope", instant)) + .expiringToken(tokenSecret("nope", instant)) .build(), Opts.EMPTY, Opts.builder() .key(keySecret("nope")) .basic(basicCredentials("nope", "basic-nope")) - .expiringToken(TokenSecret.tokenSecret("nope", instant)) + .expiringToken(tokenSecret("nope", instant)) .build(), Map.of( "base.key", - Map.of("value", "key"), + keySecret("key"), "base.basic", - Map.of("name", "basic-name", "secret", "basic-secret"), + basicCredentials("basic-name", "basic-secret"), "base.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), // arguments( Opts.EMPTY, Opts.EMPTY, - Opts.builder().expiringToken(TokenSecret.tokenSecret("exp-token", null)).build(), - Map.of("base.spec.expiringToken", Map.of("token", "exp-token"))) + Opts.builder().expiringToken(tokenSecret("exp-token", null)).build(), + Map.of("base.spec.expiringToken", tokenSecret("exp-token", null))) // ); } @@ -297,11 +267,11 @@ interface Builder { } } - static SecretsSupplier mapSecretsSupplier(Map> secretsMap) { - return names -> { - Map> resolved = new HashMap<>(); - for (String name : names) { - Map r = secretsMap.get(name); + static SecretsSupplier mapSecretsSupplier(Map secretsMap) { + return toResolve -> { + Map resolved = new HashMap<>(); + for (String name : toResolve.keySet()) { + Secret r = secretsMap.get(name); if (r != null) { resolved.put(name, r); } diff --git a/catalog/secrets/api/src/testFixtures/java/org/projectnessie/catalog/secrets/spi/DummySecretsSupplier.java b/catalog/secrets/api/src/testFixtures/java/org/projectnessie/catalog/secrets/spi/DummySecretsSupplier.java new file mode 100644 index 00000000000..2eab26f52a1 --- /dev/null +++ b/catalog/secrets/api/src/testFixtures/java/org/projectnessie/catalog/secrets/spi/DummySecretsSupplier.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.spi; + +import java.util.Map; +import java.util.stream.Collectors; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; + +public class DummySecretsSupplier implements SecretsSupplier { + @Override + public Map resolveSecrets(Map toResolve) { + return toResolve.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> + e.getValue() + .fromValueMap(Map.of("value", "key", "name", "name", "secret", "secret")))); + } +} diff --git a/catalog/secrets/aws/build.gradle.kts b/catalog/secrets/aws/build.gradle.kts new file mode 100644 index 00000000000..dd2a1b7e5d4 --- /dev/null +++ b/catalog/secrets/aws/build.gradle.kts @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets AWS" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:apache-client") { + exclude("commons-logging", "commons-logging") + } + implementation("software.amazon.awssdk:secretsmanager") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) + + intTestCompileOnly(project(":nessie-immutables")) + intTestImplementation(platform(libs.testcontainers.bom)) + intTestImplementation("org.testcontainers:testcontainers") + intTestImplementation("org.testcontainers:localstack") + intTestImplementation("org.testcontainers:junit-jupiter") + intTestImplementation(project(":nessie-container-spec-helper")) + intTestRuntimeOnly(libs.logback.classic) +} diff --git a/catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsSupplier.java b/catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsSupplier.java new file mode 100644 index 00000000000..6c9d63c7c43 --- /dev/null +++ b/catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsSupplier.java @@ -0,0 +1,179 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.aws; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.type; +import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; +import static org.projectnessie.catalog.secrets.KeySecret.keySecret; +import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; +import static org.testcontainers.containers.localstack.LocalStackContainer.Service.SECRETSMANAGER; + +import java.net.URI; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.projectnessie.catalog.secrets.BasicCredentials; +import org.projectnessie.catalog.secrets.KeySecret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.TokenSecret; +import org.projectnessie.nessie.testing.containerspec.ContainerSpecHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.CreateSecretRequest; + +@Testcontainers +@ExtendWith(SoftAssertionsExtension.class) +public class ITAwsSecretsSupplier { + private static final Logger LOGGER = LoggerFactory.getLogger(ITAwsSecretsSupplier.class); + + @InjectSoftAssertions SoftAssertions soft; + + @Container + static LocalStackContainer localstack = + new LocalStackContainer( + ContainerSpecHelper.builder() + .name("localstack") + .containerClass(ITAwsSecretsSupplier.class) + .build() + .dockerImageName(null) + .asCompatibleSubstituteFor("localstack/localstack")) + .withLogConsumer(c -> LOGGER.info("[LOCALSTACK] {}", c.getUtf8StringWithoutLineEnding())) + .withServices(SECRETSMANAGER); + + @Test + public void awsSecretsManager() { + URI secretsManagerEndpoint = localstack.getEndpointOverride(SECRETSMANAGER); + try (SecretsManagerClient client = + SecretsManagerClient.builder() + .endpointOverride(secretsManagerEndpoint) + .region(Region.of(localstack.getRegion())) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create( + localstack.getAccessKey(), localstack.getSecretKey()))) + .build()) { + + String instantStr = "2024-06-05T20:38:16Z"; + Instant instant = Instant.parse(instantStr); + + KeySecret keySecret = keySecret("secret-foo"); + BasicCredentials basicCred = basicCredentials("bar-name", "bar-secret"); + TokenSecret tokenSec = tokenSecret("the-token", instant); + + client.createSecret( + CreateSecretRequest.builder().name("key").secretString("secret-foo").build()); + client.createSecret( + CreateSecretRequest.builder() + .name("basic") + .secretString("{\"name\": \"bar-name\", \"secret\": \"bar-secret\"}") + .build()); + client.createSecret( + CreateSecretRequest.builder() + .name("tok") + .secretString("{\"token\": \"the-token\", \"expiresAt\": \"" + instantStr + "\"}") + .build()); + + AwsSecretsSupplier secretsSupplier = + new AwsSecretsSupplier(client, "", Duration.ofMinutes(1)); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of( + "key", + SecretType.KEY, + "basic", + SecretType.BASIC, + "tok", + SecretType.EXPIRING_TOKEN))) + .hasSize(3) + .hasEntrySatisfying( + "key", + s -> + assertThat(s) + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key())) + .hasEntrySatisfying( + "basic", + s -> + assertThat(s) + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret())) + .hasEntrySatisfying( + "tok", + s -> + assertThat(s) + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt())); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of( + "key", + SecretType.KEY, + "nope", + SecretType.BASIC, + "basic", + SecretType.BASIC, + "not-there", + SecretType.KEY, + "tok", + SecretType.EXPIRING_TOKEN))) + .hasSize(3) + .hasEntrySatisfying( + "key", + s -> + assertThat(s) + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key())) + .hasEntrySatisfying( + "basic", + s -> + assertThat(s) + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret())) + .hasEntrySatisfying( + "tok", + s -> + assertThat(s) + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt())); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of("nope", SecretType.BASIC, "not-there", SecretType.KEY))) + .isEmpty(); + } + } +} diff --git a/catalog/secrets/aws/src/intTest/resources/logback-test.xml b/catalog/secrets/aws/src/intTest/resources/logback-test.xml new file mode 100644 index 00000000000..ce140f00774 --- /dev/null +++ b/catalog/secrets/aws/src/intTest/resources/logback-test.xml @@ -0,0 +1,30 @@ + + + + + + + %date{ISO8601} [%thread] [%X{nessie.events.subscription.id}] %-5level %logger{36} - + %msg%n + + + + + + diff --git a/catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version b/catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version new file mode 100644 index 00000000000..5e5015a43ea --- /dev/null +++ b/catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version @@ -0,0 +1,3 @@ +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM docker.io/localstack/localstack:3.5.0 diff --git a/catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsSupplier.java b/catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsSupplier.java new file mode 100644 index 00000000000..7988aa192f1 --- /dev/null +++ b/catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsSupplier.java @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.aws; + +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.spi.SingleValueSecretsSupplier; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.BatchGetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.SecretValueEntry; + +public class AwsSecretsSupplier extends SingleValueSecretsSupplier { + private final SecretsManagerClient secretsManagerClient; + private final String prefix; + + public AwsSecretsSupplier( + SecretsManagerClient secretsManagerClient, String prefix, Duration getSecretTimeout) { + this.secretsManagerClient = secretsManagerClient; + this.prefix = prefix; + } + + @Override + protected Map resolveSingleValueSecrets(Map toResolve) { + if (toResolve.isEmpty()) { + return Map.of(); + } + + Map secretIdToNameMap = new HashMap<>(); + + for (String name : toResolve.keySet()) { + String secretId = nameToSecretId(name); + secretIdToNameMap.put(secretId, name); + } + + return secretsManagerClient + .batchGetSecretValue( + BatchGetSecretValueRequest.builder().secretIdList(secretIdToNameMap.keySet()).build()) + .secretValues() + .stream() + .collect( + Collectors.toMap( + sv -> secretIdToNameMap.get(sv.name()), SecretValueEntry::secretString)); + } + + private String nameToSecretId(String name) { + return prefix + name; + } +} diff --git a/catalog/secrets/azure/build.gradle.kts b/catalog/secrets/azure/build.gradle.kts new file mode 100644 index 00000000000..0d726850bf7 --- /dev/null +++ b/catalog/secrets/azure/build.gradle.kts @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets Azure" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(enforcedPlatform(libs.quarkus.azure.services.bom)) + implementation("com.azure:azure-security-keyvault-secrets") + implementation("com.azure:azure-identity") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) + + intTestCompileOnly(project(":nessie-immutables")) + intTestImplementation(platform(libs.testcontainers.bom)) + intTestImplementation("org.testcontainers:testcontainers") + intTestImplementation("org.testcontainers:junit-jupiter") + intTestImplementation(libs.lowkey.vault.testcontainers) + intTestImplementation(project(":nessie-container-spec-helper")) + intTestRuntimeOnly(libs.logback.classic) +} + +tasks.named("intTest") { systemProperty("javax.net.ssl.trustStore", "foo.bar.BazTrustStore") } diff --git a/catalog/secrets/azure/src/intTest/java/org/projectnessie/catalog/secrets/azure/ITAzureSecretsSupplier.java b/catalog/secrets/azure/src/intTest/java/org/projectnessie/catalog/secrets/azure/ITAzureSecretsSupplier.java new file mode 100644 index 00000000000..f67d83e9983 --- /dev/null +++ b/catalog/secrets/azure/src/intTest/java/org/projectnessie/catalog/secrets/azure/ITAzureSecretsSupplier.java @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.azure; + +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.type; +import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; +import static org.projectnessie.catalog.secrets.KeySecret.keySecret; +import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; + +import com.azure.identity.UsernamePasswordCredentialBuilder; +import com.azure.security.keyvault.secrets.SecretAsyncClient; +import com.azure.security.keyvault.secrets.SecretClientBuilder; +import com.github.nagyesta.lowkeyvault.testcontainers.LowkeyVaultContainer; +import com.github.nagyesta.lowkeyvault.testcontainers.LowkeyVaultContainerBuilder; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Map; +import java.util.Set; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.projectnessie.catalog.secrets.BasicCredentials; +import org.projectnessie.catalog.secrets.KeySecret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.TokenSecret; +import org.projectnessie.nessie.testing.containerspec.ContainerSpecHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +@ExtendWith(SoftAssertionsExtension.class) +@Disabled( + "Azure SecretClient requires an SSL connection, verifying the server certificates, which needs to used by the test container and trusted by the client. No way around it.") +public class ITAzureSecretsSupplier { + private static final Logger LOGGER = LoggerFactory.getLogger(ITAzureSecretsSupplier.class); + + @InjectSoftAssertions SoftAssertions soft; + + @Container + static LowkeyVaultContainer lowkeyVault = + LowkeyVaultContainerBuilder.lowkeyVault( + ContainerSpecHelper.builder() + .name("lowkey-vault") + .containerClass(ITAzureSecretsSupplier.class) + .build() + .dockerImageName(null) + .asCompatibleSubstituteFor("nagyesta/lowkey-vault")) + .vaultNames(Set.of("default")) + .build() + .withLogConsumer( + c -> LOGGER.info("[LOWKEY-VAULT] {}", c.getUtf8StringWithoutLineEnding())); + + @Test + public void azureSecrets() { + SecretAsyncClient client = + new SecretClientBuilder() + .vaultUrl(format("https://%s.localhost:%d", "default", lowkeyVault.getMappedPort(8443))) + .credential( + new UsernamePasswordCredentialBuilder() + .clientId("ITAzureSecretsSupplier") + .username(lowkeyVault.getUsername()) + .password(lowkeyVault.getPassword()) + .build()) + .buildAsyncClient(); + + String instantStr = "2024-06-05T20:38:16Z"; + Instant instant = Instant.parse(instantStr); + + KeySecret keySecret = keySecret("secret-foo"); + BasicCredentials basicCred = basicCredentials("bar-name", "bar-secret"); + TokenSecret tokenSec = tokenSecret("the-token", instant); + + client.setSecret("key", "secret-foo").block(Duration.of(1, ChronoUnit.MINUTES)); + client + .setSecret("basic", "{\"name\": \"bar-name\", \"secret\": \"bar-secret\"}") + .block(Duration.of(1, ChronoUnit.MINUTES)); + client + .setSecret("tok", "{\"token\": \"the-token\", \"expiresAt\": \"" + instantStr + "\"}") + .block(Duration.of(1, ChronoUnit.MINUTES)); + + AzureSecretsSupplier secretsSupplier = + new AzureSecretsSupplier(client, "", Duration.ofMinutes(1)); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of( + "key", + SecretType.KEY, + "basic", + SecretType.BASIC, + "tok", + SecretType.EXPIRING_TOKEN))) + .hasSize(3) + .hasEntrySatisfying( + "key", + s -> + assertThat(s) + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key())) + .hasEntrySatisfying( + "basic", + s -> + assertThat(s) + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret())) + .hasEntrySatisfying( + "tok", + s -> + assertThat(s) + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt())); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of( + "key", + SecretType.KEY, + "nope", + SecretType.BASIC, + "basic", + SecretType.BASIC, + "not-there", + SecretType.KEY, + "tok", + SecretType.EXPIRING_TOKEN))) + .hasSize(3) + .hasEntrySatisfying( + "key", + s -> + assertThat(s) + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key())) + .hasEntrySatisfying( + "basic", + s -> + assertThat(s) + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret())) + .hasEntrySatisfying( + "tok", + s -> + assertThat(s) + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt())); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of("nope", SecretType.BASIC, "not-there", SecretType.KEY))) + .isEmpty(); + } +} diff --git a/catalog/secrets/azure/src/intTest/resources/logback-test.xml b/catalog/secrets/azure/src/intTest/resources/logback-test.xml new file mode 100644 index 00000000000..ce140f00774 --- /dev/null +++ b/catalog/secrets/azure/src/intTest/resources/logback-test.xml @@ -0,0 +1,30 @@ + + + + + + + %date{ISO8601} [%thread] [%X{nessie.events.subscription.id}] %-5level %logger{36} - + %msg%n + + + + + + diff --git a/catalog/secrets/azure/src/intTest/resources/org/projectnessie/catalog/secrets/azure/Dockerfile-lowkey-vault-version b/catalog/secrets/azure/src/intTest/resources/org/projectnessie/catalog/secrets/azure/Dockerfile-lowkey-vault-version new file mode 100644 index 00000000000..8907a241ad1 --- /dev/null +++ b/catalog/secrets/azure/src/intTest/resources/org/projectnessie/catalog/secrets/azure/Dockerfile-lowkey-vault-version @@ -0,0 +1,3 @@ +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM docker.io/nagyesta/lowkey-vault:2.4.42 diff --git a/catalog/secrets/azure/src/main/java/org/projectnessie/catalog/secrets/azure/AzureSecretsSupplier.java b/catalog/secrets/azure/src/main/java/org/projectnessie/catalog/secrets/azure/AzureSecretsSupplier.java new file mode 100644 index 00000000000..b357ce2d000 --- /dev/null +++ b/catalog/secrets/azure/src/main/java/org/projectnessie/catalog/secrets/azure/AzureSecretsSupplier.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.azure; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import com.azure.core.exception.ResourceNotFoundException; +import com.azure.security.keyvault.secrets.SecretAsyncClient; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.spi.SingleValueSecretsSupplier; + +public class AzureSecretsSupplier extends SingleValueSecretsSupplier { + private final SecretAsyncClient client; + private final String prefix; + private final long timeout; + + public AzureSecretsSupplier(SecretAsyncClient client, String prefix, Duration getSecretTimeout) { + this.client = client; + this.prefix = prefix; + this.timeout = getSecretTimeout.toMillis(); + } + + @Override + protected Map resolveSingleValueSecrets(Map toResolve) { + if (toResolve.isEmpty()) { + return Map.of(); + } + + List>> futures = new ArrayList<>(); + + for (String name : toResolve.keySet()) { + String secretId = nameToSecretId(name); + futures.add( + client + .getSecret(secretId) + .map(s -> Map.entry(name, s.getValue())) + .toFuture() + .exceptionally( + t -> { + if (t instanceof ResourceNotFoundException) { + return null; + } + if (t instanceof RuntimeException) { + throw (RuntimeException) t; + } + throw new RuntimeException(t); + })); + } + + try { + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).get(timeout, MILLISECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new RuntimeException(e); + } + + return futures.stream() + .map(f -> f.getNow(null)) + .filter(Objects::nonNull) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private String nameToSecretId(String name) { + return prefix + name; + } +} diff --git a/catalog/secrets/cache/build.gradle.kts b/catalog/secrets/cache/build.gradle.kts new file mode 100644 index 00000000000..ae138fba28f --- /dev/null +++ b/catalog/secrets/cache/build.gradle.kts @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets Cache" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + implementation(libs.caffeine) + implementation(libs.micrometer.core) + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-databind") + implementation("com.fasterxml.jackson.core:jackson-annotations") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) +} diff --git a/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java new file mode 100644 index 00000000000..ac1053ee455 --- /dev/null +++ b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import org.projectnessie.catalog.secrets.spi.SecretsSupplier; + +public final class CachingSecrets { + + private final CachingSecretsBackend backend; + + public CachingSecrets(CachingSecretsBackend backend) { + this.backend = backend; + } + + public SecretsSupplier forRepository(String repositoryId, SecretsSupplier secretsSupplier) { + return toResolve -> backend.resolveSecrets(repositoryId, toResolve, secretsSupplier); + } +} diff --git a/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecretsBackend.java b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecretsBackend.java new file mode 100644 index 00000000000..229e1a6d2ce --- /dev/null +++ b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecretsBackend.java @@ -0,0 +1,251 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import static java.util.Collections.singletonList; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Expiry; +import com.google.common.annotations.VisibleForTesting; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.binder.cache.CaffeineStatsCounter; +import java.util.HashMap; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.LongSupplier; +import java.util.stream.Collectors; +import org.checkerframework.checker.index.qual.NonNegative; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.spi.SecretsSupplier; + +public class CachingSecretsBackend { + + private static final Secret CACHE_NEGATIVE_SENTINEL = new Secret() {}; + + public static final String CACHE_NAME = "nessie-secrets"; + private static final long NOT_CACHED = 0L; + + @VisibleForTesting final Cache cache; + private final long ttlNanos; + private final LongSupplier clock; + + public CachingSecretsBackend(SecretsCacheConfig config) { + OptionalLong ttl = config.ttlMillis(); + this.ttlNanos = ttl.isPresent() ? TimeUnit.MILLISECONDS.toNanos(ttl.getAsLong()) : 0L; + this.clock = config.clockNanos(); + + Caffeine cacheBuilder = + Caffeine.newBuilder() + .expireAfter( + new Expiry() { + @Override + public long expireAfterCreate( + CacheKeyValue key, Secret value, long currentTimeNanos) { + long expire = key.expiresAtNanosEpoch; + if (expire == NOT_CACHED) { + return 0L; + } + long remaining = expire - currentTimeNanos; + return Math.max(0L, remaining); + } + + @Override + public long expireAfterUpdate( + CacheKeyValue key, + Secret value, + long currentTimeNanos, + @NonNegative long currentDurationNanos) { + return expireAfterCreate(key, value, currentTimeNanos); + } + + @Override + public long expireAfterRead( + CacheKeyValue key, + Secret value, + long currentTimeNanos, + @NonNegative long currentDurationNanos) { + return currentDurationNanos; + } + }) + .ticker(clock::getAsLong) + .maximumSize(config.maxElements()); + config + .meterRegistry() + .ifPresent( + meterRegistry -> { + cacheBuilder.recordStats(() -> new CaffeineStatsCounter(meterRegistry, CACHE_NAME)); + meterRegistry.gauge( + "cache_max_size", + singletonList(Tag.of("cache", CACHE_NAME)), + "", + x -> config.maxElements()); + meterRegistry.gauge( + "cache_element_ttl", + singletonList(Tag.of("cache", CACHE_NAME)), + "", + x -> config.ttlMillis().orElse(0)); + }); + + this.cache = cacheBuilder.build(); + } + + private int weigher(CacheKeyValue key, Secret value) { + int size = key.heapSize(); + if (isPresentValue(value)) { + // We assume 1kB for every secret (which is probably way more than needed) + size += 1024 + MAP_OBJ_OVERHEAD; + } + size += CAFFEINE_OBJ_OVERHEAD; + return size; + } + + private static boolean isPresentValue(Secret value) { + // Yes, same instance for + return value != null && value != CACHE_NEGATIVE_SENTINEL; + } + + Map resolveSecrets( + String repositoryId, Map toResolve, SecretsSupplier backend) { + long ttl = ttlNanos; + long expires = ttl != 0L ? clock.getAsLong() + ttl : 0L; + + Map keysToResolve = new HashMap<>(); + for (String name : toResolve.keySet()) { + keysToResolve.put(name, new CacheKeyValue(repositoryId, name, expires)); + } + + Map result = new HashMap<>(); + + Map present = cache.getAllPresent(keysToResolve.values()); + present.forEach( + (k, v) -> { + String name = k.name; + keysToResolve.remove(name); + if (isPresentValue(v)) { + result.put(name, v); + } + }); + + Map resolve = + keysToResolve.keySet().stream() + .collect(Collectors.toMap(Function.identity(), toResolve::get)); + + if (!keysToResolve.isEmpty()) { + Map resolved = backend.resolveSecrets(resolve); + + Map put = new HashMap<>(); + for (CacheKeyValue key : keysToResolve.values()) { + Secret value = resolved.getOrDefault(key.name, CACHE_NEGATIVE_SENTINEL); + if (isPresentValue(value)) { + result.put(key.name, value); + } + put.put(key, value); + } + cache.putAll(put); + } + + return result; + } + + private static String safeStringValue(Map.Entry e) { + // The string doesn't necessarily need to be a string, for example if the secrets backend + // returns another object (Vault could do that?). + return e.getValue().toString(); + } + + static final class CacheKeyValue { + final String repositoryId; + final String name; + + // Revisit this field before 2262-04-11T23:47:16.854Z (64-bit signed long overflow) ;) ;) + final long expiresAtNanosEpoch; + + CacheKeyValue(String repositoryId, String name, long expiresAtNanosEpoch) { + this.repositoryId = repositoryId; + this.name = name; + this.expiresAtNanosEpoch = expiresAtNanosEpoch; + } + + int heapSize() { + int size = OBJ_SIZE; + size += STRING_OBJ_OVERHEAD + repositoryId.length(); + size += STRING_OBJ_OVERHEAD + name.length(); + return size; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CacheKeyValue)) { + return false; + } + CacheKeyValue cacheKey = (CacheKeyValue) o; + return repositoryId.equals(cacheKey.repositoryId) && name.equals(cacheKey.name); + } + + @Override + public int hashCode() { + return repositoryId.hashCode() * 31 + name.hashCode(); + } + + @Override + public String toString() { + return "{" + repositoryId + ", " + name + '}'; + } + } + + /* + CacheKeyValue object internals: + OFF SZ TYPE DESCRIPTION VALUE + 0 8 (object header: mark) 0x0000000000000001 (non-biasable; age: 0) + 8 4 (object header: class) 0x010c4800 + 12 4 java.lang.String CacheKeyValue.repositoryId null + 16 8 long CacheKeyValue.expiresAt 0 + 24 4 java.lang.String CacheKeyValue.id null + 28 4 (object alignment gap) + Instance size: 32 bytes + */ + static final int OBJ_SIZE = 32; + // rough estimate, probably good enough + static final int MAP_OBJ_OVERHEAD = OBJ_SIZE * 2; + /* + Array overhead: 16 bytes + */ + static final int ARRAY_OVERHEAD = 16; + /* + java.lang.String object internals: + OFF SZ TYPE DESCRIPTION VALUE + 0 8 (object header: mark) 0x0000000000000001 (non-biasable; age: 0) + 8 4 (object header: class) 0x0000e8d8 + 12 4 int String.hash 0 + 16 1 byte String.coder 0 + 17 1 boolean String.hashIsZero false + 18 2 (alignment/padding gap) + 20 4 byte[] String.value [] + Instance size: 24 bytes + */ + static final int STRING_OBJ_OVERHEAD = 24 + ARRAY_OVERHEAD; + /* + Assume an overhead of 2 objects for each entry (java.util.concurrent.ConcurrentHashMap$Node is 32 bytes) in Caffeine. + */ + static final int CAFFEINE_OBJ_OVERHEAD = 2 * 32; +} diff --git a/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/SecretsCacheConfig.java b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/SecretsCacheConfig.java new file mode 100644 index 00000000000..e004e9b7655 --- /dev/null +++ b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/SecretsCacheConfig.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import io.micrometer.core.instrument.MeterRegistry; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.function.LongSupplier; +import org.immutables.value.Value; + +@Value.Immutable +public interface SecretsCacheConfig { + + long maxElements(); + + OptionalLong ttlMillis(); + + Optional meterRegistry(); + + @Value.Default + default LongSupplier clockNanos() { + return System::nanoTime; + } + + static Builder builder() { + return ImmutableSecretsCacheConfig.builder(); + } + + interface Builder { + @CanIgnoreReturnValue + Builder maxElements(long maxElements); + + @CanIgnoreReturnValue + Builder ttlMillis(long ttlMillis); + + @CanIgnoreReturnValue + Builder meterRegistry(MeterRegistry meterRegistry); + + @CanIgnoreReturnValue + Builder clockNanos(LongSupplier clockNanos); + + SecretsCacheConfig build(); + } +} diff --git a/catalog/secrets/cache/src/test/java/org/projectnessie/catalog/secrets/cache/TestCachingSecretsBackend.java b/catalog/secrets/cache/src/test/java/org/projectnessie/catalog/secrets/cache/TestCachingSecretsBackend.java new file mode 100644 index 00000000000..727cfee753f --- /dev/null +++ b/catalog/secrets/cache/src/test/java/org/projectnessie/catalog/secrets/cache/TestCachingSecretsBackend.java @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import static java.util.function.Function.identity; +import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; +import static org.projectnessie.catalog.secrets.KeySecret.keySecret; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.spi.SecretsSupplier; + +@ExtendWith(SoftAssertionsExtension.class) +public class TestCachingSecretsBackend { + @InjectSoftAssertions protected SoftAssertions soft; + + static final Secret SECRET_1_REPO_1 = keySecret("secret1-repo1"); + static final Secret SECRET_2_REPO_1 = basicCredentials("secret1-name", "secret1-value"); + static final Secret SECRET_1_REPO_2 = keySecret("secret1-repo2"); + static final Secret SECRET_2_REPO_2 = basicCredentials("secret2-name", "secret2-value"); + static final Map SECRETS_1 = Map.of("secret1", SecretType.KEY); + static final Map SECRETS_1_2 = + Map.of("secret1", SecretType.KEY, "secret2", SecretType.KEY); + + AtomicLong clock; + CachingSecretsBackend backend; + CachingSecrets cachingSecrets; + SecretsSupplier secrets1; + SecretsSupplier secrets2; + SecretsSupplier caching1; + SecretsSupplier caching2; + + @BeforeEach + void setup() { + clock = new AtomicLong(); + backend = + new CachingSecretsBackend( + SecretsCacheConfig.builder() + .maxElements(100) + .ttlMillis(1000) + .clockNanos(clock::get) + .build()); + + cachingSecrets = new CachingSecrets(backend); + + secrets1 = mapSecretsSupplier(Map.of("secret1", SECRET_1_REPO_1, "secret2", SECRET_2_REPO_1)); + secrets2 = mapSecretsSupplier(Map.of("secret1", SECRET_1_REPO_2, "secret2", SECRET_2_REPO_2)); + + caching1 = cachingSecrets.forRepository("repo1", secrets1); + caching2 = cachingSecrets.forRepository("repo2", secrets2); + } + + @Test + public void ttlExpire() { + soft.assertThat(caching1.resolveSecrets(SECRETS_1)) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_1)); + soft.assertThat(caching2.resolveSecrets(SECRETS_1)) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_2)); + soft.assertThat(caching1.resolveSecrets(SECRETS_1)) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_1)); + soft.assertThat(caching2.resolveSecrets(SECRETS_1)) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_2)); + + soft.assertThat(backend.cache.asMap()) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret1", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret1", 0)); + + clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(1000)); + + soft.assertThat(backend.cache.asMap()) + .doesNotContainKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret1", 0)) + .doesNotContainKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret1", 0)); + + soft.assertThat(caching1.resolveSecrets(SECRETS_1_2)) + .containsExactlyInAnyOrderEntriesOf( + Map.of("secret1", SECRET_1_REPO_1, "secret2", SECRET_2_REPO_1)); + soft.assertThat(caching2.resolveSecrets(SECRETS_1_2)) + .containsExactlyInAnyOrderEntriesOf( + Map.of("secret1", SECRET_1_REPO_2, "secret2", SECRET_2_REPO_2)); + + soft.assertThat(backend.cache.asMap()) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret1", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret1", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret2", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret2", 0)); + } + + @Test + public void nonExisting() { + soft.assertThat( + caching1.resolveSecrets( + Map.of("secret-nope1", SecretType.KEY, "secret-nope2", SecretType.KEY))) + .isEmpty(); + soft.assertThat( + caching2.resolveSecrets( + Map.of("secret-nope1", SecretType.KEY, "secret-nope2", SecretType.KEY))) + .isEmpty(); + soft.assertThat( + caching1.resolveSecrets( + Map.of( + "secret-nope1", + SecretType.KEY, + "secret-nope2", + SecretType.KEY, + "secret1", + SecretType.KEY))) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_1)); + soft.assertThat( + caching2.resolveSecrets( + Map.of( + "secret-nope1", + SecretType.KEY, + "secret-nope2", + SecretType.KEY, + "secret1", + SecretType.KEY))) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_2)); + } + + @Test + public void tooManyEntries() { + SecretsSupplier supplier = + toResolve -> + toResolve.keySet().stream() + .collect(Collectors.toMap(identity(), name -> keySecret(name))); + SecretsSupplier caching = cachingSecrets.forRepository("repoMany", supplier); + + // Very naive test that checks that expiration happens - our "weigher" does something. + + for (int i = 0; i < 100_000; i++) { + caching.resolveSecrets( + Map.of( + "wepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoew-" + + i, + SecretType.KEY)); + // In fact, the cache should never have more than 10000 entries, but 'estimatedSize' is really + // just an estimate. + soft.assertThat(backend.cache.estimatedSize()).isLessThan(10_000); + } + } + + static SecretsSupplier mapSecretsSupplier(Map secretsMap) { + return toResolve -> { + Map resolved = new HashMap<>(); + for (String name : toResolve.keySet()) { + Secret r = secretsMap.get(name); + if (r != null) { + resolved.put(name, r); + } + } + return resolved; + }; + } +} diff --git a/catalog/secrets/gcs/build.gradle.kts b/catalog/secrets/gcs/build.gradle.kts new file mode 100644 index 00000000000..2641bced3a6 --- /dev/null +++ b/catalog/secrets/gcs/build.gradle.kts @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets GCS" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(platform(libs.google.cloud.secretmanager.bom)) + implementation("com.google.cloud:google-cloud-secretmanager") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) +} diff --git a/catalog/secrets/gcs/src/main/java/org/projectnessie/catalog/secrets/gcs/GcsSecretsSupplier.java b/catalog/secrets/gcs/src/main/java/org/projectnessie/catalog/secrets/gcs/GcsSecretsSupplier.java new file mode 100644 index 00000000000..dd03d4cfa7b --- /dev/null +++ b/catalog/secrets/gcs/src/main/java/org/projectnessie/catalog/secrets/gcs/GcsSecretsSupplier.java @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.gcs; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.cloud.secretmanager.v1.AccessSecretVersionRequest; +import com.google.cloud.secretmanager.v1.AccessSecretVersionResponse; +import com.google.cloud.secretmanager.v1.SecretManagerServiceClient; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.spi.SingleValueSecretsSupplier; + +public class GcsSecretsSupplier extends SingleValueSecretsSupplier { + private final String prefix; + private final SecretManagerServiceClient client; + private final long getSecretTimeout; + + public GcsSecretsSupplier( + SecretManagerServiceClient client, String prefix, Duration getSecretTimeout) { + this.client = client; + this.prefix = prefix; + this.getSecretTimeout = getSecretTimeout.toMillis(); + } + + @Override + protected Map resolveSingleValueSecrets(Map toResolve) { + if (toResolve.isEmpty()) { + return Map.of(); + } + + Set toResolveKeys = toResolve.keySet(); + + if (toResolve.size() == 1) { + String name = toResolveKeys.iterator().next(); + String secretId = nameToSecretId(name); + AccessSecretVersionResponse response = client.accessSecretVersion(request(secretId)); + return response.hasPayload() ? Map.of(name, secret(response)) : Map.of(); + } + + Map secretIdToNameMap = new HashMap<>(); + + for (String name : toResolveKeys) { + String secretId = nameToSecretId(name); + secretIdToNameMap.put(secretId, name); + } + + List> futures = + secretIdToNameMap.keySet().stream() + .map(id -> client.accessSecretVersionCallable().futureCall(request(id))) + .collect(Collectors.toList()); + + List results; + try { + results = ApiFutures.successfulAsList(futures).get(getSecretTimeout, MILLISECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new RuntimeException(e); + } + + return results.stream() + .collect(Collectors.toMap(resp -> secretIdToNameMap.get(resp.getName()), this::secret)); + } + + private String secret(AccessSecretVersionResponse response) { + return response.getPayload().getData().toStringUtf8(); + } + + private AccessSecretVersionRequest request(String name) { + return AccessSecretVersionRequest.newBuilder().setName(name).build(); + } + + private String nameToSecretId(String name) { + return prefix + name; + } +} diff --git a/catalog/secrets/vault/build.gradle.kts b/catalog/secrets/vault/build.gradle.kts new file mode 100644 index 00000000000..69048c5422a --- /dev/null +++ b/catalog/secrets/vault/build.gradle.kts @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets Vault" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(enforcedPlatform(libs.quarkus.bom)) + implementation("io.quarkiverse.vault:quarkus-vault") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) + + intTestImplementation(platform(libs.testcontainers.bom)) + intTestImplementation("org.testcontainers:testcontainers") + intTestImplementation("org.testcontainers:vault") + intTestImplementation("org.testcontainers:junit-jupiter") + intTestImplementation(project(":nessie-container-spec-helper")) + intTestImplementation(libs.smallrye.config.core) + intTestCompileOnly(libs.immutables.value.annotations) + intTestRuntimeOnly(libs.logback.classic) +} diff --git a/catalog/secrets/vault/src/intTest/java/org/projectnessie/catalog/secrets/vault/ITVaultSecretsSupplier.java b/catalog/secrets/vault/src/intTest/java/org/projectnessie/catalog/secrets/vault/ITVaultSecretsSupplier.java new file mode 100644 index 00000000000..be1e5393ea9 --- /dev/null +++ b/catalog/secrets/vault/src/intTest/java/org/projectnessie/catalog/secrets/vault/ITVaultSecretsSupplier.java @@ -0,0 +1,201 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.vault; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.type; +import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; +import static org.projectnessie.catalog.secrets.KeySecret.keySecret; +import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; + +import io.quarkus.vault.VaultKVSecretReactiveEngine; +import io.quarkus.vault.client.VaultClient; +import io.quarkus.vault.runtime.VaultConfigHolder; +import io.quarkus.vault.runtime.VaultKvManager; +import io.quarkus.vault.runtime.client.VaultClientProducer; +import io.quarkus.vault.runtime.config.VaultRuntimeConfig; +import io.smallrye.config.PropertiesConfigSource; +import io.smallrye.config.SmallRyeConfig; +import io.smallrye.config.SmallRyeConfigBuilder; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.projectnessie.catalog.secrets.BasicCredentials; +import org.projectnessie.catalog.secrets.KeySecret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.TokenSecret; +import org.projectnessie.nessie.testing.containerspec.ContainerSpecHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.vault.VaultContainer; + +@Testcontainers +@ExtendWith(SoftAssertionsExtension.class) +public class ITVaultSecretsSupplier { + private static final Logger LOGGER = LoggerFactory.getLogger(ITVaultSecretsSupplier.class); + + public static final String VAULT_ROOT_TOKEN = "root"; + public static final String NESSIE_SECRETS_PATH = "apps/nessie/secrets"; + + @InjectSoftAssertions SoftAssertions soft; + + @SuppressWarnings("resource") + @Container + static VaultContainer vaultContainer = + new VaultContainer<>( + ContainerSpecHelper.builder() + .name("vault") + .containerClass(ITVaultSecretsSupplier.class) + .build() + .dockerImageName(null) + .asCompatibleSubstituteFor("vault")) + .withLogConsumer(c -> LOGGER.info("[VAULT] {}", c.getUtf8StringWithoutLineEnding())) + .withVaultToken(VAULT_ROOT_TOKEN); + + @Test + public void vault() { + SmallRyeConfig config = + new SmallRyeConfigBuilder() + .setAddDefaultSources(false) + .setAddDiscoveredSources(false) + .withMapping(VaultRuntimeConfig.class) + .withSources( + new PropertiesConfigSource( + Map.of( + "quarkus.vault.url", + vaultContainer.getHttpHostAddress(), + "quarkus.vault.authentication.client-token", + VAULT_ROOT_TOKEN, + "quarkus.vault.secret-config-kv-path", + NESSIE_SECRETS_PATH), + "configSource", + 100)) + .build(); + + VaultRuntimeConfig runtimeConfig = config.getConfigMapping(VaultRuntimeConfig.class); + + VaultConfigHolder configHolder = new VaultConfigHolder(); + configHolder.setVaultRuntimeConfig(runtimeConfig); + VaultClient client = new VaultClientProducer().privateVaultClient(configHolder, true); + + VaultKVSecretReactiveEngine vault = new VaultKvManager(client, configHolder); + + Instant instant = Instant.parse("2024-06-05T20:38:16Z"); + + KeySecret keySecret = keySecret("secret-foo"); + BasicCredentials basicCred = basicCredentials("bar-name", "bar-secret"); + TokenSecret tokenSec = tokenSecret("the-token", instant); + + vault.writeSecret("key", Map.of("key", keySecret.key())).await().indefinitely(); + vault + .writeSecret("basic", Map.of("name", basicCred.name(), "secret", basicCred.secret())) + .await() + .indefinitely(); + vault + .writeSecret( + "tok", + Map.of( + "token", + tokenSec.token(), + "expiresAt", + tokenSec.expiresAt().orElseThrow().toString())) + .await() + .indefinitely(); + + VaultSecretsSupplier secretsSupplier = + new VaultSecretsSupplier(vault, "", Duration.ofMinutes(1)); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of( + "key", + SecretType.KEY, + "basic", + SecretType.BASIC, + "tok", + SecretType.EXPIRING_TOKEN))) + .hasSize(3) + .hasEntrySatisfying( + "key", + s -> + assertThat(s) + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key())) + .hasEntrySatisfying( + "basic", + s -> + assertThat(s) + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret())) + .hasEntrySatisfying( + "tok", + s -> + assertThat(s) + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt())); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of( + "key", + SecretType.KEY, + "nope", + SecretType.BASIC, + "basic", + SecretType.BASIC, + "not-there", + SecretType.KEY, + "tok", + SecretType.EXPIRING_TOKEN))) + .hasSize(3) + .hasEntrySatisfying( + "key", + s -> + assertThat(s) + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key())) + .hasEntrySatisfying( + "basic", + s -> + assertThat(s) + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret())) + .hasEntrySatisfying( + "tok", + s -> + assertThat(s) + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt())); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of("nope", SecretType.BASIC, "not-there", SecretType.KEY))) + .isEmpty(); + } +} diff --git a/catalog/secrets/vault/src/intTest/resources/logback-test.xml b/catalog/secrets/vault/src/intTest/resources/logback-test.xml new file mode 100644 index 00000000000..4f3fda900d7 --- /dev/null +++ b/catalog/secrets/vault/src/intTest/resources/logback-test.xml @@ -0,0 +1,30 @@ + + + + + + + %date{ISO8601} [%thread] [%X{nessie.events.subscription.id}] %-5level %logger{36} - + %msg%n + + + + + + diff --git a/catalog/secrets/vault/src/intTest/resources/org/projectnessie/catalog/secrets/vault/Dockerfile-vault-version b/catalog/secrets/vault/src/intTest/resources/org/projectnessie/catalog/secrets/vault/Dockerfile-vault-version new file mode 100644 index 00000000000..f019422d362 --- /dev/null +++ b/catalog/secrets/vault/src/intTest/resources/org/projectnessie/catalog/secrets/vault/Dockerfile-vault-version @@ -0,0 +1,3 @@ +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM docker.io/hashicorp/vault:1.17.0 diff --git a/catalog/secrets/vault/src/main/java/org/projectnessie/catalog/secrets/vault/VaultSecretsSupplier.java b/catalog/secrets/vault/src/main/java/org/projectnessie/catalog/secrets/vault/VaultSecretsSupplier.java new file mode 100644 index 00000000000..81c7b6835b1 --- /dev/null +++ b/catalog/secrets/vault/src/main/java/org/projectnessie/catalog/secrets/vault/VaultSecretsSupplier.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.vault; + +import io.quarkus.vault.VaultKVSecretReactiveEngine; +import io.quarkus.vault.client.VaultClientException; +import io.smallrye.mutiny.Uni; +import java.time.Duration; +import java.util.Map; +import java.util.stream.Collectors; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.spi.SecretsSupplier; + +public class VaultSecretsSupplier implements SecretsSupplier { + private final VaultKVSecretReactiveEngine secretEngine; + private final String prefix; + private final Duration getSecretTimeout; + + public VaultSecretsSupplier( + VaultKVSecretReactiveEngine secretEngine, String prefix, Duration getSecretTimeout) { + this.secretEngine = secretEngine; + this.prefix = prefix; + this.getSecretTimeout = getSecretTimeout; + } + + @Override + public Map resolveSecrets(Map toResolve) { + return Uni.join() + .all( + toResolve.keySet().stream() + .map( + name -> + secretEngine + .readSecret(path(name)) + .map(json -> Map.entry(name, json)) + .onFailure( + e -> { + if (!(e instanceof VaultClientException)) { + return false; + } + VaultClientException ex = (VaultClientException) e; + return 404 == ex.getStatus(); + }) + .recoverWithItem(Map.entry(name, Map.of()))) + .collect(Collectors.toList())) + .andCollectFailures() + .await() + .atMost(getSecretTimeout) + .stream() + .filter(e -> !e.getValue().isEmpty()) + .collect( + Collectors.toMap( + Map.Entry::getKey, e -> toResolve.get(e.getKey()).fromValueMap(e.getValue()))); + } + + private String path(String name) { + return prefix + name.replace('.', '/'); + } +} diff --git a/cli/cli/build.gradle.kts b/cli/cli/build.gradle.kts index 1ee8de3c867..95ca10454b5 100644 --- a/cli/cli/build.gradle.kts +++ b/cli/cli/build.gradle.kts @@ -90,6 +90,7 @@ dependencies { intTestImplementation(project(":nessie-keycloak-testcontainer")) intTestImplementation(project(":nessie-container-spec-helper")) intTestImplementation(project(":nessie-catalog-secrets-api")) + intTestImplementation(testFixtures(project(":nessie-catalog-secrets-api"))) nessieQuarkusServer(project(":nessie-quarkus", "quarkusRunner")) } diff --git a/cli/cli/src/intTest/java/org/projectnessie/nessie/cli/commands/WithNessie.java b/cli/cli/src/intTest/java/org/projectnessie/nessie/cli/commands/WithNessie.java index da5e8047e89..83afb99015c 100644 --- a/cli/cli/src/intTest/java/org/projectnessie/nessie/cli/commands/WithNessie.java +++ b/cli/cli/src/intTest/java/org/projectnessie/nessie/cli/commands/WithNessie.java @@ -27,7 +27,6 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; import org.junit.jupiter.api.AfterAll; import org.projectnessie.catalog.files.api.ObjectIO; import org.projectnessie.catalog.files.s3.S3BucketOptions; @@ -42,6 +41,7 @@ import org.projectnessie.catalog.formats.iceberg.fixtures.IcebergGenerateFixtures; import org.projectnessie.catalog.formats.iceberg.meta.IcebergTableMetadata; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.client.api.NessieApiV2; import org.projectnessie.model.Branch; import org.projectnessie.model.CommitMeta; @@ -127,10 +127,7 @@ protected static void setupObjectStoreAndNessie( httpClient, s3config, s3options, - new SecretsProvider( - (names) -> - names.stream() - .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))), + new SecretsProvider(new DummySecretsSupplier()), sessions); ObjectIO objectIO = new S3ObjectIO(clientSupplier, Clock.systemUTC()); diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ab23b36ef36..3afc1520624 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -65,6 +65,7 @@ gatling-charts-highcharts = { module = "io.gatling.highcharts:gatling-charts-hig google-cloud-bigtable-bom = { module = "com.google.cloud:google-cloud-bigtable-bom", version = "2.39.5" } google-cloud-gcs-connector = { module = "com.google.cloud.bigdataoss:gcs-connector", version = "hadoop3-2.2.21" } google-cloud-nio = { module = "com.google.cloud:google-cloud-nio", version = "0.127.19" } +google-cloud-secretmanager-bom = { module = "com.google.cloud:google-cloud-secretmanager-bom", version = "2.45.0" } google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.40.1" } google-java-format = { module = "com.google.googlejavaformat:google-java-format", version.ref = "googleJavaFormat" } guava = { module = "com.google.guava:guava", version = "33.2.1-jre" } @@ -104,6 +105,7 @@ junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api" } junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params" } junit-platform-reporting = { module = "org.junit.platform:junit-platform-reporting" } logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logback" } +lowkey-vault-testcontainers = { module = "com.github.nagyesta.lowkey-vault:lowkey-vault-testcontainers", version = "2.4.42" } keycloak-admin-client = { module = "org.keycloak:keycloak-admin-client", version.ref = "keycloak" } mariadb-java-client = { module = "org.mariadb.jdbc:mariadb-java-client", version = "3.4.0" } maven-resolver-supplier = { module = "org.apache.maven.resolver:maven-resolver-supplier", version.ref = "mavenResolver" } @@ -128,6 +130,7 @@ picocli-codegen = { module = "info.picocli:picocli-codegen", version.ref = "pico postgresql = { module = "org.postgresql:postgresql", version = "42.7.3" } protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" } quarkus-amazon-services-bom = { module = "io.quarkus.platform:quarkus-amazon-services-bom", version.ref = "quarkusPlatform" } +quarkus-azure-services-bom = { module = "io.quarkiverse.azureservices:quarkus-azure-services-bom", version = "1.0.4" } quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quarkusPlatform" } quarkus-cassandra-bom = { module = "io.quarkus.platform:quarkus-cassandra-bom", version.ref = "quarkusPlatform" } quarkus-google-cloud-services-bom = { module = "io.quarkus.platform:quarkus-google-cloud-services-bom", version.ref = "quarkusPlatform" } diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 0d0c16ca921..08c3959d6dd 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -90,3 +90,8 @@ nessie-catalog-service-common=catalog/service/common nessie-catalog-service-rest=catalog/service/rest nessie-catalog-service-impl=catalog/service/impl nessie-catalog-secrets-api=catalog/secrets/api +nessie-catalog-secrets-cache=catalog/secrets/cache +nessie-catalog-secrets-aws=catalog/secrets/aws +nessie-catalog-secrets-gcs=catalog/secrets/gcs +nessie-catalog-secrets-azure=catalog/secrets/azure +nessie-catalog-secrets-vault=catalog/secrets/vault