diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index a6ab0549f62..888d6615f40 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -106,6 +106,11 @@ dependencies { api(project(":nessie-catalog-service-impl")) api(project(":nessie-catalog-service-transfer")) api(project(":nessie-catalog-secrets-api")) + api(project(":nessie-catalog-secrets-cache")) + api(project(":nessie-catalog-secrets-aws")) + api(project(":nessie-catalog-secrets-gcs")) + api(project(":nessie-catalog-secrets-azure")) + api(project(":nessie-catalog-secrets-vault")) if (!isIncludedInNesQuEIT()) { api(project(":nessie-spark-antlr-runtime")) diff --git a/catalog/files/impl/build.gradle.kts b/catalog/files/impl/build.gradle.kts index cde07dd220e..6f476ac2826 100644 --- a/catalog/files/impl/build.gradle.kts +++ b/catalog/files/impl/build.gradle.kts @@ -63,11 +63,14 @@ dependencies { testFixturesApi(project(":nessie-object-storage-mock")) + testImplementation(testFixtures(project(":nessie-catalog-secrets-api"))) + testRuntimeOnly(libs.logback.classic) jmhImplementation(libs.jmh.core) jmhImplementation(project(":nessie-object-storage-mock")) jmhAnnotationProcessor(libs.jmh.generator.annprocess) + jmhImplementation(testFixtures(project(":nessie-catalog-secrets-api"))) } tasks.named("processJmhJandexIndex").configure { enabled = false } diff --git a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java index e4117f7c60b..eb87fc99ac2 100644 --- a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java +++ b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/adls/AdlsClientResourceBench.java @@ -24,8 +24,6 @@ import com.azure.core.http.HttpClient; import java.io.IOException; import java.io.InputStream; -import java.util.Map; -import java.util.stream.Collectors; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -40,6 +38,7 @@ import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.objectstoragemock.ObjectStorageMock; import org.projectnessie.storage.uri.StorageUri; @@ -76,12 +75,7 @@ public void init() { clientSupplier = new AdlsClientSupplier( - httpClient, - adlsOptions, - new SecretsProvider( - (names) -> - names.stream() - .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret"))))); + httpClient, adlsOptions, new SecretsProvider(new DummySecretsSupplier())); } @TearDown diff --git a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java index e825e0ecb5a..53d7c4edeae 100644 --- a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java +++ b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/gcs/GcsClientResourceBench.java @@ -23,8 +23,6 @@ import com.google.auth.http.HttpTransportFactory; import java.io.IOException; import java.io.InputStream; -import java.util.Map; -import java.util.stream.Collectors; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -40,6 +38,7 @@ import org.openjdk.jmh.infra.Blackhole; import org.projectnessie.catalog.secrets.SecretsProvider; import org.projectnessie.catalog.secrets.TokenSecret; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.objectstoragemock.ObjectStorageMock; import org.projectnessie.storage.uri.StorageUri; @@ -74,12 +73,7 @@ public void init() { storageSupplier = new GcsStorageSupplier( - httpTransportFactory, - gcsOptions, - new SecretsProvider( - (names) -> - names.stream() - .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret"))))); + httpTransportFactory, gcsOptions, new SecretsProvider(new DummySecretsSupplier())); } @TearDown diff --git a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java index 8c55cf1512c..d485614f54b 100644 --- a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java +++ b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3ClientResourceBench.java @@ -22,8 +22,6 @@ import java.io.IOException; import java.io.InputStream; -import java.util.Map; -import java.util.stream.Collectors; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -38,6 +36,7 @@ import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.objectstoragemock.ObjectStorageMock; import org.projectnessie.storage.uri.StorageUri; import software.amazon.awssdk.http.SdkHttpClient; @@ -62,7 +61,8 @@ public void init() { server = mockServer(mock -> {}); S3Config s3config = S3Config.builder().build(); - httpClient = S3Clients.apacheHttpClient(s3config, new SecretsProvider(names -> Map.of())); + httpClient = + S3Clients.apacheHttpClient(s3config, new SecretsProvider(new DummySecretsSupplier())); S3ProgrammaticOptions s3options = ImmutableS3ProgrammaticOptions.builder() @@ -79,13 +79,7 @@ public void init() { clientSupplier = new S3ClientSupplier( - httpClient, - s3options, - new SecretsProvider( - (names) -> - names.stream() - .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))), - sessions); + httpClient, s3options, new SecretsProvider(new DummySecretsSupplier()), sessions); } @TearDown diff --git a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3SessionCacheResourceBench.java b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3SessionCacheResourceBench.java index 4ebf9d69d2d..6076b34b297 100644 --- a/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3SessionCacheResourceBench.java +++ b/catalog/files/impl/src/jmh/java/org/projectnessie/catalog/files/s3/S3SessionCacheResourceBench.java @@ -23,7 +23,6 @@ import java.net.URI; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; @@ -43,6 +42,7 @@ import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.objectstoragemock.ObjectStorageMock; import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.regions.Region; @@ -72,7 +72,8 @@ public void init() { server = mockServer(mock -> {}); S3Config s3config = S3Config.builder().build(); - httpClient = S3Clients.apacheHttpClient(s3config, new SecretsProvider(names -> Map.of())); + httpClient = + S3Clients.apacheHttpClient(s3config, new SecretsProvider(new DummySecretsSupplier())); S3Options s3options = ImmutableS3ProgrammaticOptions.builder() diff --git a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsOptions.java b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsOptions.java index 2c683b492e5..dd8a30082d6 100644 --- a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsOptions.java +++ b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsOptions.java @@ -65,7 +65,7 @@ public interface AdlsOptions { AdlsFileSystemOptions::account, ImmutableAdlsNamedFileSystemOptions.Builder::account), secretAttribute( - "sasToken", + "sas-token", SecretType.KEY, AdlsFileSystemOptions::sasToken, ImmutableAdlsNamedFileSystemOptions.Builder::sasToken)); diff --git a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/gcs/GcsOptions.java b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/gcs/GcsOptions.java index 3cdb2ff0069..f1c63fc6dbd 100644 --- a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/gcs/GcsOptions.java +++ b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/gcs/GcsOptions.java @@ -95,22 +95,22 @@ public interface GcsOptions { List> SECRET_ATTRIBUTES = ImmutableList.of( secretAttribute( - "authCredentialsJson", + "auth-credentials-json", SecretType.KEY, GcsBucketOptions::authCredentialsJson, ImmutableGcsNamedBucketOptions.Builder::authCredentialsJson), secretAttribute( - "oauth2Token", + "oauth2-token", SecretType.EXPIRING_TOKEN, GcsBucketOptions::oauth2Token, ImmutableGcsNamedBucketOptions.Builder::oauth2Token), secretAttribute( - "encryptionKey", + "encryption-key", SecretType.KEY, GcsBucketOptions::encryptionKey, ImmutableGcsNamedBucketOptions.Builder::encryptionKey), secretAttribute( - "decryptionKey", + "decryption-key", SecretType.KEY, GcsBucketOptions::decryptionKey, ImmutableGcsNamedBucketOptions.Builder::decryptionKey)); diff --git a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/s3/S3Options.java b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/s3/S3Options.java index 07862ee10b0..98d607129c2 100644 --- a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/s3/S3Options.java +++ b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/s3/S3Options.java @@ -87,7 +87,7 @@ default int effectiveStsClientsCacheMaxEntries() { SECRET_ATTRIBUTES = ImmutableList.of( secretAttribute( - "accessKey", + "access-key", SecretType.BASIC, S3BucketOptions::accessKey, ImmutableS3NamedBucketOptions.Builder::accessKey)); diff --git a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/adls/TestAdlsClients.java b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/adls/TestAdlsClients.java index 5f8fceeee34..4bf62e1827e 100644 --- a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/adls/TestAdlsClients.java +++ b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/adls/TestAdlsClients.java @@ -18,12 +18,11 @@ import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; import com.azure.core.http.HttpClient; -import java.util.Map; -import java.util.stream.Collectors; import org.projectnessie.catalog.files.AbstractClients; import org.projectnessie.catalog.files.api.BackendExceptionMapper; import org.projectnessie.catalog.files.api.ObjectIO; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.objectstoragemock.ObjectStorageMock; import org.projectnessie.storage.uri.StorageUri; @@ -66,12 +65,7 @@ protected ObjectIO buildObjectIO( AdlsClientSupplier supplier = new AdlsClientSupplier( - httpClient, - adlsOptions.build(), - new SecretsProvider( - (names) -> - names.stream() - .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret"))))); + httpClient, adlsOptions.build(), new SecretsProvider(new DummySecretsSupplier())); return new AdlsObjectIO(supplier); } diff --git a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/gcs/TestGcsClients.java b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/gcs/TestGcsClients.java index c9bdd76f993..12ede66e7f0 100644 --- a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/gcs/TestGcsClients.java +++ b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/gcs/TestGcsClients.java @@ -18,12 +18,11 @@ import static org.projectnessie.catalog.files.gcs.GcsClients.buildSharedHttpTransportFactory; import com.google.auth.http.HttpTransportFactory; -import java.util.Map; -import java.util.stream.Collectors; import org.projectnessie.catalog.files.AbstractClients; import org.projectnessie.catalog.files.api.BackendExceptionMapper; import org.projectnessie.catalog.files.api.ObjectIO; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.objectstoragemock.ObjectStorageMock; import org.projectnessie.storage.uri.StorageUri; @@ -67,10 +66,7 @@ protected ObjectIO buildObjectIO( new GcsStorageSupplier( httpTransportFactory, gcsOptions.build(), - new SecretsProvider( - (names) -> - names.stream() - .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret"))))); + new SecretsProvider(new DummySecretsSupplier())); return new GcsObjectIO(supplier); } diff --git a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/s3/TestS3Clients.java b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/s3/TestS3Clients.java index cefc64ff049..6971b1e57ef 100644 --- a/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/s3/TestS3Clients.java +++ b/catalog/files/impl/src/test/java/org/projectnessie/catalog/files/s3/TestS3Clients.java @@ -15,7 +15,6 @@ */ package org.projectnessie.catalog.files.s3; -import static java.util.function.Function.identity; import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; import static org.projectnessie.catalog.secrets.KeySecret.keySecret; @@ -25,8 +24,6 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.security.KeyStore; -import java.util.Map; -import java.util.stream.Collectors; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -35,6 +32,7 @@ import org.projectnessie.catalog.files.api.BackendExceptionMapper; import org.projectnessie.catalog.files.api.ObjectIO; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.objectstoragemock.ObjectStorageMock; import org.projectnessie.storage.uri.StorageUri; import software.amazon.awssdk.http.SdkHttpClient; @@ -46,7 +44,8 @@ public class TestS3Clients extends AbstractClients { @BeforeAll static void createHttpClient() { S3Config s3Config = S3Config.builder().build(); - sdkHttpClient = S3Clients.apacheHttpClient(s3Config, new SecretsProvider(names -> Map.of())); + sdkHttpClient = + S3Clients.apacheHttpClient(s3Config, new SecretsProvider(new DummySecretsSupplier())); } @AfterAll @@ -84,10 +83,7 @@ protected ObjectIO buildObjectIO( new S3ClientSupplier( sdkHttpClient, s3options.build(), - new SecretsProvider( - names -> - names.stream() - .collect(Collectors.toMap(identity(), k -> Map.of("secret", "secret")))), + new SecretsProvider(new DummySecretsSupplier()), null); return new S3ObjectIO(supplier); } @@ -121,7 +117,7 @@ public void invalidTrustStore(@TempDir Path tempDir) throws Exception { () -> S3Clients.apacheHttpClient( S3Config.builder().trustStorePath(file).build(), - new SecretsProvider(names -> Map.of())) + new SecretsProvider(new DummySecretsSupplier())) .close()) .withMessage("No trust store type"); soft.assertThatThrownBy( @@ -132,7 +128,7 @@ public void invalidTrustStore(@TempDir Path tempDir) throws Exception { .trustStoreType("jks") .trustStorePassword(keySecret(password)) .build(), - new SecretsProvider(names -> Map.of())) + new SecretsProvider(new DummySecretsSupplier())) .close()) .isInstanceOf(RuntimeException.class) .cause() @@ -145,7 +141,7 @@ public void invalidTrustStore(@TempDir Path tempDir) throws Exception { .trustStoreType("jks") .trustStorePassword(keySecret("wrong_password")) .build(), - new SecretsProvider(names -> Map.of())) + new SecretsProvider(new DummySecretsSupplier())) .close()) .isInstanceOf(RuntimeException.class) .cause() @@ -158,7 +154,7 @@ public void invalidTrustStore(@TempDir Path tempDir) throws Exception { .trustStoreType("jks") .trustStorePassword(keySecret(password)) .build(), - new SecretsProvider(names -> Map.of())) + new SecretsProvider(new DummySecretsSupplier())) .close()) .doesNotThrowAnyException(); } @@ -181,7 +177,7 @@ public void invalidKeyStore(@TempDir Path tempDir) throws Exception { () -> S3Clients.apacheHttpClient( S3Config.builder().keyStorePath(file).build(), - new SecretsProvider(names -> Map.of())) + new SecretsProvider(new DummySecretsSupplier())) .close()) .withMessage("No key store type"); soft.assertThatThrownBy( @@ -192,7 +188,7 @@ public void invalidKeyStore(@TempDir Path tempDir) throws Exception { .keyStoreType("jks") .keyStorePassword(keySecret(password)) .build(), - new SecretsProvider(names -> Map.of())) + new SecretsProvider(new DummySecretsSupplier())) .close()) .isInstanceOf(RuntimeException.class) .cause() @@ -205,7 +201,7 @@ public void invalidKeyStore(@TempDir Path tempDir) throws Exception { .keyStoreType("jks") .keyStorePassword(keySecret("wrong_password")) .build(), - new SecretsProvider(names -> Map.of())) + new SecretsProvider(new DummySecretsSupplier())) .close()) .isInstanceOf(RuntimeException.class) .cause() @@ -218,7 +214,7 @@ public void invalidKeyStore(@TempDir Path tempDir) throws Exception { .keyStoreType("jks") .keyStorePassword(keySecret(password)) .build(), - new SecretsProvider(names -> Map.of())) + new SecretsProvider(new DummySecretsSupplier())) .close()) .doesNotThrowAnyException(); } diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java index 87af479ab8e..fffe0a6f863 100644 --- a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java @@ -15,5 +15,12 @@ */ package org.projectnessie.catalog.secrets; -/** Base interface for all secrets. */ +/** + * Base interface for all secrets. + * + *

Secrets must not implement (override) any of these functions in a way: {@link + * Object#toString()}, {@link Object#hashCode()} or {@link Object#equals(Object)} that would + * directly (for example return a secret value from {@code toString()}) or indirectly (compare the + * instance itself against another instance) expose the values of a secret. + */ public interface Secret {} diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretJsonParser.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretJsonParser.java new file mode 100644 index 00000000000..cf05230c820 --- /dev/null +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretJsonParser.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Map; + +final class SecretJsonParser { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private SecretJsonParser() {} + + @SuppressWarnings("unchecked") + static Map parseOrSingle(String s) { + if (!s.trim().startsWith("{")) { + return Map.of("value", s); + } + try { + return MAPPER.readValue(s, Map.class); + } catch (JsonProcessingException e) { + return Map.of("value", s); + } + } +} diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java index d92db7079d8..2fe802e6e52 100644 --- a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java @@ -17,6 +17,7 @@ import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; import static org.projectnessie.catalog.secrets.KeySecret.keySecret; +import static org.projectnessie.catalog.secrets.SecretJsonParser.parseOrSingle; import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; import java.util.Map; @@ -33,6 +34,16 @@ public Secret fromValueMap(Map value) { public Secret fromValueMap(Map value) { return keySecret(value); } + + @Override + public Secret parse(String string) { + return keySecret(string); + } + + @Override + public boolean singleValued() { + return true; + } }, EXPIRING_TOKEN() { @Override @@ -44,4 +55,12 @@ public Secret fromValueMap(Map value) { /** Construct a {@link Secret} instance from its map representation. */ public abstract Secret fromValueMap(Map value); + + public boolean singleValued() { + return false; + } + + public Secret parse(String string) { + return fromValueMap(parseOrSingle(string)); + } } diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretsProvider.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretsProvider.java index f2d6731d535..d18835bfb15 100644 --- a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretsProvider.java +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretsProvider.java @@ -17,11 +17,10 @@ import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; -import java.util.HashSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import org.projectnessie.catalog.secrets.spi.SecretsSupplier; /** @@ -79,7 +78,7 @@ public B applySecrets( @Nullable R specific, @Nonnull List> attributes) { - Set names = new HashSet<>(); + Map toResolve = new HashMap<>(); String basePrefix = baseName + '.'; boolean hasSpecific = specificName != null; @@ -93,35 +92,34 @@ public B applySecrets( specific != null ? attr.current().apply(specific) : Optional.empty(); if (specificSecret.isEmpty()) { if (hasSpecific) { - names.add(specificPrefix + attr.name()); + toResolve.put(specificPrefix + attr.name(), attr.type()); } Optional baseSecret = attr.current().apply(base); if (baseSecret.isEmpty()) { - names.add(basePrefix + attr.name()); + toResolve.put(basePrefix + attr.name(), attr.type()); } } } - if (names.isEmpty()) { + if (toResolve.isEmpty()) { // All secrets present, no need to ask for secrets. return builder; } - Map> secretsMap = secretsSupplier.resolveSecrets(names); + Map secretsMap = secretsSupplier.resolveSecrets(toResolve); for (SecretAttribute attr : casted) { Optional specificSecret = specific != null ? attr.current().apply(specific) : Optional.empty(); if (specificSecret.isEmpty()) { - Map value = - hasSpecific ? secretsMap.get(specificPrefix + attr.name()) : null; + Secret value = hasSpecific ? secretsMap.get(specificPrefix + attr.name()) : null; if (value != null) { - applySecretValue(builder, attr, value); + attr.applicator().accept(builder, value); } else { value = secretsMap.get(basePrefix + attr.name()); if (value != null) { - applySecretValue(builder, attr, value); + attr.applicator().accept(builder, value); } } } @@ -129,13 +127,4 @@ public B applySecrets( return builder; } - - private void applySecretValue( - B builder, SecretAttribute attr, Map value) { - Secret v = attr.type().fromValueMap(value); - if (v == null) { - return; - } - attr.applicator().accept(builder, v); - } } diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SecretsSupplier.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SecretsSupplier.java index c9f3905b998..d4989dcde20 100644 --- a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SecretsSupplier.java +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SecretsSupplier.java @@ -15,21 +15,17 @@ */ package org.projectnessie.catalog.secrets.spi; -import java.util.Collection; import java.util.Map; -import org.projectnessie.catalog.secrets.BasicCredentials; -import org.projectnessie.catalog.secrets.KeySecret; -import org.projectnessie.catalog.secrets.TokenSecret; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; /** SPI interface for actual secrets managers. */ public interface SecretsSupplier { /** * Resolve secrets. * - * @param names names of the secrets to resolve - * @return map of secret names to a map of key-value pairs representing the secret. The keys and - * -values depend on the type of secret. See {@link KeySecret#keySecret(Map)}, {@link - * BasicCredentials#basicCredentials(Map)}, {@link TokenSecret#tokenSecret(Map)} + * @param toResolve names and types of the secrets to resolve + * @return map of secret names to secrets */ - Map> resolveSecrets(Collection names); + Map resolveSecrets(Map toResolve); } diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SingleValueSecretsSupplier.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SingleValueSecretsSupplier.java index 06ad54d62a4..66a874e4fd6 100644 --- a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SingleValueSecretsSupplier.java +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/spi/SingleValueSecretsSupplier.java @@ -15,13 +15,12 @@ */ package org.projectnessie.catalog.secrets.spi; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.Collection; import java.util.Map; import java.util.stream.Collectors; import org.projectnessie.catalog.secrets.BasicCredentials; import org.projectnessie.catalog.secrets.KeySecret; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; import org.projectnessie.catalog.secrets.TokenSecret; /** @@ -29,34 +28,24 @@ * parse each secret from JSON as a map. */ public abstract class SingleValueSecretsSupplier implements SecretsSupplier { - private static final ObjectMapper MAPPER = new ObjectMapper(); @Override - public final Map> resolveSecrets(Collection names) { - Map maps = resolveSingleValueSecrets(names); + public final Map resolveSecrets(Map toResolve) { + Map maps = resolveSingleValueSecrets(toResolve); return maps.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> parseOrSingle(e.getValue()))); - } - - @SuppressWarnings("unchecked") - static Map parseOrSingle(String s) { - if (!s.trim().startsWith("{")) { - return Map.of("value", s); - } - try { - return MAPPER.readValue(s, Map.class); - } catch (JsonProcessingException e) { - return Map.of("value", s); - } + .collect( + Collectors.toMap( + Map.Entry::getKey, e -> toResolve.get(e.getKey()).parse(e.getValue()))); } /** * Resolve secrets. * - * @param names names of the secrets to resolve + * @param toResolve names and types of the secrets to resolve * @return map of secret names to either the JSON representations of the secrets or, if not a JSON * document, the single value for the secret. See {@link KeySecret#keySecret(Map)}, {@link * BasicCredentials#basicCredentials(Map)}, {@link TokenSecret#tokenSecret(Map)} */ - protected abstract Map resolveSingleValueSecrets(Collection names); + protected abstract Map resolveSingleValueSecrets( + Map toResolve); } diff --git a/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretType.java b/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretType.java new file mode 100644 index 00000000000..9d59857a226 --- /dev/null +++ b/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretType.java @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets; + +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +public class TestSecretType { + @InjectSoftAssertions protected SoftAssertions soft; + + @Test + public void keySecret() { + soft.assertThat(SecretType.KEY.parse("foo")) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("foo"); + soft.assertThat(SecretType.KEY.parse("{\"key\": \"foo\"}")) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("{\"key\": \"foo\"}"); + soft.assertThat(SecretType.KEY.parse("{\"value\": \"foo\"}")) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("{\"value\": \"foo\"}"); + + soft.assertThat(SecretType.KEY.fromValueMap(Map.of("key", "foo"))) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("foo"); + soft.assertThat(SecretType.KEY.fromValueMap(Map.of("value", "foo"))) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("foo"); + soft.assertThat(SecretType.KEY.fromValueMap(Map.of("blah", "foo"))).isNull(); + soft.assertThat(SecretType.KEY.fromValueMap(Map.of())).isNull(); + } + + @Test + public void tokenSecret() { + String instantStr = "2024-12-24T12:12:12Z"; + Instant instant = Instant.parse(instantStr); + + soft.assertThat(SecretType.EXPIRING_TOKEN.parse("tok")) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.empty()); + soft.assertThat(SecretType.EXPIRING_TOKEN.parse("{\"value\": \"tok\"}")) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.empty()); + soft.assertThat( + SecretType.EXPIRING_TOKEN.parse( + "{\"value\": \"tok\", \"expiresAt\": \"" + instantStr + "\"}")) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.of(instant)); + soft.assertThat(SecretType.EXPIRING_TOKEN.parse("{\"blah\": \"foo\"}")).isNull(); + soft.assertThat(SecretType.EXPIRING_TOKEN.parse("{}")).isNull(); + + soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("token", "tok"))) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.empty()); + soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("value", "tok"))) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.empty()); + soft.assertThat( + SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("token", "tok", "expiresAt", instantStr))) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.of(instant)); + soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("blah", "foo"))).isNull(); + soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of())).isNull(); + } + + @Test + public void basicCredential() { + soft.assertThat(SecretType.BASIC.parse("tok")).isNull(); + soft.assertThat(SecretType.BASIC.parse("{\"name\": \"user\", \"secret\": \"pass\"}")) + .extracting(BasicCredentials.class::cast) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly("user", "pass"); + soft.assertThat(SecretType.BASIC.parse("{\"blah\": \"foo\"}")).isNull(); + soft.assertThat(SecretType.BASIC.parse("{}")).isNull(); + + soft.assertThat(SecretType.BASIC.fromValueMap(Map.of("name", "user", "secret", "pass"))) + .extracting(BasicCredentials.class::cast) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly("user", "pass"); + soft.assertThat(SecretType.BASIC.fromValueMap(Map.of("name", "user"))).isNull(); + soft.assertThat(SecretType.BASIC.fromValueMap(Map.of("blah", "foo"))).isNull(); + soft.assertThat(SecretType.BASIC.fromValueMap(Map.of())).isNull(); + } +} diff --git a/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretsProvider.java b/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretsProvider.java index 4a7d26322cd..c5c802ecca5 100644 --- a/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretsProvider.java +++ b/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretsProvider.java @@ -20,6 +20,7 @@ import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; import static org.projectnessie.catalog.secrets.KeySecret.keySecret; import static org.projectnessie.catalog.secrets.SecretAttribute.secretAttribute; +import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; import java.time.Instant; import java.util.HashMap; @@ -43,8 +44,7 @@ public class TestSecretsProvider { @ParameterizedTest @MethodSource - public void secretsHelper( - Opts base, Opts spec, Opts expected, Map> secrets) { + public void secretsHelper(Opts base, Opts spec, Opts expected, Map secrets) { List> attributes = List.of( secretAttribute("key", SecretType.KEY, Opts::key, Opts.Builder::key), @@ -78,36 +78,6 @@ public static Stream secretsHelper() { arguments(Opts.EMPTY, null, Opts.EMPTY, Map.of()), arguments(Opts.EMPTY, Opts.EMPTY, Opts.EMPTY, Map.of()), // - // invalid "name" + "token" fields - arguments( - Opts.EMPTY, - Opts.EMPTY, - Opts.EMPTY, - Map.of( - "base.spec.basic", - Map.of("namex", "basic-name", "secret", "basic-secret"), - "base.spec.expiringToken", - Map.of("tokenx", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), - // invalid "expiresAt" + "secret" - arguments( - Opts.EMPTY, - Opts.EMPTY, - Opts.builder().expiringToken(TokenSecret.tokenSecret("exp-token", null)).build(), - Map.of( - "base.spec.basic", - Map.of("name", "basic-name", "secretx", "basic-secret"), - "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "ewpfoijkewoipfjewijfo"))), - // invalid "expiresAt" - arguments( - Opts.EMPTY, - Opts.EMPTY, - Opts.builder().expiringToken(TokenSecret.tokenSecret("exp-token", null)).build(), - Map.of( - "base.spec.basic", - Map.of("name", "basic-name", "secretx", "basic-secret"), - "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "42"))), // all secrets via "spec" from secrets-supplier arguments( Opts.EMPTY, @@ -115,30 +85,30 @@ public static Stream secretsHelper() { Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.key", - Map.of("value", "key"), + keySecret("key"), "base.spec.basic", - Map.of("name", "basic-name", "secret", "basic-secret"), + basicCredentials("basic-name", "basic-secret"), "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), arguments( Opts.EMPTY, Opts.EMPTY, Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.key", - Map.of("value", "key"), + keySecret("key"), "base.basic", - Map.of("name", "basic-name", "secret", "basic-secret"), + basicCredentials("basic-name", "basic-secret"), "base.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), // key present arguments( Opts.EMPTY, @@ -146,26 +116,26 @@ public static Stream secretsHelper() { Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.basic", - Map.of("name", "basic-name", "secret", "basic-secret"), + basicCredentials("basic-name", "basic-secret"), "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), arguments( Opts.builder().key(keySecret("key")).build(), Opts.EMPTY, Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.basic", - Map.of("name", "basic-name", "secret", "basic-secret"), + basicCredentials("basic-name", "basic-secret"), "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), // basic present arguments( Opts.builder().basic(basicCredentials("basic-name", "basic-secret")).build(), @@ -173,99 +143,99 @@ public static Stream secretsHelper() { Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.key", - Map.of("value", "key"), + keySecret("key"), "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), arguments( Opts.EMPTY, Opts.builder().basic(basicCredentials("basic-name", "basic-secret")).build(), Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.key", - Map.of("value", "key"), + keySecret("key"), "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), // expiringToken present arguments( - Opts.builder().expiringToken(TokenSecret.tokenSecret("exp-token", instant)).build(), + Opts.builder().expiringToken(tokenSecret("exp-token", instant)).build(), Opts.EMPTY, Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.key", - Map.of("value", "key"), + keySecret("key"), "base.spec.basic", - Map.of("name", "basic-name", "secret", "basic-secret"))), + basicCredentials("basic-name", "basic-secret"))), arguments( Opts.EMPTY, - Opts.builder().expiringToken(TokenSecret.tokenSecret("exp-token", instant)).build(), + Opts.builder().expiringToken(tokenSecret("exp-token", instant)).build(), Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.key", - Map.of("value", "key"), + keySecret("key"), "base.spec.basic", - Map.of("name", "basic-name", "secret", "basic-secret"))), + basicCredentials("basic-name", "basic-secret"))), // override in "base" arguments( Opts.builder() .key(keySecret("nope")) .basic(basicCredentials("nope", "basic-nope")) - .expiringToken(TokenSecret.tokenSecret("nope", instant)) + .expiringToken(tokenSecret("nope", instant)) .build(), Opts.EMPTY, Opts.builder() .key(keySecret("key")) .basic(basicCredentials("basic-name", "basic-secret")) - .expiringToken(TokenSecret.tokenSecret("exp-token", instant)) + .expiringToken(tokenSecret("exp-token", instant)) .build(), Map.of( "base.spec.key", - Map.of("value", "key"), + keySecret("key"), "base.spec.basic", - Map.of("name", "basic-name", "secret", "basic-secret"), + basicCredentials("basic-name", "basic-secret"), "base.spec.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), // NO override in "base" arguments( Opts.builder() .key(keySecret("nope")) .basic(basicCredentials("nope", "basic-nope")) - .expiringToken(TokenSecret.tokenSecret("nope", instant)) + .expiringToken(tokenSecret("nope", instant)) .build(), Opts.EMPTY, Opts.builder() .key(keySecret("nope")) .basic(basicCredentials("nope", "basic-nope")) - .expiringToken(TokenSecret.tokenSecret("nope", instant)) + .expiringToken(tokenSecret("nope", instant)) .build(), Map.of( "base.key", - Map.of("value", "key"), + keySecret("key"), "base.basic", - Map.of("name", "basic-name", "secret", "basic-secret"), + basicCredentials("basic-name", "basic-secret"), "base.expiringToken", - Map.of("token", "exp-token", "expiresAt", "2024-12-24T12:12:12Z"))), + tokenSecret("exp-token", instant))), // arguments( Opts.EMPTY, Opts.EMPTY, - Opts.builder().expiringToken(TokenSecret.tokenSecret("exp-token", null)).build(), - Map.of("base.spec.expiringToken", Map.of("token", "exp-token"))) + Opts.builder().expiringToken(tokenSecret("exp-token", null)).build(), + Map.of("base.spec.expiringToken", tokenSecret("exp-token", null))) // ); } @@ -297,11 +267,11 @@ interface Builder { } } - static SecretsSupplier mapSecretsSupplier(Map> secretsMap) { - return names -> { - Map> resolved = new HashMap<>(); - for (String name : names) { - Map r = secretsMap.get(name); + static SecretsSupplier mapSecretsSupplier(Map secretsMap) { + return toResolve -> { + Map resolved = new HashMap<>(); + for (String name : toResolve.keySet()) { + Secret r = secretsMap.get(name); if (r != null) { resolved.put(name, r); } diff --git a/catalog/secrets/api/src/testFixtures/java/org/projectnessie/catalog/secrets/spi/DummySecretsSupplier.java b/catalog/secrets/api/src/testFixtures/java/org/projectnessie/catalog/secrets/spi/DummySecretsSupplier.java new file mode 100644 index 00000000000..2eab26f52a1 --- /dev/null +++ b/catalog/secrets/api/src/testFixtures/java/org/projectnessie/catalog/secrets/spi/DummySecretsSupplier.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.spi; + +import java.util.Map; +import java.util.stream.Collectors; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; + +public class DummySecretsSupplier implements SecretsSupplier { + @Override + public Map resolveSecrets(Map toResolve) { + return toResolve.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> + e.getValue() + .fromValueMap(Map.of("value", "key", "name", "name", "secret", "secret")))); + } +} diff --git a/catalog/secrets/aws/build.gradle.kts b/catalog/secrets/aws/build.gradle.kts new file mode 100644 index 00000000000..dd2a1b7e5d4 --- /dev/null +++ b/catalog/secrets/aws/build.gradle.kts @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets AWS" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:apache-client") { + exclude("commons-logging", "commons-logging") + } + implementation("software.amazon.awssdk:secretsmanager") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) + + intTestCompileOnly(project(":nessie-immutables")) + intTestImplementation(platform(libs.testcontainers.bom)) + intTestImplementation("org.testcontainers:testcontainers") + intTestImplementation("org.testcontainers:localstack") + intTestImplementation("org.testcontainers:junit-jupiter") + intTestImplementation(project(":nessie-container-spec-helper")) + intTestRuntimeOnly(libs.logback.classic) +} diff --git a/catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsSupplier.java b/catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsSupplier.java new file mode 100644 index 00000000000..6c9d63c7c43 --- /dev/null +++ b/catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsSupplier.java @@ -0,0 +1,179 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.aws; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.type; +import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; +import static org.projectnessie.catalog.secrets.KeySecret.keySecret; +import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; +import static org.testcontainers.containers.localstack.LocalStackContainer.Service.SECRETSMANAGER; + +import java.net.URI; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.projectnessie.catalog.secrets.BasicCredentials; +import org.projectnessie.catalog.secrets.KeySecret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.TokenSecret; +import org.projectnessie.nessie.testing.containerspec.ContainerSpecHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.CreateSecretRequest; + +@Testcontainers +@ExtendWith(SoftAssertionsExtension.class) +public class ITAwsSecretsSupplier { + private static final Logger LOGGER = LoggerFactory.getLogger(ITAwsSecretsSupplier.class); + + @InjectSoftAssertions SoftAssertions soft; + + @Container + static LocalStackContainer localstack = + new LocalStackContainer( + ContainerSpecHelper.builder() + .name("localstack") + .containerClass(ITAwsSecretsSupplier.class) + .build() + .dockerImageName(null) + .asCompatibleSubstituteFor("localstack/localstack")) + .withLogConsumer(c -> LOGGER.info("[LOCALSTACK] {}", c.getUtf8StringWithoutLineEnding())) + .withServices(SECRETSMANAGER); + + @Test + public void awsSecretsManager() { + URI secretsManagerEndpoint = localstack.getEndpointOverride(SECRETSMANAGER); + try (SecretsManagerClient client = + SecretsManagerClient.builder() + .endpointOverride(secretsManagerEndpoint) + .region(Region.of(localstack.getRegion())) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create( + localstack.getAccessKey(), localstack.getSecretKey()))) + .build()) { + + String instantStr = "2024-06-05T20:38:16Z"; + Instant instant = Instant.parse(instantStr); + + KeySecret keySecret = keySecret("secret-foo"); + BasicCredentials basicCred = basicCredentials("bar-name", "bar-secret"); + TokenSecret tokenSec = tokenSecret("the-token", instant); + + client.createSecret( + CreateSecretRequest.builder().name("key").secretString("secret-foo").build()); + client.createSecret( + CreateSecretRequest.builder() + .name("basic") + .secretString("{\"name\": \"bar-name\", \"secret\": \"bar-secret\"}") + .build()); + client.createSecret( + CreateSecretRequest.builder() + .name("tok") + .secretString("{\"token\": \"the-token\", \"expiresAt\": \"" + instantStr + "\"}") + .build()); + + AwsSecretsSupplier secretsSupplier = + new AwsSecretsSupplier(client, "", Duration.ofMinutes(1)); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of( + "key", + SecretType.KEY, + "basic", + SecretType.BASIC, + "tok", + SecretType.EXPIRING_TOKEN))) + .hasSize(3) + .hasEntrySatisfying( + "key", + s -> + assertThat(s) + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key())) + .hasEntrySatisfying( + "basic", + s -> + assertThat(s) + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret())) + .hasEntrySatisfying( + "tok", + s -> + assertThat(s) + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt())); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of( + "key", + SecretType.KEY, + "nope", + SecretType.BASIC, + "basic", + SecretType.BASIC, + "not-there", + SecretType.KEY, + "tok", + SecretType.EXPIRING_TOKEN))) + .hasSize(3) + .hasEntrySatisfying( + "key", + s -> + assertThat(s) + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key())) + .hasEntrySatisfying( + "basic", + s -> + assertThat(s) + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret())) + .hasEntrySatisfying( + "tok", + s -> + assertThat(s) + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt())); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of("nope", SecretType.BASIC, "not-there", SecretType.KEY))) + .isEmpty(); + } + } +} diff --git a/catalog/secrets/aws/src/intTest/resources/logback-test.xml b/catalog/secrets/aws/src/intTest/resources/logback-test.xml new file mode 100644 index 00000000000..ce140f00774 --- /dev/null +++ b/catalog/secrets/aws/src/intTest/resources/logback-test.xml @@ -0,0 +1,30 @@ + + + + + + + %date{ISO8601} [%thread] [%X{nessie.events.subscription.id}] %-5level %logger{36} - + %msg%n + + + + + + diff --git a/catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version b/catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version new file mode 100644 index 00000000000..5e5015a43ea --- /dev/null +++ b/catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version @@ -0,0 +1,3 @@ +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM docker.io/localstack/localstack:3.5.0 diff --git a/catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsSupplier.java b/catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsSupplier.java new file mode 100644 index 00000000000..7988aa192f1 --- /dev/null +++ b/catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsSupplier.java @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.aws; + +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.spi.SingleValueSecretsSupplier; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.BatchGetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.SecretValueEntry; + +public class AwsSecretsSupplier extends SingleValueSecretsSupplier { + private final SecretsManagerClient secretsManagerClient; + private final String prefix; + + public AwsSecretsSupplier( + SecretsManagerClient secretsManagerClient, String prefix, Duration getSecretTimeout) { + this.secretsManagerClient = secretsManagerClient; + this.prefix = prefix; + } + + @Override + protected Map resolveSingleValueSecrets(Map toResolve) { + if (toResolve.isEmpty()) { + return Map.of(); + } + + Map secretIdToNameMap = new HashMap<>(); + + for (String name : toResolve.keySet()) { + String secretId = nameToSecretId(name); + secretIdToNameMap.put(secretId, name); + } + + return secretsManagerClient + .batchGetSecretValue( + BatchGetSecretValueRequest.builder().secretIdList(secretIdToNameMap.keySet()).build()) + .secretValues() + .stream() + .collect( + Collectors.toMap( + sv -> secretIdToNameMap.get(sv.name()), SecretValueEntry::secretString)); + } + + private String nameToSecretId(String name) { + return prefix + name; + } +} diff --git a/catalog/secrets/azure/build.gradle.kts b/catalog/secrets/azure/build.gradle.kts new file mode 100644 index 00000000000..0d726850bf7 --- /dev/null +++ b/catalog/secrets/azure/build.gradle.kts @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets Azure" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(enforcedPlatform(libs.quarkus.azure.services.bom)) + implementation("com.azure:azure-security-keyvault-secrets") + implementation("com.azure:azure-identity") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) + + intTestCompileOnly(project(":nessie-immutables")) + intTestImplementation(platform(libs.testcontainers.bom)) + intTestImplementation("org.testcontainers:testcontainers") + intTestImplementation("org.testcontainers:junit-jupiter") + intTestImplementation(libs.lowkey.vault.testcontainers) + intTestImplementation(project(":nessie-container-spec-helper")) + intTestRuntimeOnly(libs.logback.classic) +} + +tasks.named("intTest") { systemProperty("javax.net.ssl.trustStore", "foo.bar.BazTrustStore") } diff --git a/catalog/secrets/azure/src/intTest/java/org/projectnessie/catalog/secrets/azure/ITAzureSecretsSupplier.java b/catalog/secrets/azure/src/intTest/java/org/projectnessie/catalog/secrets/azure/ITAzureSecretsSupplier.java new file mode 100644 index 00000000000..f67d83e9983 --- /dev/null +++ b/catalog/secrets/azure/src/intTest/java/org/projectnessie/catalog/secrets/azure/ITAzureSecretsSupplier.java @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.azure; + +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.type; +import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; +import static org.projectnessie.catalog.secrets.KeySecret.keySecret; +import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; + +import com.azure.identity.UsernamePasswordCredentialBuilder; +import com.azure.security.keyvault.secrets.SecretAsyncClient; +import com.azure.security.keyvault.secrets.SecretClientBuilder; +import com.github.nagyesta.lowkeyvault.testcontainers.LowkeyVaultContainer; +import com.github.nagyesta.lowkeyvault.testcontainers.LowkeyVaultContainerBuilder; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Map; +import java.util.Set; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.projectnessie.catalog.secrets.BasicCredentials; +import org.projectnessie.catalog.secrets.KeySecret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.TokenSecret; +import org.projectnessie.nessie.testing.containerspec.ContainerSpecHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +@ExtendWith(SoftAssertionsExtension.class) +@Disabled( + "Azure SecretClient requires an SSL connection, verifying the server certificates, which needs to used by the test container and trusted by the client. No way around it.") +public class ITAzureSecretsSupplier { + private static final Logger LOGGER = LoggerFactory.getLogger(ITAzureSecretsSupplier.class); + + @InjectSoftAssertions SoftAssertions soft; + + @Container + static LowkeyVaultContainer lowkeyVault = + LowkeyVaultContainerBuilder.lowkeyVault( + ContainerSpecHelper.builder() + .name("lowkey-vault") + .containerClass(ITAzureSecretsSupplier.class) + .build() + .dockerImageName(null) + .asCompatibleSubstituteFor("nagyesta/lowkey-vault")) + .vaultNames(Set.of("default")) + .build() + .withLogConsumer( + c -> LOGGER.info("[LOWKEY-VAULT] {}", c.getUtf8StringWithoutLineEnding())); + + @Test + public void azureSecrets() { + SecretAsyncClient client = + new SecretClientBuilder() + .vaultUrl(format("https://%s.localhost:%d", "default", lowkeyVault.getMappedPort(8443))) + .credential( + new UsernamePasswordCredentialBuilder() + .clientId("ITAzureSecretsSupplier") + .username(lowkeyVault.getUsername()) + .password(lowkeyVault.getPassword()) + .build()) + .buildAsyncClient(); + + String instantStr = "2024-06-05T20:38:16Z"; + Instant instant = Instant.parse(instantStr); + + KeySecret keySecret = keySecret("secret-foo"); + BasicCredentials basicCred = basicCredentials("bar-name", "bar-secret"); + TokenSecret tokenSec = tokenSecret("the-token", instant); + + client.setSecret("key", "secret-foo").block(Duration.of(1, ChronoUnit.MINUTES)); + client + .setSecret("basic", "{\"name\": \"bar-name\", \"secret\": \"bar-secret\"}") + .block(Duration.of(1, ChronoUnit.MINUTES)); + client + .setSecret("tok", "{\"token\": \"the-token\", \"expiresAt\": \"" + instantStr + "\"}") + .block(Duration.of(1, ChronoUnit.MINUTES)); + + AzureSecretsSupplier secretsSupplier = + new AzureSecretsSupplier(client, "", Duration.ofMinutes(1)); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of( + "key", + SecretType.KEY, + "basic", + SecretType.BASIC, + "tok", + SecretType.EXPIRING_TOKEN))) + .hasSize(3) + .hasEntrySatisfying( + "key", + s -> + assertThat(s) + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key())) + .hasEntrySatisfying( + "basic", + s -> + assertThat(s) + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret())) + .hasEntrySatisfying( + "tok", + s -> + assertThat(s) + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt())); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of( + "key", + SecretType.KEY, + "nope", + SecretType.BASIC, + "basic", + SecretType.BASIC, + "not-there", + SecretType.KEY, + "tok", + SecretType.EXPIRING_TOKEN))) + .hasSize(3) + .hasEntrySatisfying( + "key", + s -> + assertThat(s) + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key())) + .hasEntrySatisfying( + "basic", + s -> + assertThat(s) + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret())) + .hasEntrySatisfying( + "tok", + s -> + assertThat(s) + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt())); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of("nope", SecretType.BASIC, "not-there", SecretType.KEY))) + .isEmpty(); + } +} diff --git a/catalog/secrets/azure/src/intTest/resources/logback-test.xml b/catalog/secrets/azure/src/intTest/resources/logback-test.xml new file mode 100644 index 00000000000..ce140f00774 --- /dev/null +++ b/catalog/secrets/azure/src/intTest/resources/logback-test.xml @@ -0,0 +1,30 @@ + + + + + + + %date{ISO8601} [%thread] [%X{nessie.events.subscription.id}] %-5level %logger{36} - + %msg%n + + + + + + diff --git a/catalog/secrets/azure/src/intTest/resources/org/projectnessie/catalog/secrets/azure/Dockerfile-lowkey-vault-version b/catalog/secrets/azure/src/intTest/resources/org/projectnessie/catalog/secrets/azure/Dockerfile-lowkey-vault-version new file mode 100644 index 00000000000..8907a241ad1 --- /dev/null +++ b/catalog/secrets/azure/src/intTest/resources/org/projectnessie/catalog/secrets/azure/Dockerfile-lowkey-vault-version @@ -0,0 +1,3 @@ +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM docker.io/nagyesta/lowkey-vault:2.4.42 diff --git a/catalog/secrets/azure/src/main/java/org/projectnessie/catalog/secrets/azure/AzureSecretsSupplier.java b/catalog/secrets/azure/src/main/java/org/projectnessie/catalog/secrets/azure/AzureSecretsSupplier.java new file mode 100644 index 00000000000..b357ce2d000 --- /dev/null +++ b/catalog/secrets/azure/src/main/java/org/projectnessie/catalog/secrets/azure/AzureSecretsSupplier.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.azure; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import com.azure.core.exception.ResourceNotFoundException; +import com.azure.security.keyvault.secrets.SecretAsyncClient; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.spi.SingleValueSecretsSupplier; + +public class AzureSecretsSupplier extends SingleValueSecretsSupplier { + private final SecretAsyncClient client; + private final String prefix; + private final long timeout; + + public AzureSecretsSupplier(SecretAsyncClient client, String prefix, Duration getSecretTimeout) { + this.client = client; + this.prefix = prefix; + this.timeout = getSecretTimeout.toMillis(); + } + + @Override + protected Map resolveSingleValueSecrets(Map toResolve) { + if (toResolve.isEmpty()) { + return Map.of(); + } + + List>> futures = new ArrayList<>(); + + for (String name : toResolve.keySet()) { + String secretId = nameToSecretId(name); + futures.add( + client + .getSecret(secretId) + .map(s -> Map.entry(name, s.getValue())) + .toFuture() + .exceptionally( + t -> { + if (t instanceof ResourceNotFoundException) { + return null; + } + if (t instanceof RuntimeException) { + throw (RuntimeException) t; + } + throw new RuntimeException(t); + })); + } + + try { + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).get(timeout, MILLISECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new RuntimeException(e); + } + + return futures.stream() + .map(f -> f.getNow(null)) + .filter(Objects::nonNull) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private String nameToSecretId(String name) { + return prefix + name; + } +} diff --git a/catalog/secrets/cache/build.gradle.kts b/catalog/secrets/cache/build.gradle.kts new file mode 100644 index 00000000000..ae138fba28f --- /dev/null +++ b/catalog/secrets/cache/build.gradle.kts @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets Cache" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + implementation(libs.caffeine) + implementation(libs.micrometer.core) + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-databind") + implementation("com.fasterxml.jackson.core:jackson-annotations") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) +} diff --git a/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java new file mode 100644 index 00000000000..ac1053ee455 --- /dev/null +++ b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import org.projectnessie.catalog.secrets.spi.SecretsSupplier; + +public final class CachingSecrets { + + private final CachingSecretsBackend backend; + + public CachingSecrets(CachingSecretsBackend backend) { + this.backend = backend; + } + + public SecretsSupplier forRepository(String repositoryId, SecretsSupplier secretsSupplier) { + return toResolve -> backend.resolveSecrets(repositoryId, toResolve, secretsSupplier); + } +} diff --git a/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecretsBackend.java b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecretsBackend.java new file mode 100644 index 00000000000..7620ee33d57 --- /dev/null +++ b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecretsBackend.java @@ -0,0 +1,241 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import static java.util.Collections.singletonList; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Expiry; +import com.google.common.annotations.VisibleForTesting; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.binder.cache.CaffeineStatsCounter; +import java.util.HashMap; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.LongSupplier; +import java.util.stream.Collectors; +import org.checkerframework.checker.index.qual.NonNegative; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.spi.SecretsSupplier; + +public class CachingSecretsBackend { + + private static final Secret CACHE_NEGATIVE_SENTINEL = new Secret() {}; + + public static final String CACHE_NAME = "nessie-secrets"; + private static final long NOT_CACHED = 0L; + + @VisibleForTesting final Cache cache; + private final long ttlNanos; + private final LongSupplier clock; + + public CachingSecretsBackend(SecretsCacheConfig config) { + OptionalLong ttl = config.ttlMillis(); + this.ttlNanos = ttl.isPresent() ? TimeUnit.MILLISECONDS.toNanos(ttl.getAsLong()) : 0L; + this.clock = config.clockNanos(); + + Caffeine cacheBuilder = + Caffeine.newBuilder() + .expireAfter( + new Expiry() { + @Override + public long expireAfterCreate( + CacheKeyValue key, Secret value, long currentTimeNanos) { + long expire = key.expiresAtNanosEpoch; + if (expire == NOT_CACHED) { + return 0L; + } + long remaining = expire - currentTimeNanos; + return Math.max(0L, remaining); + } + + @Override + public long expireAfterUpdate( + CacheKeyValue key, + Secret value, + long currentTimeNanos, + @NonNegative long currentDurationNanos) { + return expireAfterCreate(key, value, currentTimeNanos); + } + + @Override + public long expireAfterRead( + CacheKeyValue key, + Secret value, + long currentTimeNanos, + @NonNegative long currentDurationNanos) { + return currentDurationNanos; + } + }) + .ticker(clock::getAsLong) + .maximumSize(config.maxElements()); + config + .meterRegistry() + .ifPresent( + meterRegistry -> { + cacheBuilder.recordStats(() -> new CaffeineStatsCounter(meterRegistry, CACHE_NAME)); + meterRegistry.gauge( + "cache_max_size", + singletonList(Tag.of("cache", CACHE_NAME)), + "", + x -> config.maxElements()); + meterRegistry.gauge( + "cache_element_ttl", + singletonList(Tag.of("cache", CACHE_NAME)), + "", + x -> config.ttlMillis().orElse(0)); + }); + + this.cache = cacheBuilder.build(); + } + + private static boolean isPresentValue(Secret value) { + // Yes, same instance for + return value != null && value != CACHE_NEGATIVE_SENTINEL; + } + + Map resolveSecrets( + String repositoryId, Map toResolve, SecretsSupplier backend) { + long ttl = ttlNanos; + long expires = ttl != 0L ? clock.getAsLong() + ttl : 0L; + + Map keysToResolve = new HashMap<>(); + for (String name : toResolve.keySet()) { + keysToResolve.put(name, new CacheKeyValue(repositoryId, name, expires)); + } + + Map result = new HashMap<>(); + + Map present = cache.getAllPresent(keysToResolve.values()); + present.forEach( + (k, v) -> { + String name = k.name; + keysToResolve.remove(name); + if (isPresentValue(v)) { + result.put(name, v); + } + }); + + Map resolve = + keysToResolve.keySet().stream() + .collect(Collectors.toMap(Function.identity(), toResolve::get)); + + if (!keysToResolve.isEmpty()) { + Map resolved = backend.resolveSecrets(resolve); + + Map put = new HashMap<>(); + for (CacheKeyValue key : keysToResolve.values()) { + Secret value = resolved.getOrDefault(key.name, CACHE_NEGATIVE_SENTINEL); + if (isPresentValue(value)) { + result.put(key.name, value); + } + put.put(key, value); + } + cache.putAll(put); + } + + return result; + } + + private static String safeStringValue(Map.Entry e) { + // The string doesn't necessarily need to be a string, for example if the secrets backend + // returns another object (Vault could do that?). + return e.getValue().toString(); + } + + static final class CacheKeyValue { + final String repositoryId; + final String name; + + // Revisit this field before 2262-04-11T23:47:16.854Z (64-bit signed long overflow) ;) ;) + final long expiresAtNanosEpoch; + + CacheKeyValue(String repositoryId, String name, long expiresAtNanosEpoch) { + this.repositoryId = repositoryId; + this.name = name; + this.expiresAtNanosEpoch = expiresAtNanosEpoch; + } + + int heapSize() { + int size = OBJ_SIZE; + size += STRING_OBJ_OVERHEAD + repositoryId.length(); + size += STRING_OBJ_OVERHEAD + name.length(); + return size; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CacheKeyValue)) { + return false; + } + CacheKeyValue cacheKey = (CacheKeyValue) o; + return repositoryId.equals(cacheKey.repositoryId) && name.equals(cacheKey.name); + } + + @Override + public int hashCode() { + return repositoryId.hashCode() * 31 + name.hashCode(); + } + + @Override + public String toString() { + return "{" + repositoryId + ", " + name + '}'; + } + } + + /* + CacheKeyValue object internals: + OFF SZ TYPE DESCRIPTION VALUE + 0 8 (object header: mark) 0x0000000000000001 (non-biasable; age: 0) + 8 4 (object header: class) 0x010c4800 + 12 4 java.lang.String CacheKeyValue.repositoryId null + 16 8 long CacheKeyValue.expiresAt 0 + 24 4 java.lang.String CacheKeyValue.id null + 28 4 (object alignment gap) + Instance size: 32 bytes + */ + static final int OBJ_SIZE = 32; + // rough estimate, probably good enough + static final int MAP_OBJ_OVERHEAD = OBJ_SIZE * 2; + /* + Array overhead: 16 bytes + */ + static final int ARRAY_OVERHEAD = 16; + /* + java.lang.String object internals: + OFF SZ TYPE DESCRIPTION VALUE + 0 8 (object header: mark) 0x0000000000000001 (non-biasable; age: 0) + 8 4 (object header: class) 0x0000e8d8 + 12 4 int String.hash 0 + 16 1 byte String.coder 0 + 17 1 boolean String.hashIsZero false + 18 2 (alignment/padding gap) + 20 4 byte[] String.value [] + Instance size: 24 bytes + */ + static final int STRING_OBJ_OVERHEAD = 24 + ARRAY_OVERHEAD; + /* + Assume an overhead of 2 objects for each entry (java.util.concurrent.ConcurrentHashMap$Node is 32 bytes) in Caffeine. + */ + static final int CAFFEINE_OBJ_OVERHEAD = 2 * 32; +} diff --git a/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/SecretsCacheConfig.java b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/SecretsCacheConfig.java new file mode 100644 index 00000000000..e004e9b7655 --- /dev/null +++ b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/SecretsCacheConfig.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import io.micrometer.core.instrument.MeterRegistry; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.function.LongSupplier; +import org.immutables.value.Value; + +@Value.Immutable +public interface SecretsCacheConfig { + + long maxElements(); + + OptionalLong ttlMillis(); + + Optional meterRegistry(); + + @Value.Default + default LongSupplier clockNanos() { + return System::nanoTime; + } + + static Builder builder() { + return ImmutableSecretsCacheConfig.builder(); + } + + interface Builder { + @CanIgnoreReturnValue + Builder maxElements(long maxElements); + + @CanIgnoreReturnValue + Builder ttlMillis(long ttlMillis); + + @CanIgnoreReturnValue + Builder meterRegistry(MeterRegistry meterRegistry); + + @CanIgnoreReturnValue + Builder clockNanos(LongSupplier clockNanos); + + SecretsCacheConfig build(); + } +} diff --git a/catalog/secrets/cache/src/test/java/org/projectnessie/catalog/secrets/cache/TestCachingSecretsBackend.java b/catalog/secrets/cache/src/test/java/org/projectnessie/catalog/secrets/cache/TestCachingSecretsBackend.java new file mode 100644 index 00000000000..727cfee753f --- /dev/null +++ b/catalog/secrets/cache/src/test/java/org/projectnessie/catalog/secrets/cache/TestCachingSecretsBackend.java @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import static java.util.function.Function.identity; +import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; +import static org.projectnessie.catalog.secrets.KeySecret.keySecret; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.spi.SecretsSupplier; + +@ExtendWith(SoftAssertionsExtension.class) +public class TestCachingSecretsBackend { + @InjectSoftAssertions protected SoftAssertions soft; + + static final Secret SECRET_1_REPO_1 = keySecret("secret1-repo1"); + static final Secret SECRET_2_REPO_1 = basicCredentials("secret1-name", "secret1-value"); + static final Secret SECRET_1_REPO_2 = keySecret("secret1-repo2"); + static final Secret SECRET_2_REPO_2 = basicCredentials("secret2-name", "secret2-value"); + static final Map SECRETS_1 = Map.of("secret1", SecretType.KEY); + static final Map SECRETS_1_2 = + Map.of("secret1", SecretType.KEY, "secret2", SecretType.KEY); + + AtomicLong clock; + CachingSecretsBackend backend; + CachingSecrets cachingSecrets; + SecretsSupplier secrets1; + SecretsSupplier secrets2; + SecretsSupplier caching1; + SecretsSupplier caching2; + + @BeforeEach + void setup() { + clock = new AtomicLong(); + backend = + new CachingSecretsBackend( + SecretsCacheConfig.builder() + .maxElements(100) + .ttlMillis(1000) + .clockNanos(clock::get) + .build()); + + cachingSecrets = new CachingSecrets(backend); + + secrets1 = mapSecretsSupplier(Map.of("secret1", SECRET_1_REPO_1, "secret2", SECRET_2_REPO_1)); + secrets2 = mapSecretsSupplier(Map.of("secret1", SECRET_1_REPO_2, "secret2", SECRET_2_REPO_2)); + + caching1 = cachingSecrets.forRepository("repo1", secrets1); + caching2 = cachingSecrets.forRepository("repo2", secrets2); + } + + @Test + public void ttlExpire() { + soft.assertThat(caching1.resolveSecrets(SECRETS_1)) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_1)); + soft.assertThat(caching2.resolveSecrets(SECRETS_1)) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_2)); + soft.assertThat(caching1.resolveSecrets(SECRETS_1)) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_1)); + soft.assertThat(caching2.resolveSecrets(SECRETS_1)) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_2)); + + soft.assertThat(backend.cache.asMap()) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret1", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret1", 0)); + + clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(1000)); + + soft.assertThat(backend.cache.asMap()) + .doesNotContainKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret1", 0)) + .doesNotContainKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret1", 0)); + + soft.assertThat(caching1.resolveSecrets(SECRETS_1_2)) + .containsExactlyInAnyOrderEntriesOf( + Map.of("secret1", SECRET_1_REPO_1, "secret2", SECRET_2_REPO_1)); + soft.assertThat(caching2.resolveSecrets(SECRETS_1_2)) + .containsExactlyInAnyOrderEntriesOf( + Map.of("secret1", SECRET_1_REPO_2, "secret2", SECRET_2_REPO_2)); + + soft.assertThat(backend.cache.asMap()) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret1", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret1", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret2", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret2", 0)); + } + + @Test + public void nonExisting() { + soft.assertThat( + caching1.resolveSecrets( + Map.of("secret-nope1", SecretType.KEY, "secret-nope2", SecretType.KEY))) + .isEmpty(); + soft.assertThat( + caching2.resolveSecrets( + Map.of("secret-nope1", SecretType.KEY, "secret-nope2", SecretType.KEY))) + .isEmpty(); + soft.assertThat( + caching1.resolveSecrets( + Map.of( + "secret-nope1", + SecretType.KEY, + "secret-nope2", + SecretType.KEY, + "secret1", + SecretType.KEY))) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_1)); + soft.assertThat( + caching2.resolveSecrets( + Map.of( + "secret-nope1", + SecretType.KEY, + "secret-nope2", + SecretType.KEY, + "secret1", + SecretType.KEY))) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_2)); + } + + @Test + public void tooManyEntries() { + SecretsSupplier supplier = + toResolve -> + toResolve.keySet().stream() + .collect(Collectors.toMap(identity(), name -> keySecret(name))); + SecretsSupplier caching = cachingSecrets.forRepository("repoMany", supplier); + + // Very naive test that checks that expiration happens - our "weigher" does something. + + for (int i = 0; i < 100_000; i++) { + caching.resolveSecrets( + Map.of( + "wepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoew-" + + i, + SecretType.KEY)); + // In fact, the cache should never have more than 10000 entries, but 'estimatedSize' is really + // just an estimate. + soft.assertThat(backend.cache.estimatedSize()).isLessThan(10_000); + } + } + + static SecretsSupplier mapSecretsSupplier(Map secretsMap) { + return toResolve -> { + Map resolved = new HashMap<>(); + for (String name : toResolve.keySet()) { + Secret r = secretsMap.get(name); + if (r != null) { + resolved.put(name, r); + } + } + return resolved; + }; + } +} diff --git a/catalog/secrets/gcs/build.gradle.kts b/catalog/secrets/gcs/build.gradle.kts new file mode 100644 index 00000000000..2641bced3a6 --- /dev/null +++ b/catalog/secrets/gcs/build.gradle.kts @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets GCS" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(platform(libs.google.cloud.secretmanager.bom)) + implementation("com.google.cloud:google-cloud-secretmanager") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) +} diff --git a/catalog/secrets/gcs/src/main/java/org/projectnessie/catalog/secrets/gcs/GcsSecretsSupplier.java b/catalog/secrets/gcs/src/main/java/org/projectnessie/catalog/secrets/gcs/GcsSecretsSupplier.java new file mode 100644 index 00000000000..dd03d4cfa7b --- /dev/null +++ b/catalog/secrets/gcs/src/main/java/org/projectnessie/catalog/secrets/gcs/GcsSecretsSupplier.java @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.gcs; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.cloud.secretmanager.v1.AccessSecretVersionRequest; +import com.google.cloud.secretmanager.v1.AccessSecretVersionResponse; +import com.google.cloud.secretmanager.v1.SecretManagerServiceClient; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.spi.SingleValueSecretsSupplier; + +public class GcsSecretsSupplier extends SingleValueSecretsSupplier { + private final String prefix; + private final SecretManagerServiceClient client; + private final long getSecretTimeout; + + public GcsSecretsSupplier( + SecretManagerServiceClient client, String prefix, Duration getSecretTimeout) { + this.client = client; + this.prefix = prefix; + this.getSecretTimeout = getSecretTimeout.toMillis(); + } + + @Override + protected Map resolveSingleValueSecrets(Map toResolve) { + if (toResolve.isEmpty()) { + return Map.of(); + } + + Set toResolveKeys = toResolve.keySet(); + + if (toResolve.size() == 1) { + String name = toResolveKeys.iterator().next(); + String secretId = nameToSecretId(name); + AccessSecretVersionResponse response = client.accessSecretVersion(request(secretId)); + return response.hasPayload() ? Map.of(name, secret(response)) : Map.of(); + } + + Map secretIdToNameMap = new HashMap<>(); + + for (String name : toResolveKeys) { + String secretId = nameToSecretId(name); + secretIdToNameMap.put(secretId, name); + } + + List> futures = + secretIdToNameMap.keySet().stream() + .map(id -> client.accessSecretVersionCallable().futureCall(request(id))) + .collect(Collectors.toList()); + + List results; + try { + results = ApiFutures.successfulAsList(futures).get(getSecretTimeout, MILLISECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new RuntimeException(e); + } + + return results.stream() + .collect(Collectors.toMap(resp -> secretIdToNameMap.get(resp.getName()), this::secret)); + } + + private String secret(AccessSecretVersionResponse response) { + return response.getPayload().getData().toStringUtf8(); + } + + private AccessSecretVersionRequest request(String name) { + return AccessSecretVersionRequest.newBuilder().setName(name).build(); + } + + private String nameToSecretId(String name) { + return prefix + name; + } +} diff --git a/catalog/secrets/vault/build.gradle.kts b/catalog/secrets/vault/build.gradle.kts new file mode 100644 index 00000000000..69048c5422a --- /dev/null +++ b/catalog/secrets/vault/build.gradle.kts @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets Vault" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(enforcedPlatform(libs.quarkus.bom)) + implementation("io.quarkiverse.vault:quarkus-vault") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) + + intTestImplementation(platform(libs.testcontainers.bom)) + intTestImplementation("org.testcontainers:testcontainers") + intTestImplementation("org.testcontainers:vault") + intTestImplementation("org.testcontainers:junit-jupiter") + intTestImplementation(project(":nessie-container-spec-helper")) + intTestImplementation(libs.smallrye.config.core) + intTestCompileOnly(libs.immutables.value.annotations) + intTestRuntimeOnly(libs.logback.classic) +} diff --git a/catalog/secrets/vault/src/intTest/java/org/projectnessie/catalog/secrets/vault/ITVaultSecretsSupplier.java b/catalog/secrets/vault/src/intTest/java/org/projectnessie/catalog/secrets/vault/ITVaultSecretsSupplier.java new file mode 100644 index 00000000000..be1e5393ea9 --- /dev/null +++ b/catalog/secrets/vault/src/intTest/java/org/projectnessie/catalog/secrets/vault/ITVaultSecretsSupplier.java @@ -0,0 +1,201 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.vault; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.type; +import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; +import static org.projectnessie.catalog.secrets.KeySecret.keySecret; +import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; + +import io.quarkus.vault.VaultKVSecretReactiveEngine; +import io.quarkus.vault.client.VaultClient; +import io.quarkus.vault.runtime.VaultConfigHolder; +import io.quarkus.vault.runtime.VaultKvManager; +import io.quarkus.vault.runtime.client.VaultClientProducer; +import io.quarkus.vault.runtime.config.VaultRuntimeConfig; +import io.smallrye.config.PropertiesConfigSource; +import io.smallrye.config.SmallRyeConfig; +import io.smallrye.config.SmallRyeConfigBuilder; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.projectnessie.catalog.secrets.BasicCredentials; +import org.projectnessie.catalog.secrets.KeySecret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.TokenSecret; +import org.projectnessie.nessie.testing.containerspec.ContainerSpecHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.vault.VaultContainer; + +@Testcontainers +@ExtendWith(SoftAssertionsExtension.class) +public class ITVaultSecretsSupplier { + private static final Logger LOGGER = LoggerFactory.getLogger(ITVaultSecretsSupplier.class); + + public static final String VAULT_ROOT_TOKEN = "root"; + public static final String NESSIE_SECRETS_PATH = "apps/nessie/secrets"; + + @InjectSoftAssertions SoftAssertions soft; + + @SuppressWarnings("resource") + @Container + static VaultContainer vaultContainer = + new VaultContainer<>( + ContainerSpecHelper.builder() + .name("vault") + .containerClass(ITVaultSecretsSupplier.class) + .build() + .dockerImageName(null) + .asCompatibleSubstituteFor("vault")) + .withLogConsumer(c -> LOGGER.info("[VAULT] {}", c.getUtf8StringWithoutLineEnding())) + .withVaultToken(VAULT_ROOT_TOKEN); + + @Test + public void vault() { + SmallRyeConfig config = + new SmallRyeConfigBuilder() + .setAddDefaultSources(false) + .setAddDiscoveredSources(false) + .withMapping(VaultRuntimeConfig.class) + .withSources( + new PropertiesConfigSource( + Map.of( + "quarkus.vault.url", + vaultContainer.getHttpHostAddress(), + "quarkus.vault.authentication.client-token", + VAULT_ROOT_TOKEN, + "quarkus.vault.secret-config-kv-path", + NESSIE_SECRETS_PATH), + "configSource", + 100)) + .build(); + + VaultRuntimeConfig runtimeConfig = config.getConfigMapping(VaultRuntimeConfig.class); + + VaultConfigHolder configHolder = new VaultConfigHolder(); + configHolder.setVaultRuntimeConfig(runtimeConfig); + VaultClient client = new VaultClientProducer().privateVaultClient(configHolder, true); + + VaultKVSecretReactiveEngine vault = new VaultKvManager(client, configHolder); + + Instant instant = Instant.parse("2024-06-05T20:38:16Z"); + + KeySecret keySecret = keySecret("secret-foo"); + BasicCredentials basicCred = basicCredentials("bar-name", "bar-secret"); + TokenSecret tokenSec = tokenSecret("the-token", instant); + + vault.writeSecret("key", Map.of("key", keySecret.key())).await().indefinitely(); + vault + .writeSecret("basic", Map.of("name", basicCred.name(), "secret", basicCred.secret())) + .await() + .indefinitely(); + vault + .writeSecret( + "tok", + Map.of( + "token", + tokenSec.token(), + "expiresAt", + tokenSec.expiresAt().orElseThrow().toString())) + .await() + .indefinitely(); + + VaultSecretsSupplier secretsSupplier = + new VaultSecretsSupplier(vault, "", Duration.ofMinutes(1)); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of( + "key", + SecretType.KEY, + "basic", + SecretType.BASIC, + "tok", + SecretType.EXPIRING_TOKEN))) + .hasSize(3) + .hasEntrySatisfying( + "key", + s -> + assertThat(s) + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key())) + .hasEntrySatisfying( + "basic", + s -> + assertThat(s) + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret())) + .hasEntrySatisfying( + "tok", + s -> + assertThat(s) + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt())); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of( + "key", + SecretType.KEY, + "nope", + SecretType.BASIC, + "basic", + SecretType.BASIC, + "not-there", + SecretType.KEY, + "tok", + SecretType.EXPIRING_TOKEN))) + .hasSize(3) + .hasEntrySatisfying( + "key", + s -> + assertThat(s) + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key())) + .hasEntrySatisfying( + "basic", + s -> + assertThat(s) + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret())) + .hasEntrySatisfying( + "tok", + s -> + assertThat(s) + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt())); + + soft.assertThat( + secretsSupplier.resolveSecrets( + Map.of("nope", SecretType.BASIC, "not-there", SecretType.KEY))) + .isEmpty(); + } +} diff --git a/catalog/secrets/vault/src/intTest/resources/logback-test.xml b/catalog/secrets/vault/src/intTest/resources/logback-test.xml new file mode 100644 index 00000000000..4f3fda900d7 --- /dev/null +++ b/catalog/secrets/vault/src/intTest/resources/logback-test.xml @@ -0,0 +1,30 @@ + + + + + + + %date{ISO8601} [%thread] [%X{nessie.events.subscription.id}] %-5level %logger{36} - + %msg%n + + + + + + diff --git a/catalog/secrets/vault/src/intTest/resources/org/projectnessie/catalog/secrets/vault/Dockerfile-vault-version b/catalog/secrets/vault/src/intTest/resources/org/projectnessie/catalog/secrets/vault/Dockerfile-vault-version new file mode 100644 index 00000000000..f019422d362 --- /dev/null +++ b/catalog/secrets/vault/src/intTest/resources/org/projectnessie/catalog/secrets/vault/Dockerfile-vault-version @@ -0,0 +1,3 @@ +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM docker.io/hashicorp/vault:1.17.0 diff --git a/catalog/secrets/vault/src/main/java/org/projectnessie/catalog/secrets/vault/VaultSecretsSupplier.java b/catalog/secrets/vault/src/main/java/org/projectnessie/catalog/secrets/vault/VaultSecretsSupplier.java new file mode 100644 index 00000000000..81c7b6835b1 --- /dev/null +++ b/catalog/secrets/vault/src/main/java/org/projectnessie/catalog/secrets/vault/VaultSecretsSupplier.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.vault; + +import io.quarkus.vault.VaultKVSecretReactiveEngine; +import io.quarkus.vault.client.VaultClientException; +import io.smallrye.mutiny.Uni; +import java.time.Duration; +import java.util.Map; +import java.util.stream.Collectors; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.spi.SecretsSupplier; + +public class VaultSecretsSupplier implements SecretsSupplier { + private final VaultKVSecretReactiveEngine secretEngine; + private final String prefix; + private final Duration getSecretTimeout; + + public VaultSecretsSupplier( + VaultKVSecretReactiveEngine secretEngine, String prefix, Duration getSecretTimeout) { + this.secretEngine = secretEngine; + this.prefix = prefix; + this.getSecretTimeout = getSecretTimeout; + } + + @Override + public Map resolveSecrets(Map toResolve) { + return Uni.join() + .all( + toResolve.keySet().stream() + .map( + name -> + secretEngine + .readSecret(path(name)) + .map(json -> Map.entry(name, json)) + .onFailure( + e -> { + if (!(e instanceof VaultClientException)) { + return false; + } + VaultClientException ex = (VaultClientException) e; + return 404 == ex.getStatus(); + }) + .recoverWithItem(Map.entry(name, Map.of()))) + .collect(Collectors.toList())) + .andCollectFailures() + .await() + .atMost(getSecretTimeout) + .stream() + .filter(e -> !e.getValue().isEmpty()) + .collect( + Collectors.toMap( + Map.Entry::getKey, e -> toResolve.get(e.getKey()).fromValueMap(e.getValue()))); + } + + private String path(String name) { + return prefix + name.replace('.', '/'); + } +} diff --git a/catalog/service/impl/build.gradle.kts b/catalog/service/impl/build.gradle.kts index 1e4449bf67e..21c1714ea34 100644 --- a/catalog/service/impl/build.gradle.kts +++ b/catalog/service/impl/build.gradle.kts @@ -71,6 +71,7 @@ dependencies { testFixturesApi(project(":nessie-versioned-spi")) testFixturesApi(project(":nessie-versioned-storage-store")) testFixturesApi(project(":nessie-rest-services")) + testImplementation(testFixtures(project(":nessie-catalog-secrets-api"))) testImplementation(platform(libs.awssdk.bom)) testImplementation("software.amazon.awssdk:s3") diff --git a/catalog/service/impl/src/test/java/org/projectnessie/catalog/service/impl/AbstractCatalogService.java b/catalog/service/impl/src/test/java/org/projectnessie/catalog/service/impl/AbstractCatalogService.java index 0b1574642db..f7c3f22669b 100644 --- a/catalog/service/impl/src/test/java/org/projectnessie/catalog/service/impl/AbstractCatalogService.java +++ b/catalog/service/impl/src/test/java/org/projectnessie/catalog/service/impl/AbstractCatalogService.java @@ -39,7 +39,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Function; -import java.util.stream.Collectors; import org.assertj.core.api.SoftAssertions; import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; @@ -63,6 +62,7 @@ import org.projectnessie.catalog.formats.iceberg.meta.IcebergSortOrder; import org.projectnessie.catalog.formats.iceberg.rest.IcebergCatalogOperation; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.catalog.service.api.CatalogCommit; import org.projectnessie.catalog.service.config.CatalogConfig; import org.projectnessie.catalog.service.config.WarehouseConfig; @@ -206,7 +206,8 @@ private void setupCatalogService() { private void setupObjectIO() { S3Sessions sessions = new S3Sessions("foo", null); S3Config s3config = S3Config.builder().build(); - httpClient = S3Clients.apacheHttpClient(s3config, new SecretsProvider(names -> Map.of())); + httpClient = + S3Clients.apacheHttpClient(s3config, new SecretsProvider(new DummySecretsSupplier())); S3ProgrammaticOptions s3options = ImmutableS3ProgrammaticOptions.builder() .defaultOptions( @@ -219,13 +220,7 @@ private void setupObjectIO() { .build(); S3ClientSupplier clientSupplier = new S3ClientSupplier( - httpClient, - s3options, - new SecretsProvider( - (names) -> - names.stream() - .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))), - sessions); + httpClient, s3options, new SecretsProvider(new DummySecretsSupplier()), sessions); objectIO = new S3ObjectIO(clientSupplier); } diff --git a/cli/cli/build.gradle.kts b/cli/cli/build.gradle.kts index 75941149488..fbfdceee2ea 100644 --- a/cli/cli/build.gradle.kts +++ b/cli/cli/build.gradle.kts @@ -96,6 +96,7 @@ dependencies { intTestImplementation(project(":nessie-keycloak-testcontainer")) intTestImplementation(project(":nessie-container-spec-helper")) intTestImplementation(project(":nessie-catalog-secrets-api")) + intTestImplementation(testFixtures(project(":nessie-catalog-secrets-api"))) nessieQuarkusServer(project(":nessie-quarkus", "quarkusRunner")) } diff --git a/cli/cli/src/intTest/java/org/projectnessie/nessie/cli/commands/WithNessie.java b/cli/cli/src/intTest/java/org/projectnessie/nessie/cli/commands/WithNessie.java index e37a3739599..3f16047caf4 100644 --- a/cli/cli/src/intTest/java/org/projectnessie/nessie/cli/commands/WithNessie.java +++ b/cli/cli/src/intTest/java/org/projectnessie/nessie/cli/commands/WithNessie.java @@ -26,7 +26,6 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; import org.junit.jupiter.api.AfterAll; import org.projectnessie.catalog.files.api.ObjectIO; import org.projectnessie.catalog.files.s3.ImmutableS3NamedBucketOptions; @@ -40,6 +39,7 @@ import org.projectnessie.catalog.formats.iceberg.fixtures.IcebergGenerateFixtures; import org.projectnessie.catalog.formats.iceberg.meta.IcebergTableMetadata; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.spi.DummySecretsSupplier; import org.projectnessie.client.api.NessieApiV2; import org.projectnessie.model.Branch; import org.projectnessie.model.CommitMeta; @@ -106,7 +106,7 @@ protected static void setupObjectStoreAndNessie( S3Config s3config = S3Config.builder().build(); SdkHttpClient httpClient = - S3Clients.apacheHttpClient(s3config, new SecretsProvider(names -> Map.of())); + S3Clients.apacheHttpClient(s3config, new SecretsProvider(new DummySecretsSupplier())); S3ProgrammaticOptions s3options = ImmutableS3ProgrammaticOptions.builder() @@ -123,13 +123,7 @@ protected static void setupObjectStoreAndNessie( S3ClientSupplier clientSupplier = new S3ClientSupplier( - httpClient, - s3options, - new SecretsProvider( - (names) -> - names.stream() - .collect(Collectors.toMap(k -> k, k -> Map.of("secret", "secret")))), - sessions); + httpClient, s3options, new SecretsProvider(new DummySecretsSupplier()), sessions); ObjectIO objectIO = new S3ObjectIO(clientSupplier); diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ca621473bd0..40eb60314d3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -66,6 +66,7 @@ gatling-charts-highcharts = { module = "io.gatling.highcharts:gatling-charts-hig google-cloud-bigtable-bom = { module = "com.google.cloud:google-cloud-bigtable-bom", version = "2.40.0" } google-cloud-gcs-connector = { module = "com.google.cloud.bigdataoss:gcs-connector", version = "hadoop3-2.2.21" } google-cloud-nio = { module = "com.google.cloud:google-cloud-nio", version = "0.127.20" } +google-cloud-secretmanager-bom = { module = "com.google.cloud:google-cloud-secretmanager-bom", version = "2.46.0" } google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.40.1" } google-java-format = { module = "com.google.googlejavaformat:google-java-format", version.ref = "googleJavaFormat" } guava = { module = "com.google.guava:guava", version = "33.2.1-jre" } @@ -105,6 +106,7 @@ junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api" } junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params" } junit-platform-reporting = { module = "org.junit.platform:junit-platform-reporting" } logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logback" } +lowkey-vault-testcontainers = { module = "com.github.nagyesta.lowkey-vault:lowkey-vault-testcontainers", version = "2.4.42" } keycloak-admin-client = { module = "org.keycloak:keycloak-admin-client", version.ref = "keycloak" } mariadb-java-client = { module = "org.mariadb.jdbc:mariadb-java-client", version = "3.4.1" } maven-resolver-supplier = { module = "org.apache.maven.resolver:maven-resolver-supplier", version.ref = "mavenResolver" } @@ -129,6 +131,7 @@ picocli-codegen = { module = "info.picocli:picocli-codegen", version.ref = "pico postgresql = { module = "org.postgresql:postgresql", version = "42.7.3" } protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" } quarkus-amazon-services-bom = { module = "io.quarkus.platform:quarkus-amazon-services-bom", version.ref = "quarkusPlatform" } +quarkus-azure-services-bom = { module = "io.quarkiverse.azureservices:quarkus-azure-services-bom", version = "1.0.4" } quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quarkusPlatform" } quarkus-cassandra-bom = { module = "io.quarkus.platform:quarkus-cassandra-bom", version.ref = "quarkusPlatform" } quarkus-google-cloud-services-bom = { module = "io.quarkus.platform:quarkus-google-cloud-services-bom", version.ref = "quarkusPlatform" } diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 32b2b42fff2..27fe8a6b5a3 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -94,3 +94,8 @@ nessie-catalog-service-rest=catalog/service/rest nessie-catalog-service-impl=catalog/service/impl nessie-catalog-service-transfer=catalog/service/transfer nessie-catalog-secrets-api=catalog/secrets/api +nessie-catalog-secrets-cache=catalog/secrets/cache +nessie-catalog-secrets-aws=catalog/secrets/aws +nessie-catalog-secrets-gcs=catalog/secrets/gcs +nessie-catalog-secrets-azure=catalog/secrets/azure +nessie-catalog-secrets-vault=catalog/secrets/vault