diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index eb5b9cc0869..a1646a197ec 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -103,6 +103,10 @@ dependencies { api(project(":nessie-catalog-service-rest")) api(project(":nessie-catalog-service-impl")) api(project(":nessie-catalog-secrets-api")) + api(project(":nessie-catalog-secrets-cache")) + api(project(":nessie-catalog-secrets-aws")) + api(project(":nessie-catalog-secrets-gcs")) + api(project(":nessie-catalog-secrets-vault")) if (!isIncludedInNesQuEIT()) { api(project(":nessie-spark-antlr-runtime")) diff --git a/catalog/secrets/aws/build.gradle.kts b/catalog/secrets/aws/build.gradle.kts new file mode 100644 index 00000000000..dd2a1b7e5d4 --- /dev/null +++ b/catalog/secrets/aws/build.gradle.kts @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets AWS" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:apache-client") { + exclude("commons-logging", "commons-logging") + } + implementation("software.amazon.awssdk:secretsmanager") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) + + intTestCompileOnly(project(":nessie-immutables")) + intTestImplementation(platform(libs.testcontainers.bom)) + intTestImplementation("org.testcontainers:testcontainers") + intTestImplementation("org.testcontainers:localstack") + intTestImplementation("org.testcontainers:junit-jupiter") + intTestImplementation(project(":nessie-container-spec-helper")) + intTestRuntimeOnly(libs.logback.classic) +} diff --git a/catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsSupplier.java b/catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsSupplier.java new file mode 100644 index 00000000000..ccb0a0d577f --- /dev/null +++ b/catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsSupplier.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.aws; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.testcontainers.containers.localstack.LocalStackContainer.Service.SECRETSMANAGER; + +import java.net.URI; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.projectnessie.nessie.testing.containerspec.ContainerSpecHelper; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.CreateSecretRequest; + +@Testcontainers +public class ITAwsSecretsSupplier { + @Container + static LocalStackContainer localstack = + new LocalStackContainer( + ContainerSpecHelper.builder() + .name("localstack") + .containerClass(ITAwsSecretsSupplier.class) + .build() + .dockerImageName(null) + .asCompatibleSubstituteFor("localstack/localstack")) + .withLogConsumer( + new Slf4jLogConsumer(LoggerFactory.getLogger(ITAwsSecretsSupplier.class))) + .withServices(SECRETSMANAGER); + + @Test + public void awsSecretsManager() { + URI secretsManagerEndpoint = localstack.getEndpointOverride(SECRETSMANAGER); + try (SecretsManagerClient client = + SecretsManagerClient.builder() + .endpointOverride(secretsManagerEndpoint) + .region(Region.of(localstack.getRegion())) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create( + localstack.getAccessKey(), localstack.getSecretKey()))) + .build()) { + client.createSecret( + CreateSecretRequest.builder().name("foo").secretString("secret-foo").build()); + client.createSecret( + CreateSecretRequest.builder() + .name("single") + .secretString("{\"bar\": \"secret-single\"}") + .build()); + client.createSecret( + CreateSecretRequest.builder() + .name("multi") + .secretString("{\"name\": \"the-name\", \"value\": \"the-value\"}") + .build()); + + AwsSecretsSupplier awsSecretsSupplier = new AwsSecretsSupplier(client); + + assertThat(awsSecretsSupplier.resolveSecrets(List.of("foo", "single", "multi"))) + .containsEntry("foo", Map.of("value", "secret-foo")) + .containsEntry("single", Map.of("bar", "secret-single")) + .containsEntry("multi", Map.of("name", "the-name", "value", "the-value")) + .hasSize(3); + } + } +} diff --git a/catalog/secrets/aws/src/intTest/resources/logback-test.xml b/catalog/secrets/aws/src/intTest/resources/logback-test.xml new file mode 100644 index 00000000000..ce140f00774 --- /dev/null +++ b/catalog/secrets/aws/src/intTest/resources/logback-test.xml @@ -0,0 +1,30 @@ + + + + + + + %date{ISO8601} [%thread] [%X{nessie.events.subscription.id}] %-5level %logger{36} - + %msg%n + + + + + + diff --git a/catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version b/catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version new file mode 100644 index 00000000000..814ae8b25c1 --- /dev/null +++ b/catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version @@ -0,0 +1,3 @@ +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM docker.io/localstack/localstack:3.4.0 diff --git a/catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsSupplier.java b/catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsSupplier.java new file mode 100644 index 00000000000..20528daca69 --- /dev/null +++ b/catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsSupplier.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.aws; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import org.projectnessie.catalog.secrets.spi.SingleValueSecretsSupplier; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.BatchGetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.SecretValueEntry; + +public class AwsSecretsSupplier extends SingleValueSecretsSupplier { + private final SecretsManagerClient secretsManagerClient; + + public AwsSecretsSupplier(SecretsManagerClient secretsManagerClient) { + this.secretsManagerClient = secretsManagerClient; + } + + @Override + protected Map resolveSingleValueSecrets(Collection names) { + if (names.isEmpty()) { + return Map.of(); + } + + Map secretIdToNameMap = new HashMap<>(); + + for (String name : names) { + String secretId = nameToSecretId(name); + secretIdToNameMap.put(secretId, name); + } + + return secretsManagerClient + .batchGetSecretValue( + BatchGetSecretValueRequest.builder().secretIdList(secretIdToNameMap.keySet()).build()) + .secretValues() + .stream() + .collect( + Collectors.toMap( + sv -> secretIdToNameMap.get(sv.name()), SecretValueEntry::secretString)); + } + + private String nameToSecretId(String name) { + // TODO can either return the name (as known to AWS Secrets Manager) or the fully qualified ARN + return name; + } +} diff --git a/catalog/secrets/cache/build.gradle.kts b/catalog/secrets/cache/build.gradle.kts new file mode 100644 index 00000000000..ae138fba28f --- /dev/null +++ b/catalog/secrets/cache/build.gradle.kts @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets Cache" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + implementation(libs.caffeine) + implementation(libs.micrometer.core) + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-databind") + implementation("com.fasterxml.jackson.core:jackson-annotations") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) +} diff --git a/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java new file mode 100644 index 00000000000..dda9f49f8cb --- /dev/null +++ b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import org.projectnessie.catalog.secrets.spi.SecretsSupplier; + +public final class CachingSecrets { + + private final CachingSecretsBackend backend; + + public CachingSecrets(CachingSecretsBackend backend) { + this.backend = backend; + } + + public SecretsSupplier forRepository(String repositoryId, SecretsSupplier secretsSupplier) { + return names -> backend.resolveSecrets(repositoryId, names, secretsSupplier); + } +} diff --git a/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecretsBackend.java b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecretsBackend.java new file mode 100644 index 00000000000..1bd0147858d --- /dev/null +++ b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecretsBackend.java @@ -0,0 +1,260 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import static java.util.Collections.singletonList; +import static java.util.Collections.unmodifiableMap; +import static java.util.stream.Collectors.toMap; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Expiry; +import com.google.common.annotations.VisibleForTesting; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.binder.cache.CaffeineStatsCounter; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; +import org.checkerframework.checker.index.qual.NonNegative; +import org.projectnessie.catalog.secrets.spi.SecretsSupplier; + +public class CachingSecretsBackend { + + private static final Map CACHE_NEGATIVE_SENTINEL = new HashMap<>(); + + public static final String CACHE_NAME = "nessie-secrets"; + private static final long NOT_CACHED = 0L; + + @VisibleForTesting final Cache> cache; + private final long ttlNanos; + private final LongSupplier clock; + + public CachingSecretsBackend(SecretsCacheConfig config) { + OptionalLong ttl = config.ttlMillis(); + this.ttlNanos = ttl.isPresent() ? TimeUnit.MILLISECONDS.toNanos(ttl.getAsLong()) : 0L; + this.clock = config.clockNanos(); + + Caffeine> cacheBuilder = + Caffeine.newBuilder() + .expireAfter( + new Expiry>() { + @Override + public long expireAfterCreate( + CacheKeyValue key, Map value, long currentTimeNanos) { + long expire = key.expiresAtNanosEpoch; + if (expire == NOT_CACHED) { + return 0L; + } + long remaining = expire - currentTimeNanos; + return Math.max(0L, remaining); + } + + @Override + public long expireAfterUpdate( + CacheKeyValue key, + Map value, + long currentTimeNanos, + @NonNegative long currentDurationNanos) { + return expireAfterCreate(key, value, currentTimeNanos); + } + + @Override + public long expireAfterRead( + CacheKeyValue key, + Map value, + long currentTimeNanos, + @NonNegative long currentDurationNanos) { + return currentDurationNanos; + } + }) + .ticker(clock::getAsLong); + config + .capacityMb() + .ifPresent(mb -> cacheBuilder.maximumWeight(mb * 1024L * 1024L).weigher(this::weigher)); + config.maxElements().ifPresent(cacheBuilder::maximumSize); + config + .meterRegistry() + .ifPresent( + meterRegistry -> { + cacheBuilder.recordStats(() -> new CaffeineStatsCounter(meterRegistry, CACHE_NAME)); + meterRegistry.gauge( + "cache_capacity_mb", + singletonList(Tag.of("cache", CACHE_NAME)), + "", + x -> config.capacityMb().orElse(0)); + meterRegistry.gauge( + "cache_max_size", + singletonList(Tag.of("cache", CACHE_NAME)), + "", + x -> config.maxElements().orElse(0)); + meterRegistry.gauge( + "cache_element_ttl", + singletonList(Tag.of("cache", CACHE_NAME)), + "", + x -> config.ttlMillis().orElse(0)); + }); + + this.cache = cacheBuilder.build(); + } + + private int weigher(CacheKeyValue key, Map value) { + int size = key.heapSize(); + if (isPresentValue(value)) { + for (Map.Entry e : value.entrySet()) { + size += STRING_OBJ_OVERHEAD + e.getKey().length(); + size += STRING_OBJ_OVERHEAD + e.getValue().length(); + } + size += MAP_OBJ_OVERHEAD; + } + size += CAFFEINE_OBJ_OVERHEAD; + return size; + } + + private static boolean isPresentValue(Map value) { + // Yes, same instance for + return value != null && value != CACHE_NEGATIVE_SENTINEL; + } + + Map> resolveSecrets( + String repositoryId, Collection names, SecretsSupplier backend) { + long ttl = ttlNanos; + long expires = ttl != 0L ? clock.getAsLong() + ttl : 0L; + Map keys = new HashMap<>(); + for (String name : names) { + keys.put(name, new CacheKeyValue(repositoryId, name, expires)); + } + + Map> result = new HashMap<>(); + + Map> present = cache.getAllPresent(keys.values()); + present.forEach( + (k, v) -> { + String name = k.name; + keys.remove(name); + if (isPresentValue(v)) { + result.put(name, v); + } + }); + + if (!keys.isEmpty()) { + Map> resolved = backend.resolveSecrets(keys.keySet()); + + Map> put = new HashMap<>(); + for (CacheKeyValue key : keys.values()) { + Map value = resolved.getOrDefault(key.name, CACHE_NEGATIVE_SENTINEL); + if (isPresentValue(value)) { + value = + unmodifiableMap( + value.entrySet().stream() + .collect(toMap(Map.Entry::getKey, CachingSecretsBackend::safeStringValue))); + result.put(key.name, value); + } + put.put(key, value); + } + cache.putAll(put); + } + + return result; + } + + private static String safeStringValue(Map.Entry e) { + // The string doesn't necessarily need to be a string, for example if the secrets backend + // returns another object (Vault could do that?). + return e.getValue().toString(); + } + + static final class CacheKeyValue { + final String repositoryId; + final String name; + + // Revisit this field before 2262-04-11T23:47:16.854Z (64-bit signed long overflow) ;) ;) + final long expiresAtNanosEpoch; + + CacheKeyValue(String repositoryId, String name, long expiresAtNanosEpoch) { + this.repositoryId = repositoryId; + this.name = name; + this.expiresAtNanosEpoch = expiresAtNanosEpoch; + } + + int heapSize() { + int size = OBJ_SIZE; + size += STRING_OBJ_OVERHEAD + repositoryId.length(); + size += STRING_OBJ_OVERHEAD + name.length(); + return size; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CacheKeyValue)) { + return false; + } + CacheKeyValue cacheKey = (CacheKeyValue) o; + return repositoryId.equals(cacheKey.repositoryId) && name.equals(cacheKey.name); + } + + @Override + public int hashCode() { + return repositoryId.hashCode() * 31 + name.hashCode(); + } + + @Override + public String toString() { + return "{" + repositoryId + ", " + name + '}'; + } + } + + /* + CacheKeyValue object internals: + OFF SZ TYPE DESCRIPTION VALUE + 0 8 (object header: mark) 0x0000000000000001 (non-biasable; age: 0) + 8 4 (object header: class) 0x010c4800 + 12 4 java.lang.String CacheKeyValue.repositoryId null + 16 8 long CacheKeyValue.expiresAt 0 + 24 4 java.lang.String CacheKeyValue.id null + 28 4 (object alignment gap) + Instance size: 32 bytes + */ + static final int OBJ_SIZE = 32; + // rough estimate, probably good enough + static final int MAP_OBJ_OVERHEAD = OBJ_SIZE * 2; + /* + Array overhead: 16 bytes + */ + static final int ARRAY_OVERHEAD = 16; + /* + java.lang.String object internals: + OFF SZ TYPE DESCRIPTION VALUE + 0 8 (object header: mark) 0x0000000000000001 (non-biasable; age: 0) + 8 4 (object header: class) 0x0000e8d8 + 12 4 int String.hash 0 + 16 1 byte String.coder 0 + 17 1 boolean String.hashIsZero false + 18 2 (alignment/padding gap) + 20 4 byte[] String.value [] + Instance size: 24 bytes + */ + static final int STRING_OBJ_OVERHEAD = 24 + ARRAY_OVERHEAD; + /* + Assume an overhead of 2 objects for each entry (java.util.concurrent.ConcurrentHashMap$Node is 32 bytes) in Caffeine. + */ + static final int CAFFEINE_OBJ_OVERHEAD = 2 * 32; +} diff --git a/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/SecretsCacheConfig.java b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/SecretsCacheConfig.java new file mode 100644 index 00000000000..e3c5649527b --- /dev/null +++ b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/SecretsCacheConfig.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import io.micrometer.core.instrument.MeterRegistry; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.function.LongSupplier; +import org.immutables.value.Value; + +@Value.Immutable +public interface SecretsCacheConfig { + + OptionalLong maxElements(); + + OptionalLong capacityMb(); + + OptionalLong ttlMillis(); + + Optional meterRegistry(); + + @Value.Default + default LongSupplier clockNanos() { + return System::nanoTime; + } + + static Builder builder() { + return ImmutableSecretsCacheConfig.builder(); + } + + interface Builder { + @CanIgnoreReturnValue + Builder capacityMb(long capacityMb); + + @CanIgnoreReturnValue + Builder maxElements(long maxElements); + + @CanIgnoreReturnValue + Builder ttlMillis(long ttlMillis); + + @CanIgnoreReturnValue + Builder meterRegistry(MeterRegistry meterRegistry); + + @CanIgnoreReturnValue + Builder clockNanos(LongSupplier clockNanos); + + SecretsCacheConfig build(); + } +} diff --git a/catalog/secrets/cache/src/test/java/org/projectnessie/catalog/secrets/cache/TestCachingSecretsBackend.java b/catalog/secrets/cache/src/test/java/org/projectnessie/catalog/secrets/cache/TestCachingSecretsBackend.java new file mode 100644 index 00000000000..b143bf04f6b --- /dev/null +++ b/catalog/secrets/cache/src/test/java/org/projectnessie/catalog/secrets/cache/TestCachingSecretsBackend.java @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import static java.util.function.Function.identity; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.projectnessie.catalog.secrets.spi.SecretsSupplier; + +@ExtendWith(SoftAssertionsExtension.class) +public class TestCachingSecretsBackend { + @InjectSoftAssertions protected SoftAssertions soft; + + static final Map SECRET_1_REPO_1 = Map.of("value", "secret1-repo1"); + static final Map SECRET_2_REPO_1 = Map.of("value", "secret2-repo1"); + static final Map SECRET_1_REPO_2 = Map.of("value", "secret1-repo2"); + static final Map SECRET_2_REPO_2 = Map.of("value", "secret2-repo2"); + + AtomicLong clock; + CachingSecretsBackend backend; + CachingSecrets cachingSecrets; + SecretsSupplier secrets1; + SecretsSupplier secrets2; + SecretsSupplier caching1; + SecretsSupplier caching2; + + @BeforeEach + void setup() { + clock = new AtomicLong(); + backend = + new CachingSecretsBackend( + SecretsCacheConfig.builder() + .capacityMb(1) + .ttlMillis(1000) + .clockNanos(clock::get) + .build()); + + cachingSecrets = new CachingSecrets(backend); + + secrets1 = mapSecretsSupplier(Map.of("secret1", SECRET_1_REPO_1, "secret2", SECRET_2_REPO_1)); + secrets2 = mapSecretsSupplier(Map.of("secret1", SECRET_1_REPO_2, "secret2", SECRET_2_REPO_2)); + + caching1 = cachingSecrets.forRepository("repo1", secrets1); + caching2 = cachingSecrets.forRepository("repo2", secrets2); + } + + @Test + public void ttlExpire() { + soft.assertThat(caching1.resolveSecrets(List.of("secret1"))) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_1)); + soft.assertThat(caching2.resolveSecrets(List.of("secret1"))) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_2)); + soft.assertThat(caching1.resolveSecrets(List.of("secret1"))) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_1)); + soft.assertThat(caching2.resolveSecrets(List.of("secret1"))) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_2)); + + soft.assertThat(backend.cache.asMap()) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret1", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret1", 0)); + + clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(1000)); + + soft.assertThat(backend.cache.asMap()) + .doesNotContainKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret1", 0)) + .doesNotContainKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret1", 0)); + + soft.assertThat(caching1.resolveSecrets(List.of("secret1", "secret2"))) + .containsExactlyInAnyOrderEntriesOf( + Map.of("secret1", SECRET_1_REPO_1, "secret2", SECRET_2_REPO_1)); + soft.assertThat(caching2.resolveSecrets(List.of("secret1", "secret2"))) + .containsExactlyInAnyOrderEntriesOf( + Map.of("secret1", SECRET_1_REPO_2, "secret2", SECRET_2_REPO_2)); + + soft.assertThat(backend.cache.asMap()) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret1", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret1", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret2", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret2", 0)); + } + + @Test + public void nonExisting() { + soft.assertThat(caching1.resolveSecrets(List.of("secret-nope1", "secret-nope2"))).isEmpty(); + soft.assertThat(caching2.resolveSecrets(List.of("secret-nope1", "secret-nope2"))).isEmpty(); + soft.assertThat(caching1.resolveSecrets(List.of("secret-nope1", "secret-nope2", "secret1"))) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_1)); + soft.assertThat(caching2.resolveSecrets(List.of("secret-nope1", "secret-nope2", "secret1"))) + .containsExactlyInAnyOrderEntriesOf(Map.of("secret1", SECRET_1_REPO_2)); + } + + @Test + public void tooManyEntries() { + SecretsSupplier supplier = + names -> names.stream().collect(Collectors.toMap(identity(), name -> Map.of(name, name))); + SecretsSupplier caching = cachingSecrets.forRepository("repoMany", supplier); + + // Very naive test that checks that expiration happens - our "weigher" does something. + + for (int i = 0; i < 100_000; i++) { + caching.resolveSecrets( + List.of( + "wepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoew-" + + i)); + // In fact, the cache should never have more than 10000 entries, but 'estimatedSize' is really + // just an estimate. + soft.assertThat(backend.cache.estimatedSize()).isLessThan(10_000); + } + } + + static SecretsSupplier mapSecretsSupplier(Map> secretsMap) { + return names -> { + Map> resolved = new HashMap<>(); + for (String name : names) { + Map r = secretsMap.get(name); + if (r != null) { + resolved.put(name, r); + } + } + return resolved; + }; + } +} diff --git a/catalog/secrets/gcs/build.gradle.kts b/catalog/secrets/gcs/build.gradle.kts new file mode 100644 index 00000000000..2641bced3a6 --- /dev/null +++ b/catalog/secrets/gcs/build.gradle.kts @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets GCS" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(platform(libs.google.cloud.secretmanager.bom)) + implementation("com.google.cloud:google-cloud-secretmanager") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) +} diff --git a/catalog/secrets/gcs/src/main/java/org/projectnessie/catalog/secrets/gcs/GcsSecretsSupplier.java b/catalog/secrets/gcs/src/main/java/org/projectnessie/catalog/secrets/gcs/GcsSecretsSupplier.java new file mode 100644 index 00000000000..ef882633812 --- /dev/null +++ b/catalog/secrets/gcs/src/main/java/org/projectnessie/catalog/secrets/gcs/GcsSecretsSupplier.java @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.gcs; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.cloud.secretmanager.v1.AccessSecretVersionRequest; +import com.google.cloud.secretmanager.v1.AccessSecretVersionResponse; +import com.google.cloud.secretmanager.v1.SecretManagerServiceClient; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import org.projectnessie.catalog.secrets.spi.SingleValueSecretsSupplier; + +public class GcsSecretsSupplier extends SingleValueSecretsSupplier { + private final SecretManagerServiceClient client; + + public GcsSecretsSupplier(SecretManagerServiceClient client) { + this.client = client; + } + + @Override + protected Map resolveSingleValueSecrets(Collection names) { + if (names.isEmpty()) { + return Map.of(); + } + + if (names.size() == 1) { + String name = names.iterator().next(); + AccessSecretVersionResponse response = client.accessSecretVersion(request(name)); + return response.hasPayload() ? Map.of(name, secret(response)) : Map.of(); + } + + List> futures = + names.stream() + .map(name -> client.accessSecretVersionCallable().futureCall(request(name))) + .collect(Collectors.toList()); + + List results; + try { + results = ApiFutures.successfulAsList(futures).get(1000, TimeUnit.MILLISECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new RuntimeException(e); + } + + return results.stream() + .collect(Collectors.toMap(AccessSecretVersionResponse::getName, this::secret)); + } + + private String secret(AccessSecretVersionResponse response) { + return response.getPayload().getData().toStringUtf8(); + } + + private AccessSecretVersionRequest request(String name) { + return AccessSecretVersionRequest.newBuilder().setName(name).build(); + } +} diff --git a/catalog/secrets/vault/build.gradle.kts b/catalog/secrets/vault/build.gradle.kts new file mode 100644 index 00000000000..69048c5422a --- /dev/null +++ b/catalog/secrets/vault/build.gradle.kts @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("nessie-conventions-server") + id("nessie-jacoco") +} + +extra["maven.name"] = "Nessie - Catalog - Secrets Vault" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(enforcedPlatform(libs.quarkus.bom)) + implementation("io.quarkiverse.vault:quarkus-vault") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) + + intTestImplementation(platform(libs.testcontainers.bom)) + intTestImplementation("org.testcontainers:testcontainers") + intTestImplementation("org.testcontainers:vault") + intTestImplementation("org.testcontainers:junit-jupiter") + intTestImplementation(project(":nessie-container-spec-helper")) + intTestImplementation(libs.smallrye.config.core) + intTestCompileOnly(libs.immutables.value.annotations) + intTestRuntimeOnly(libs.logback.classic) +} diff --git a/catalog/secrets/vault/src/intTest/java/org/projectnessie/catalog/secrets/vault/ITVaultSecretsSupplier.java b/catalog/secrets/vault/src/intTest/java/org/projectnessie/catalog/secrets/vault/ITVaultSecretsSupplier.java new file mode 100644 index 00000000000..4d7bd3d9004 --- /dev/null +++ b/catalog/secrets/vault/src/intTest/java/org/projectnessie/catalog/secrets/vault/ITVaultSecretsSupplier.java @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.vault; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.quarkus.runtime.TlsConfig; +import io.quarkus.vault.VaultKVSecretReactiveEngine; +import io.quarkus.vault.client.VaultClient; +import io.quarkus.vault.runtime.VaultConfigHolder; +import io.quarkus.vault.runtime.VaultKvManager; +import io.quarkus.vault.runtime.client.VaultClientProducer; +import io.quarkus.vault.runtime.config.VaultRuntimeConfig; +import io.smallrye.config.PropertiesConfigSource; +import io.smallrye.config.SmallRyeConfig; +import io.smallrye.config.SmallRyeConfigBuilder; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.projectnessie.nessie.testing.containerspec.ContainerSpecHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.vault.VaultContainer; + +@Testcontainers +public class ITVaultSecretsSupplier { + private static final Logger LOGGER = LoggerFactory.getLogger(ITVaultSecretsSupplier.class); + + public static final String VAULT_TOKEN = "root"; + public static final String NESSIE_SECRETS_PATH = "apps/nessie/secrets"; + + @Container + static VaultContainer vaultContainer = + new VaultContainer<>( + ContainerSpecHelper.builder() + .name("vault") + .containerClass(ITVaultSecretsSupplier.class) + .build() + .dockerImageName(null) + .asCompatibleSubstituteFor("vault")) + .withLogConsumer(c -> LOGGER.info("[VAULT] {}", c.getUtf8StringWithoutLineEnding())) + .withVaultToken(VAULT_TOKEN); + + @Test + public void vault() { + SmallRyeConfig config = + new SmallRyeConfigBuilder() + .setAddDefaultSources(false) + .setAddDiscoveredSources(false) + .withMapping(VaultRuntimeConfig.class) + .withSources( + new PropertiesConfigSource( + Map.of( + "quarkus.vault.url", + vaultContainer.getHttpHostAddress(), + "quarkus.vault.authentication.client-token", + VAULT_TOKEN, + "quarkus.vault.secret-config-kv-path", + NESSIE_SECRETS_PATH), + "configSource", + 100)) + .build(); + + VaultRuntimeConfig runtimeConfig = config.getConfigMapping(VaultRuntimeConfig.class); + + VaultConfigHolder configHolder = new VaultConfigHolder(); + configHolder.setVaultRuntimeConfig(runtimeConfig); + TlsConfig tlsConfig = new TlsConfig(); + tlsConfig.trustAll = true; + VaultClient client = new VaultClientProducer().privateVaultClient(configHolder, tlsConfig); + + VaultKVSecretReactiveEngine vault = new VaultKvManager(client, configHolder); + + vault.writeSecret("foo", Map.of("value", "secret-foo")).await().indefinitely(); + vault.writeSecret("single", Map.of("bar", "secret-single")).await().indefinitely(); + vault + .writeSecret("multi", Map.of("name", "the-name", "value", "the-value")) + .await() + .indefinitely(); + + VaultSecretsSupplier vaultSecretsSupplier = new VaultSecretsSupplier(vault); + + assertThat(vaultSecretsSupplier.resolveSecrets(List.of("foo", "single", "multi"))) + .containsEntry("foo", Map.of("value", "secret-foo")) + .containsEntry("single", Map.of("bar", "secret-single")) + .containsEntry("multi", Map.of("name", "the-name", "value", "the-value")) + .hasSize(3); + } +} diff --git a/catalog/secrets/vault/src/intTest/resources/logback-test.xml b/catalog/secrets/vault/src/intTest/resources/logback-test.xml new file mode 100644 index 00000000000..4f3fda900d7 --- /dev/null +++ b/catalog/secrets/vault/src/intTest/resources/logback-test.xml @@ -0,0 +1,30 @@ + + + + + + + %date{ISO8601} [%thread] [%X{nessie.events.subscription.id}] %-5level %logger{36} - + %msg%n + + + + + + diff --git a/catalog/secrets/vault/src/intTest/resources/org/projectnessie/catalog/secrets/vault/Dockerfile-vault-version b/catalog/secrets/vault/src/intTest/resources/org/projectnessie/catalog/secrets/vault/Dockerfile-vault-version new file mode 100644 index 00000000000..5d45fa761ed --- /dev/null +++ b/catalog/secrets/vault/src/intTest/resources/org/projectnessie/catalog/secrets/vault/Dockerfile-vault-version @@ -0,0 +1,3 @@ +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM docker.io/hashicorp/vault:1.16.3 diff --git a/catalog/secrets/vault/src/main/java/org/projectnessie/catalog/secrets/vault/VaultSecretsSupplier.java b/catalog/secrets/vault/src/main/java/org/projectnessie/catalog/secrets/vault/VaultSecretsSupplier.java new file mode 100644 index 00000000000..2f9c7a72e73 --- /dev/null +++ b/catalog/secrets/vault/src/main/java/org/projectnessie/catalog/secrets/vault/VaultSecretsSupplier.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.vault; + +import io.quarkus.vault.VaultKVSecretReactiveEngine; +import io.smallrye.mutiny.Uni; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.Collection; +import java.util.Map; +import java.util.stream.Collectors; +import org.projectnessie.catalog.secrets.spi.SecretsSupplier; + +public class VaultSecretsSupplier implements SecretsSupplier { + private final VaultKVSecretReactiveEngine secretEngine; + + public VaultSecretsSupplier(VaultKVSecretReactiveEngine secretEngine) { + this.secretEngine = secretEngine; + } + + @Override + public Map> resolveSecrets(Collection names) { + return Uni.join() + .all( + names.stream() + .map(name -> secretEngine.readSecret(path(name)).map(json -> Map.entry(name, json))) + .collect(Collectors.toList())) + .andCollectFailures() + .await() + .atMost(Duration.of(1, ChronoUnit.SECONDS)) + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private String path(String name) { + // TODO - is this good enough to rely on quarkus.vault.secret-config-kv-path ?? + return name; + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1eabc462534..1830f7da835 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -65,6 +65,7 @@ gatling-charts-highcharts = { module = "io.gatling.highcharts:gatling-charts-hig google-cloud-bigtable-bom = { module = "com.google.cloud:google-cloud-bigtable-bom", version = "2.39.5" } google-cloud-gcs-connector = { module = "com.google.cloud.bigdataoss:gcs-connector", version = "hadoop3-2.2.21" } google-cloud-nio = { module = "com.google.cloud:google-cloud-nio", version = "0.127.19" } +google-cloud-secretmanager-bom = { module = "com.google.cloud:google-cloud-secretmanager-bom", version = "2.45.0" } google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.40.0" } google-java-format = { module = "com.google.googlejavaformat:google-java-format", version.ref = "googleJavaFormat" } guava = { module = "com.google.guava:guava", version = "33.2.1-jre" } diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 0d0c16ca921..91dd7f1a4f4 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -90,3 +90,7 @@ nessie-catalog-service-common=catalog/service/common nessie-catalog-service-rest=catalog/service/rest nessie-catalog-service-impl=catalog/service/impl nessie-catalog-secrets-api=catalog/secrets/api +nessie-catalog-secrets-cache=catalog/secrets/cache +nessie-catalog-secrets-aws=catalog/secrets/aws +nessie-catalog-secrets-gcs=catalog/secrets/gcs +nessie-catalog-secrets-vault=catalog/secrets/vault