diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index 82b6857ce24..288bd8292b5 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -113,6 +113,11 @@ dependencies { api(project(":nessie-catalog-service-impl")) api(project(":nessie-catalog-service-transfer")) api(project(":nessie-catalog-secrets-api")) + api(project(":nessie-catalog-secrets-cache")) + api(project(":nessie-catalog-secrets-aws")) + api(project(":nessie-catalog-secrets-gcs")) + api(project(":nessie-catalog-secrets-azure")) + api(project(":nessie-catalog-secrets-vault")) if (!isIncludedInNesQuEIT()) { api(project(":nessie-gc-iceberg")) diff --git a/catalog/files/impl/build.gradle.kts b/catalog/files/impl/build.gradle.kts index 31a2b0e3d8f..77c88d727a2 100644 --- a/catalog/files/impl/build.gradle.kts +++ b/catalog/files/impl/build.gradle.kts @@ -65,11 +65,13 @@ dependencies { testImplementation(platform(libs.cel.bom)) testImplementation("org.projectnessie.cel:cel-standalone") + testRuntimeOnly(libs.logback.classic) jmhImplementation(libs.jmh.core) jmhImplementation(project(":nessie-object-storage-mock")) jmhAnnotationProcessor(libs.jmh.generator.annprocess) + jmhImplementation(testFixtures(project(":nessie-catalog-secrets-api"))) } tasks.named("processJmhJandexIndex").configure { enabled = false } diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/AbstractStringBasedSecretsProvider.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/AbstractStringBasedSecretsProvider.java new file mode 100644 index 00000000000..3a1611d3b6c --- /dev/null +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/AbstractStringBasedSecretsProvider.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets; + +import jakarta.annotation.Nonnull; +import java.util.Map; + +public abstract class AbstractStringBasedSecretsProvider extends AbstractMapBasedSecretsProvider { + @Override + protected Map resolveSecret(@Nonnull String name) { + String value = resolveSecretString(name); + return value != null ? parseOrSingle(value) : null; + } + + protected abstract String resolveSecretString(String name); +} diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java index 9e8429159a9..ae3d85bc8b1 100644 --- a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/Secret.java @@ -17,7 +17,14 @@ import java.util.Map; -/** Base interface for all secrets. */ +/** + * Base interface for all secrets. + * + *

Secrets must not implement (override) any of these functions in a way: {@link + * Object#toString()}, {@link Object#hashCode()} or {@link Object#equals(Object)} that would + * directly (for example return a secret value from {@code toString()}) or indirectly (compare the + * instance itself against another instance) expose the values of a secret. + */ public interface Secret { Map asMap(); } diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretJsonParser.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretJsonParser.java new file mode 100644 index 00000000000..cf05230c820 --- /dev/null +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretJsonParser.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Map; + +final class SecretJsonParser { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private SecretJsonParser() {} + + @SuppressWarnings("unchecked") + static Map parseOrSingle(String s) { + if (!s.trim().startsWith("{")) { + return Map.of("value", s); + } + try { + return MAPPER.readValue(s, Map.class); + } catch (JsonProcessingException e) { + return Map.of("value", s); + } + } +} diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java index d92db7079d8..2fe802e6e52 100644 --- a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java @@ -17,6 +17,7 @@ import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; import static org.projectnessie.catalog.secrets.KeySecret.keySecret; +import static org.projectnessie.catalog.secrets.SecretJsonParser.parseOrSingle; import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; import java.util.Map; @@ -33,6 +34,16 @@ public Secret fromValueMap(Map value) { public Secret fromValueMap(Map value) { return keySecret(value); } + + @Override + public Secret parse(String string) { + return keySecret(string); + } + + @Override + public boolean singleValued() { + return true; + } }, EXPIRING_TOKEN() { @Override @@ -44,4 +55,12 @@ public Secret fromValueMap(Map value) { /** Construct a {@link Secret} instance from its map representation. */ public abstract Secret fromValueMap(Map value); + + public boolean singleValued() { + return false; + } + + public Secret parse(String string) { + return fromValueMap(parseOrSingle(string)); + } } diff --git a/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretType.java b/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretType.java new file mode 100644 index 00000000000..9d59857a226 --- /dev/null +++ b/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretType.java @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets; + +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +public class TestSecretType { + @InjectSoftAssertions protected SoftAssertions soft; + + @Test + public void keySecret() { + soft.assertThat(SecretType.KEY.parse("foo")) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("foo"); + soft.assertThat(SecretType.KEY.parse("{\"key\": \"foo\"}")) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("{\"key\": \"foo\"}"); + soft.assertThat(SecretType.KEY.parse("{\"value\": \"foo\"}")) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("{\"value\": \"foo\"}"); + + soft.assertThat(SecretType.KEY.fromValueMap(Map.of("key", "foo"))) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("foo"); + soft.assertThat(SecretType.KEY.fromValueMap(Map.of("value", "foo"))) + .extracting(KeySecret.class::cast) + .extracting(KeySecret::key) + .isEqualTo("foo"); + soft.assertThat(SecretType.KEY.fromValueMap(Map.of("blah", "foo"))).isNull(); + soft.assertThat(SecretType.KEY.fromValueMap(Map.of())).isNull(); + } + + @Test + public void tokenSecret() { + String instantStr = "2024-12-24T12:12:12Z"; + Instant instant = Instant.parse(instantStr); + + soft.assertThat(SecretType.EXPIRING_TOKEN.parse("tok")) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.empty()); + soft.assertThat(SecretType.EXPIRING_TOKEN.parse("{\"value\": \"tok\"}")) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.empty()); + soft.assertThat( + SecretType.EXPIRING_TOKEN.parse( + "{\"value\": \"tok\", \"expiresAt\": \"" + instantStr + "\"}")) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.of(instant)); + soft.assertThat(SecretType.EXPIRING_TOKEN.parse("{\"blah\": \"foo\"}")).isNull(); + soft.assertThat(SecretType.EXPIRING_TOKEN.parse("{}")).isNull(); + + soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("token", "tok"))) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.empty()); + soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("value", "tok"))) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.empty()); + soft.assertThat( + SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("token", "tok", "expiresAt", instantStr))) + .extracting(TokenSecret.class::cast) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly("tok", Optional.of(instant)); + soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("blah", "foo"))).isNull(); + soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of())).isNull(); + } + + @Test + public void basicCredential() { + soft.assertThat(SecretType.BASIC.parse("tok")).isNull(); + soft.assertThat(SecretType.BASIC.parse("{\"name\": \"user\", \"secret\": \"pass\"}")) + .extracting(BasicCredentials.class::cast) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly("user", "pass"); + soft.assertThat(SecretType.BASIC.parse("{\"blah\": \"foo\"}")).isNull(); + soft.assertThat(SecretType.BASIC.parse("{}")).isNull(); + + soft.assertThat(SecretType.BASIC.fromValueMap(Map.of("name", "user", "secret", "pass"))) + .extracting(BasicCredentials.class::cast) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly("user", "pass"); + soft.assertThat(SecretType.BASIC.fromValueMap(Map.of("name", "user"))).isNull(); + soft.assertThat(SecretType.BASIC.fromValueMap(Map.of("blah", "foo"))).isNull(); + soft.assertThat(SecretType.BASIC.fromValueMap(Map.of())).isNull(); + } +} diff --git a/catalog/secrets/aws/build.gradle.kts b/catalog/secrets/aws/build.gradle.kts new file mode 100644 index 00000000000..1f1f92eddb4 --- /dev/null +++ b/catalog/secrets/aws/build.gradle.kts @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id("nessie-conventions-server") } + +extra["maven.name"] = "Nessie - Catalog - Secrets AWS" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:apache-client") { + exclude("commons-logging", "commons-logging") + } + implementation("software.amazon.awssdk:secretsmanager") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) + + intTestCompileOnly(project(":nessie-immutables")) + intTestImplementation(platform(libs.testcontainers.bom)) + intTestImplementation("org.testcontainers:testcontainers") + intTestImplementation("org.testcontainers:localstack") + intTestImplementation("org.testcontainers:junit-jupiter") + intTestImplementation(project(":nessie-container-spec-helper")) + intTestRuntimeOnly(libs.logback.classic) +} diff --git a/catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsProvider.java b/catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsProvider.java new file mode 100644 index 00000000000..753cfdd1e93 --- /dev/null +++ b/catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsProvider.java @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.aws; + +import static org.assertj.core.api.InstanceOfAssertFactories.type; +import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; +import static org.projectnessie.catalog.secrets.KeySecret.keySecret; +import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; +import static org.testcontainers.containers.localstack.LocalStackContainer.Service.SECRETSMANAGER; + +import java.net.URI; +import java.time.Duration; +import java.time.Instant; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.projectnessie.catalog.secrets.BasicCredentials; +import org.projectnessie.catalog.secrets.KeySecret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.TokenSecret; +import org.projectnessie.nessie.testing.containerspec.ContainerSpecHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.CreateSecretRequest; + +@Testcontainers +@ExtendWith(SoftAssertionsExtension.class) +public class ITAwsSecretsProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(ITAwsSecretsProvider.class); + + @InjectSoftAssertions SoftAssertions soft; + + @Container + static LocalStackContainer localstack = + new LocalStackContainer( + ContainerSpecHelper.builder() + .name("localstack") + .containerClass(ITAwsSecretsProvider.class) + .build() + .dockerImageName(null) + .asCompatibleSubstituteFor("localstack/localstack")) + .withLogConsumer(c -> LOGGER.info("[LOCALSTACK] {}", c.getUtf8StringWithoutLineEnding())) + .withServices(SECRETSMANAGER); + + @Test + public void awsSecretsManager() { + URI secretsManagerEndpoint = localstack.getEndpointOverride(SECRETSMANAGER); + try (SecretsManagerClient client = + SecretsManagerClient.builder() + .endpointOverride(secretsManagerEndpoint) + .region(Region.of(localstack.getRegion())) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create( + localstack.getAccessKey(), localstack.getSecretKey()))) + .build()) { + + String instantStr = "2024-06-05T20:38:16Z"; + Instant instant = Instant.parse(instantStr); + + KeySecret keySecret = keySecret("secret-foo"); + BasicCredentials basicCred = basicCredentials("bar-name", "bar-secret"); + TokenSecret tokenSec = tokenSecret("the-token", instant); + + client.createSecret( + CreateSecretRequest.builder().name("key").secretString("secret-foo").build()); + client.createSecret( + CreateSecretRequest.builder() + .name("basic") + .secretString("{\"name\": \"bar-name\", \"secret\": \"bar-secret\"}") + .build()); + client.createSecret( + CreateSecretRequest.builder() + .name("tok") + .secretString("{\"token\": \"the-token\", \"expiresAt\": \"" + instantStr + "\"}") + .build()); + + AwsSecretsProvider secretsProvider = + new AwsSecretsProvider(client, "", Duration.ofMinutes(1)); + + soft.assertThat(secretsProvider.getSecret("key", SecretType.KEY, KeySecret.class)) + .get() + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key()); + soft.assertThat(secretsProvider.getSecret("basic", SecretType.BASIC, BasicCredentials.class)) + .get() + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret()); + soft.assertThat( + secretsProvider.getSecret("tok", SecretType.EXPIRING_TOKEN, TokenSecret.class)) + .get() + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt()); + + soft.assertThat(secretsProvider.getSecret("not-there", SecretType.KEY, KeySecret.class)) + .isEmpty(); + soft.assertThat(secretsProvider.getSecret("nope", SecretType.BASIC, BasicCredentials.class)) + .isEmpty(); + } + } +} diff --git a/catalog/secrets/aws/src/intTest/resources/logback-test.xml b/catalog/secrets/aws/src/intTest/resources/logback-test.xml new file mode 100644 index 00000000000..ce140f00774 --- /dev/null +++ b/catalog/secrets/aws/src/intTest/resources/logback-test.xml @@ -0,0 +1,30 @@ + + + + + + + %date{ISO8601} [%thread] [%X{nessie.events.subscription.id}] %-5level %logger{36} - + %msg%n + + + + + + diff --git a/catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version b/catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version new file mode 100644 index 00000000000..5e5015a43ea --- /dev/null +++ b/catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version @@ -0,0 +1,3 @@ +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM docker.io/localstack/localstack:3.5.0 diff --git a/catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsProvider.java b/catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsProvider.java new file mode 100644 index 00000000000..08589c36de9 --- /dev/null +++ b/catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsProvider.java @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.aws; + +import java.time.Duration; +import org.projectnessie.catalog.secrets.AbstractStringBasedSecretsProvider; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.ResourceNotFoundException; + +public class AwsSecretsProvider extends AbstractStringBasedSecretsProvider { + private final SecretsManagerClient secretsManagerClient; + private final String prefix; + + public AwsSecretsProvider( + SecretsManagerClient secretsManagerClient, String prefix, Duration getSecretTimeout) { + this.secretsManagerClient = secretsManagerClient; + this.prefix = prefix; + } + + @Override + protected String resolveSecretString(String name) { + String secretId = nameToSecretId(name); + + try { + return secretsManagerClient + .getSecretValue(GetSecretValueRequest.builder().secretId(secretId).build()) + .secretString(); + } catch (ResourceNotFoundException nf) { + return null; + } + } + + private String nameToSecretId(String name) { + return prefix + name; + } +} diff --git a/catalog/secrets/azure/build.gradle.kts b/catalog/secrets/azure/build.gradle.kts new file mode 100644 index 00000000000..acc0490fd25 --- /dev/null +++ b/catalog/secrets/azure/build.gradle.kts @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id("nessie-conventions-server") } + +extra["maven.name"] = "Nessie - Catalog - Secrets Azure" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(enforcedPlatform(libs.quarkus.azure.services.bom)) + implementation("com.azure:azure-security-keyvault-secrets") + implementation("com.azure:azure-identity") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) + + intTestCompileOnly(project(":nessie-immutables")) + intTestImplementation(platform(libs.testcontainers.bom)) + intTestImplementation("org.testcontainers:testcontainers") + intTestImplementation("org.testcontainers:junit-jupiter") + intTestImplementation(libs.lowkey.vault.testcontainers) + intTestImplementation(project(":nessie-container-spec-helper")) + intTestRuntimeOnly(libs.logback.classic) +} + +tasks.named("intTest") { systemProperty("javax.net.ssl.trustStore", "foo.bar.BazTrustStore") } diff --git a/catalog/secrets/azure/src/intTest/java/org/projectnessie/catalog/secrets/azure/ITAzureSecretsProvider.java b/catalog/secrets/azure/src/intTest/java/org/projectnessie/catalog/secrets/azure/ITAzureSecretsProvider.java new file mode 100644 index 00000000000..5615ac76c3e --- /dev/null +++ b/catalog/secrets/azure/src/intTest/java/org/projectnessie/catalog/secrets/azure/ITAzureSecretsProvider.java @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.azure; + +import static java.lang.String.format; +import static org.assertj.core.api.InstanceOfAssertFactories.type; +import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; +import static org.projectnessie.catalog.secrets.KeySecret.keySecret; +import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; + +import com.azure.identity.UsernamePasswordCredentialBuilder; +import com.azure.security.keyvault.secrets.SecretAsyncClient; +import com.azure.security.keyvault.secrets.SecretClientBuilder; +import com.github.nagyesta.lowkeyvault.testcontainers.LowkeyVaultContainer; +import com.github.nagyesta.lowkeyvault.testcontainers.LowkeyVaultContainerBuilder; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Set; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.projectnessie.catalog.secrets.BasicCredentials; +import org.projectnessie.catalog.secrets.KeySecret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.TokenSecret; +import org.projectnessie.nessie.testing.containerspec.ContainerSpecHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +@ExtendWith(SoftAssertionsExtension.class) +@Disabled( + "Azure SecretClient requires an SSL connection, verifying the server certificates, which needs to used by the test container and trusted by the client. No way around it.") +public class ITAzureSecretsProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(ITAzureSecretsProvider.class); + + @InjectSoftAssertions SoftAssertions soft; + + @Container + static LowkeyVaultContainer lowkeyVault = + LowkeyVaultContainerBuilder.lowkeyVault( + ContainerSpecHelper.builder() + .name("lowkey-vault") + .containerClass(ITAzureSecretsProvider.class) + .build() + .dockerImageName(null) + .asCompatibleSubstituteFor("nagyesta/lowkey-vault")) + .vaultNames(Set.of("default")) + .build() + .withLogConsumer( + c -> LOGGER.info("[LOWKEY-VAULT] {}", c.getUtf8StringWithoutLineEnding())); + + @Test + public void azureSecrets() { + SecretAsyncClient client = + new SecretClientBuilder() + .vaultUrl(format("https://%s.localhost:%d", "default", lowkeyVault.getMappedPort(8443))) + .credential( + new UsernamePasswordCredentialBuilder() + .clientId("ITAzureSecretsSupplier") + .username(lowkeyVault.getUsername()) + .password(lowkeyVault.getPassword()) + .build()) + .buildAsyncClient(); + + String instantStr = "2024-06-05T20:38:16Z"; + Instant instant = Instant.parse(instantStr); + + KeySecret keySecret = keySecret("secret-foo"); + BasicCredentials basicCred = basicCredentials("bar-name", "bar-secret"); + TokenSecret tokenSec = tokenSecret("the-token", instant); + + client.setSecret("key", "secret-foo").block(Duration.of(1, ChronoUnit.MINUTES)); + client + .setSecret("basic", "{\"name\": \"bar-name\", \"secret\": \"bar-secret\"}") + .block(Duration.of(1, ChronoUnit.MINUTES)); + client + .setSecret("tok", "{\"token\": \"the-token\", \"expiresAt\": \"" + instantStr + "\"}") + .block(Duration.of(1, ChronoUnit.MINUTES)); + + AzureSecretsProvider secretsProvider = + new AzureSecretsProvider(client, "", Duration.ofMinutes(1)); + + soft.assertThat(secretsProvider.getSecret("key", SecretType.KEY, KeySecret.class)) + .get() + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key()); + soft.assertThat(secretsProvider.getSecret("basic", SecretType.BASIC, BasicCredentials.class)) + .get() + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret()); + soft.assertThat(secretsProvider.getSecret("tok", SecretType.EXPIRING_TOKEN, TokenSecret.class)) + .get() + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt()); + + soft.assertThat(secretsProvider.getSecret("not-there", SecretType.KEY, KeySecret.class)) + .isEmpty(); + soft.assertThat(secretsProvider.getSecret("nope", SecretType.BASIC, BasicCredentials.class)) + .isEmpty(); + } +} diff --git a/catalog/secrets/azure/src/intTest/resources/logback-test.xml b/catalog/secrets/azure/src/intTest/resources/logback-test.xml new file mode 100644 index 00000000000..ce140f00774 --- /dev/null +++ b/catalog/secrets/azure/src/intTest/resources/logback-test.xml @@ -0,0 +1,30 @@ + + + + + + + %date{ISO8601} [%thread] [%X{nessie.events.subscription.id}] %-5level %logger{36} - + %msg%n + + + + + + diff --git a/catalog/secrets/azure/src/intTest/resources/org/projectnessie/catalog/secrets/azure/Dockerfile-lowkey-vault-version b/catalog/secrets/azure/src/intTest/resources/org/projectnessie/catalog/secrets/azure/Dockerfile-lowkey-vault-version new file mode 100644 index 00000000000..8907a241ad1 --- /dev/null +++ b/catalog/secrets/azure/src/intTest/resources/org/projectnessie/catalog/secrets/azure/Dockerfile-lowkey-vault-version @@ -0,0 +1,3 @@ +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM docker.io/nagyesta/lowkey-vault:2.4.42 diff --git a/catalog/secrets/azure/src/main/java/org/projectnessie/catalog/secrets/azure/AzureSecretsProvider.java b/catalog/secrets/azure/src/main/java/org/projectnessie/catalog/secrets/azure/AzureSecretsProvider.java new file mode 100644 index 00000000000..853c02b793f --- /dev/null +++ b/catalog/secrets/azure/src/main/java/org/projectnessie/catalog/secrets/azure/AzureSecretsProvider.java @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.azure; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import com.azure.core.exception.ResourceNotFoundException; +import com.azure.security.keyvault.secrets.SecretAsyncClient; +import com.azure.security.keyvault.secrets.models.KeyVaultSecret; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import org.projectnessie.catalog.secrets.AbstractStringBasedSecretsProvider; + +public class AzureSecretsProvider extends AbstractStringBasedSecretsProvider { + private final SecretAsyncClient client; + private final String prefix; + private final long timeout; + + public AzureSecretsProvider(SecretAsyncClient client, String prefix, Duration getSecretTimeout) { + this.client = client; + this.prefix = prefix; + this.timeout = getSecretTimeout.toMillis(); + } + + @Override + protected String resolveSecretString(String name) { + String secretId = nameToSecretId(name); + + CompletableFuture future = + client + .getSecret(secretId) + .toFuture() + .exceptionally( + t -> { + if (t instanceof ResourceNotFoundException) { + return null; + } + if (t instanceof RuntimeException) { + throw (RuntimeException) t; + } + throw new RuntimeException(t); + }); + + try { + return future.get(timeout, MILLISECONDS).getValue(); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new RuntimeException(e); + } + } + + private String nameToSecretId(String name) { + return prefix + name; + } +} diff --git a/catalog/secrets/cache/build.gradle.kts b/catalog/secrets/cache/build.gradle.kts new file mode 100644 index 00000000000..3e2ffca2fb0 --- /dev/null +++ b/catalog/secrets/cache/build.gradle.kts @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id("nessie-conventions-server") } + +extra["maven.name"] = "Nessie - Catalog - Secrets Cache" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + implementation(libs.caffeine) + implementation(libs.micrometer.core) + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-databind") + implementation("com.fasterxml.jackson.core:jackson-annotations") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) +} diff --git a/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java new file mode 100644 index 00000000000..0ca44985aed --- /dev/null +++ b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import jakarta.annotation.Nonnull; +import java.util.Optional; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.SecretsProvider; + +public final class CachingSecrets { + + private final CachingSecretsBackend backend; + + public CachingSecrets(CachingSecretsBackend backend) { + this.backend = backend; + } + + public SecretsProvider forRepository(String repositoryId, SecretsProvider secretsSupplier) { + return new SecretsProvider() { + @Override + public Optional getSecret( + @Nonnull String name, @Nonnull SecretType secretType, @Nonnull Class secretJavaType) { + return backend.resolveSecret( + repositoryId, secretsSupplier, name, secretType, secretJavaType); + } + }; + } +} diff --git a/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecretsBackend.java b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecretsBackend.java new file mode 100644 index 00000000000..8da641bb20b --- /dev/null +++ b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecretsBackend.java @@ -0,0 +1,213 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import static java.util.Collections.singletonList; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Expiry; +import com.google.common.annotations.VisibleForTesting; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.binder.cache.CaffeineStatsCounter; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; +import org.checkerframework.checker.index.qual.NonNegative; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.SecretsProvider; + +public class CachingSecretsBackend { + + private static final Secret CACHE_NEGATIVE_SENTINEL = Map::of; + + public static final String CACHE_NAME = "nessie-secrets"; + private static final long NOT_CACHED = 0L; + + @VisibleForTesting final Cache cache; + private final long ttlNanos; + private final LongSupplier clock; + + public CachingSecretsBackend(SecretsCacheConfig config) { + OptionalLong ttl = config.ttlMillis(); + this.ttlNanos = ttl.isPresent() ? TimeUnit.MILLISECONDS.toNanos(ttl.getAsLong()) : 0L; + this.clock = config.clockNanos(); + + Caffeine cacheBuilder = + Caffeine.newBuilder() + .expireAfter( + new Expiry() { + @Override + public long expireAfterCreate( + CacheKeyValue key, Secret value, long currentTimeNanos) { + long expire = key.expiresAtNanosEpoch; + if (expire == NOT_CACHED) { + return 0L; + } + long remaining = expire - currentTimeNanos; + return Math.max(0L, remaining); + } + + @Override + public long expireAfterUpdate( + CacheKeyValue key, + Secret value, + long currentTimeNanos, + @NonNegative long currentDurationNanos) { + return expireAfterCreate(key, value, currentTimeNanos); + } + + @Override + public long expireAfterRead( + CacheKeyValue key, + Secret value, + long currentTimeNanos, + @NonNegative long currentDurationNanos) { + return currentDurationNanos; + } + }) + .ticker(clock::getAsLong) + .maximumSize(config.maxElements()); + config + .meterRegistry() + .ifPresent( + meterRegistry -> { + cacheBuilder.recordStats(() -> new CaffeineStatsCounter(meterRegistry, CACHE_NAME)); + meterRegistry.gauge( + "cache_max_size", + singletonList(Tag.of("cache", CACHE_NAME)), + "", + x -> config.maxElements()); + meterRegistry.gauge( + "cache_element_ttl", + singletonList(Tag.of("cache", CACHE_NAME)), + "", + x -> config.ttlMillis().orElse(0)); + }); + + this.cache = cacheBuilder.build(); + } + + Optional resolveSecret( + String repositoryId, + SecretsProvider backend, + String name, + SecretType secretType, + Class secretJavaType) { + long ttl = ttlNanos; + long expires = ttl != 0L ? clock.getAsLong() + ttl : 0L; + + CacheKeyValue key = new CacheKeyValue(repositoryId, name, expires); + + Secret fromCache = + cache.get( + key, + k -> { + @SuppressWarnings("unchecked") + Optional loaded = + (Optional) backend.getSecret(name, secretType, secretJavaType); + return loaded.orElse(CACHE_NEGATIVE_SENTINEL); + }); + if (fromCache == CACHE_NEGATIVE_SENTINEL) { + return Optional.empty(); + } + @SuppressWarnings("unchecked") + S casted = (S) fromCache; + return Optional.of(casted); + } + + static final class CacheKeyValue { + final String repositoryId; + final String name; + + // Revisit this field before 2262-04-11T23:47:16.854Z (64-bit signed long overflow) ;) ;) + final long expiresAtNanosEpoch; + + CacheKeyValue(String repositoryId, String name, long expiresAtNanosEpoch) { + this.repositoryId = repositoryId; + this.name = name; + this.expiresAtNanosEpoch = expiresAtNanosEpoch; + } + + int heapSize() { + int size = OBJ_SIZE; + size += STRING_OBJ_OVERHEAD + repositoryId.length(); + size += STRING_OBJ_OVERHEAD + name.length(); + return size; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CacheKeyValue)) { + return false; + } + CacheKeyValue cacheKey = (CacheKeyValue) o; + return repositoryId.equals(cacheKey.repositoryId) && name.equals(cacheKey.name); + } + + @Override + public int hashCode() { + return repositoryId.hashCode() * 31 + name.hashCode(); + } + + @Override + public String toString() { + return "{" + repositoryId + ", " + name + '}'; + } + } + + /* + CacheKeyValue object internals: + OFF SZ TYPE DESCRIPTION VALUE + 0 8 (object header: mark) 0x0000000000000001 (non-biasable; age: 0) + 8 4 (object header: class) 0x010c4800 + 12 4 java.lang.String CacheKeyValue.repositoryId null + 16 8 long CacheKeyValue.expiresAt 0 + 24 4 java.lang.String CacheKeyValue.id null + 28 4 (object alignment gap) + Instance size: 32 bytes + */ + static final int OBJ_SIZE = 32; + // rough estimate, probably good enough + static final int MAP_OBJ_OVERHEAD = OBJ_SIZE * 2; + /* + Array overhead: 16 bytes + */ + static final int ARRAY_OVERHEAD = 16; + /* + java.lang.String object internals: + OFF SZ TYPE DESCRIPTION VALUE + 0 8 (object header: mark) 0x0000000000000001 (non-biasable; age: 0) + 8 4 (object header: class) 0x0000e8d8 + 12 4 int String.hash 0 + 16 1 byte String.coder 0 + 17 1 boolean String.hashIsZero false + 18 2 (alignment/padding gap) + 20 4 byte[] String.value [] + Instance size: 24 bytes + */ + static final int STRING_OBJ_OVERHEAD = 24 + ARRAY_OVERHEAD; + /* + Assume an overhead of 2 objects for each entry (java.util.concurrent.ConcurrentHashMap$Node is 32 bytes) in Caffeine. + */ + static final int CAFFEINE_OBJ_OVERHEAD = 2 * 32; +} diff --git a/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/SecretsCacheConfig.java b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/SecretsCacheConfig.java new file mode 100644 index 00000000000..e004e9b7655 --- /dev/null +++ b/catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/SecretsCacheConfig.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import io.micrometer.core.instrument.MeterRegistry; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.function.LongSupplier; +import org.immutables.value.Value; + +@Value.Immutable +public interface SecretsCacheConfig { + + long maxElements(); + + OptionalLong ttlMillis(); + + Optional meterRegistry(); + + @Value.Default + default LongSupplier clockNanos() { + return System::nanoTime; + } + + static Builder builder() { + return ImmutableSecretsCacheConfig.builder(); + } + + interface Builder { + @CanIgnoreReturnValue + Builder maxElements(long maxElements); + + @CanIgnoreReturnValue + Builder ttlMillis(long ttlMillis); + + @CanIgnoreReturnValue + Builder meterRegistry(MeterRegistry meterRegistry); + + @CanIgnoreReturnValue + Builder clockNanos(LongSupplier clockNanos); + + SecretsCacheConfig build(); + } +} diff --git a/catalog/secrets/cache/src/test/java/org/projectnessie/catalog/secrets/cache/TestCachingSecretsBackend.java b/catalog/secrets/cache/src/test/java/org/projectnessie/catalog/secrets/cache/TestCachingSecretsBackend.java new file mode 100644 index 00000000000..92c89ab443a --- /dev/null +++ b/catalog/secrets/cache/src/test/java/org/projectnessie/catalog/secrets/cache/TestCachingSecretsBackend.java @@ -0,0 +1,177 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.cache; + +import static org.assertj.core.api.InstanceOfAssertFactories.map; +import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; +import static org.projectnessie.catalog.secrets.KeySecret.keySecret; +import static org.projectnessie.catalog.secrets.SecretType.BASIC; +import static org.projectnessie.catalog.secrets.SecretType.KEY; +import static org.projectnessie.catalog.secrets.UnsafePlainTextSecretsProvider.unsafePlainTextSecretsProvider; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.projectnessie.catalog.secrets.BasicCredentials; +import org.projectnessie.catalog.secrets.KeySecret; +import org.projectnessie.catalog.secrets.Secret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.SecretsProvider; + +@ExtendWith(SoftAssertionsExtension.class) +public class TestCachingSecretsBackend { + @InjectSoftAssertions protected SoftAssertions soft; + + static final Secret SECRET_1_REPO_1 = keySecret("secret1-repo1"); + static final Secret SECRET_2_REPO_1 = basicCredentials("secret1-name", "secret1-value"); + static final Secret SECRET_1_REPO_2 = keySecret("secret1-repo2"); + static final Secret SECRET_2_REPO_2 = basicCredentials("secret2-name", "secret2-value"); + + AtomicLong clock; + CachingSecretsBackend backend; + CachingSecrets cachingSecrets; + SecretsProvider secrets1; + SecretsProvider secrets2; + SecretsProvider caching1; + SecretsProvider caching2; + + @BeforeEach + void setup() { + clock = new AtomicLong(); + backend = + new CachingSecretsBackend( + SecretsCacheConfig.builder() + .maxElements(100) + .ttlMillis(1000) + .clockNanos(clock::get) + .build()); + + cachingSecrets = new CachingSecrets(backend); + + secrets1 = + unsafePlainTextSecretsProvider( + Map.of("secret1", SECRET_1_REPO_1.asMap(), "secret2", SECRET_2_REPO_1.asMap())); + secrets2 = + unsafePlainTextSecretsProvider( + Map.of("secret1", SECRET_1_REPO_2.asMap(), "secret2", SECRET_2_REPO_2.asMap())); + + caching1 = cachingSecrets.forRepository("repo1", secrets1); + caching2 = cachingSecrets.forRepository("repo2", secrets2); + } + + @Test + public void ttlExpire() { + + soft.assertThat(caching1.getSecret("secret1", KEY, KeySecret.class)) + .get() + .extracting(Secret::asMap, map(String.class, String.class)) + .containsExactlyInAnyOrderEntriesOf(SECRET_1_REPO_1.asMap()); + soft.assertThat(caching2.getSecret("secret1", KEY, KeySecret.class)) + .get() + .extracting(Secret::asMap, map(String.class, String.class)) + .containsExactlyInAnyOrderEntriesOf(SECRET_1_REPO_2.asMap()); + soft.assertThat(caching1.getSecret("secret2", BASIC, BasicCredentials.class)) + .get() + .extracting(Secret::asMap, map(String.class, String.class)) + .containsExactlyInAnyOrderEntriesOf(SECRET_2_REPO_1.asMap()); + soft.assertThat(caching2.getSecret("secret2", BASIC, BasicCredentials.class)) + .get() + .extracting(Secret::asMap, map(String.class, String.class)) + .containsExactlyInAnyOrderEntriesOf(SECRET_2_REPO_2.asMap()); + + soft.assertThat(backend.cache.asMap()) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret1", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret1", 0)); + + clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(1000)); + + soft.assertThat(backend.cache.asMap()) + .doesNotContainKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret1", 0)) + .doesNotContainKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret1", 0)); + + soft.assertThat(caching1.getSecret("secret1", KEY, KeySecret.class)) + .get() + .extracting(Secret::asMap, map(String.class, String.class)) + .containsExactlyInAnyOrderEntriesOf(SECRET_1_REPO_1.asMap()); + soft.assertThat(caching2.getSecret("secret1", KEY, KeySecret.class)) + .get() + .extracting(Secret::asMap, map(String.class, String.class)) + .containsExactlyInAnyOrderEntriesOf(SECRET_1_REPO_2.asMap()); + soft.assertThat(caching1.getSecret("secret2", BASIC, BasicCredentials.class)) + .get() + .extracting(Secret::asMap, map(String.class, String.class)) + .containsExactlyInAnyOrderEntriesOf(SECRET_2_REPO_1.asMap()); + soft.assertThat(caching2.getSecret("secret2", BASIC, BasicCredentials.class)) + .get() + .extracting(Secret::asMap, map(String.class, String.class)) + .containsExactlyInAnyOrderEntriesOf(SECRET_2_REPO_2.asMap()); + + soft.assertThat(backend.cache.asMap()) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret1", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret1", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo1", "secret2", 0)) + .containsKey(new CachingSecretsBackend.CacheKeyValue("repo2", "secret2", 0)); + } + + @Test + public void nonExisting() { + soft.assertThat(caching1.getSecret("secret-nope1", KEY, KeySecret.class)).isEmpty(); + soft.assertThat(caching1.getSecret("secret-nope2", KEY, KeySecret.class)).isEmpty(); + soft.assertThat(caching2.getSecret("secret-nope1", KEY, KeySecret.class)).isEmpty(); + soft.assertThat(caching2.getSecret("secret-nope2", KEY, KeySecret.class)).isEmpty(); + soft.assertThat(caching1.getSecret("secret1", KEY, KeySecret.class)) + .get() + .extracting(Secret::asMap, map(String.class, String.class)) + .containsExactlyInAnyOrderEntriesOf(SECRET_1_REPO_1.asMap()); + soft.assertThat(caching2.getSecret("secret1", KEY, KeySecret.class)) + .get() + .extracting(Secret::asMap, map(String.class, String.class)) + .containsExactlyInAnyOrderEntriesOf(SECRET_1_REPO_2.asMap()); + } + + @Test + public void tooManyEntries() { + SecretsProvider supplier = + new SecretsProvider() { + @Override + public Optional getSecret( + String name, SecretType secretType, Class secretJavaType) { + return Optional.of((S) keySecret(name)); + } + }; + SecretsProvider caching = cachingSecrets.forRepository("repoMany", supplier); + + // Very naive test that checks that expiration happens - our "weigher" does something. + + for (int i = 0; i < 100_000; i++) { + caching.getSecret( + "wepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoewwepofkjeopwkfopwekfopkewfkpoew-" + + i, + KEY, + KeySecret.class); + // In fact, the cache should never have more than 10000 entries, but 'estimatedSize' is really + // just an estimate. + soft.assertThat(backend.cache.estimatedSize()).isLessThan(10_000); + } + } +} diff --git a/catalog/secrets/gcs/build.gradle.kts b/catalog/secrets/gcs/build.gradle.kts new file mode 100644 index 00000000000..b6438337183 --- /dev/null +++ b/catalog/secrets/gcs/build.gradle.kts @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id("nessie-conventions-server") } + +extra["maven.name"] = "Nessie - Catalog - Secrets GCS" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(platform(libs.google.cloud.secretmanager.bom)) + implementation("com.google.cloud:google-cloud-secretmanager") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) +} diff --git a/catalog/secrets/gcs/src/main/java/org/projectnessie/catalog/secrets/gcs/GcsSecretsProvider.java b/catalog/secrets/gcs/src/main/java/org/projectnessie/catalog/secrets/gcs/GcsSecretsProvider.java new file mode 100644 index 00000000000..b0d84ec0cee --- /dev/null +++ b/catalog/secrets/gcs/src/main/java/org/projectnessie/catalog/secrets/gcs/GcsSecretsProvider.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.gcs; + +import static com.google.api.gax.rpc.StatusCode.Code.NOT_FOUND; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import com.google.api.core.ApiFuture; +import com.google.api.gax.rpc.ApiException; +import com.google.api.gax.rpc.StatusCode; +import com.google.cloud.secretmanager.v1.AccessSecretVersionRequest; +import com.google.cloud.secretmanager.v1.AccessSecretVersionResponse; +import com.google.cloud.secretmanager.v1.SecretManagerServiceClient; +import java.time.Duration; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import org.projectnessie.catalog.secrets.AbstractStringBasedSecretsProvider; + +public class GcsSecretsProvider extends AbstractStringBasedSecretsProvider { + private final String prefix; + private final SecretManagerServiceClient client; + private final long getSecretTimeout; + + public GcsSecretsProvider( + SecretManagerServiceClient client, String prefix, Duration getSecretTimeout) { + this.client = client; + this.prefix = prefix; + this.getSecretTimeout = getSecretTimeout.toMillis(); + } + + @Override + protected String resolveSecretString(String name) { + String secretId = nameToSecretId(name); + + ApiFuture responseCallable = + client.accessSecretVersionCallable().futureCall(request(secretId)); + try { + AccessSecretVersionResponse response = responseCallable.get(getSecretTimeout, MILLISECONDS); + return response.hasPayload() ? secret(response) : null; + } catch (ExecutionException e) { + Throwable c = e.getCause(); + if (c instanceof ApiException) { + ApiException apiException = (ApiException) c; + StatusCode status = apiException.getStatusCode(); + if (status.getCode() == NOT_FOUND) { + return null; + } + } + // Let all other API exceptions bubble up. + throw new RuntimeException(c); + } catch (InterruptedException | TimeoutException e) { + throw new RuntimeException(e); + } + } + + private String secret(AccessSecretVersionResponse response) { + return response.getPayload().getData().toStringUtf8(); + } + + private AccessSecretVersionRequest request(String name) { + return AccessSecretVersionRequest.newBuilder().setName(name).build(); + } + + private String nameToSecretId(String name) { + return prefix + name; + } +} diff --git a/catalog/secrets/vault/build.gradle.kts b/catalog/secrets/vault/build.gradle.kts new file mode 100644 index 00000000000..ecbd9bc5b81 --- /dev/null +++ b/catalog/secrets/vault/build.gradle.kts @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id("nessie-conventions-server") } + +extra["maven.name"] = "Nessie - Catalog - Secrets Vault" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + + implementation(enforcedPlatform(libs.quarkus.bom)) + implementation("io.quarkiverse.vault:quarkus-vault") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) + + intTestImplementation(platform(libs.testcontainers.bom)) + intTestImplementation("org.testcontainers:testcontainers") + intTestImplementation("org.testcontainers:vault") + intTestImplementation("org.testcontainers:junit-jupiter") + intTestImplementation(project(":nessie-container-spec-helper")) + intTestImplementation(libs.smallrye.config.core) + intTestCompileOnly(libs.immutables.value.annotations) + intTestRuntimeOnly(libs.logback.classic) +} diff --git a/catalog/secrets/vault/src/intTest/java/org/projectnessie/catalog/secrets/vault/ITVaultSecretsProvider.java b/catalog/secrets/vault/src/intTest/java/org/projectnessie/catalog/secrets/vault/ITVaultSecretsProvider.java new file mode 100644 index 00000000000..d9f27320e47 --- /dev/null +++ b/catalog/secrets/vault/src/intTest/java/org/projectnessie/catalog/secrets/vault/ITVaultSecretsProvider.java @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.vault; + +import static org.assertj.core.api.InstanceOfAssertFactories.type; +import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials; +import static org.projectnessie.catalog.secrets.KeySecret.keySecret; +import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret; + +import io.quarkus.vault.VaultKVSecretReactiveEngine; +import io.quarkus.vault.client.VaultClient; +import io.quarkus.vault.runtime.VaultConfigHolder; +import io.quarkus.vault.runtime.VaultKvManager; +import io.quarkus.vault.runtime.client.VaultClientProducer; +import io.quarkus.vault.runtime.config.VaultRuntimeConfig; +import io.smallrye.config.PropertiesConfigSource; +import io.smallrye.config.SmallRyeConfig; +import io.smallrye.config.SmallRyeConfigBuilder; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.projectnessie.catalog.secrets.BasicCredentials; +import org.projectnessie.catalog.secrets.KeySecret; +import org.projectnessie.catalog.secrets.SecretType; +import org.projectnessie.catalog.secrets.TokenSecret; +import org.projectnessie.nessie.testing.containerspec.ContainerSpecHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.vault.VaultContainer; + +@Testcontainers +@ExtendWith(SoftAssertionsExtension.class) +public class ITVaultSecretsProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(ITVaultSecretsProvider.class); + + public static final String VAULT_ROOT_TOKEN = "root"; + public static final String NESSIE_SECRETS_PATH = "apps/nessie/secrets"; + + @InjectSoftAssertions SoftAssertions soft; + + @SuppressWarnings("resource") + @Container + static VaultContainer vaultContainer = + new VaultContainer<>( + ContainerSpecHelper.builder() + .name("vault") + .containerClass(ITVaultSecretsProvider.class) + .build() + .dockerImageName(null) + .asCompatibleSubstituteFor("vault")) + .withLogConsumer(c -> LOGGER.info("[VAULT] {}", c.getUtf8StringWithoutLineEnding())) + .withVaultToken(VAULT_ROOT_TOKEN); + + @Test + public void vault() { + SmallRyeConfig config = + new SmallRyeConfigBuilder() + .setAddDefaultSources(false) + .setAddDiscoveredSources(false) + .withMapping(VaultRuntimeConfig.class) + .withSources( + new PropertiesConfigSource( + Map.of( + "quarkus.vault.url", + vaultContainer.getHttpHostAddress(), + "quarkus.vault.authentication.client-token", + VAULT_ROOT_TOKEN, + "quarkus.vault.secret-config-kv-path", + NESSIE_SECRETS_PATH), + "configSource", + 100)) + .build(); + + VaultRuntimeConfig runtimeConfig = config.getConfigMapping(VaultRuntimeConfig.class); + + VaultConfigHolder configHolder = new VaultConfigHolder(); + configHolder.setVaultRuntimeConfig(runtimeConfig); + VaultClient client = new VaultClientProducer().privateVaultClient(configHolder, true); + + VaultKVSecretReactiveEngine vault = new VaultKvManager(client, configHolder); + + Instant instant = Instant.parse("2024-06-05T20:38:16Z"); + + KeySecret keySecret = keySecret("secret-foo"); + BasicCredentials basicCred = basicCredentials("bar-name", "bar-secret"); + TokenSecret tokenSec = tokenSecret("the-token", instant); + + vault.writeSecret("key", Map.of("key", keySecret.key())).await().indefinitely(); + vault + .writeSecret("basic", Map.of("name", basicCred.name(), "secret", basicCred.secret())) + .await() + .indefinitely(); + vault + .writeSecret( + "tok", + Map.of( + "token", + tokenSec.token(), + "expiresAt", + tokenSec.expiresAt().orElseThrow().toString())) + .await() + .indefinitely(); + + VaultSecretsProvider secretsProvider = + new VaultSecretsProvider(vault, "", Duration.ofMinutes(1)); + + soft.assertThat(secretsProvider.getSecret("key", SecretType.KEY, KeySecret.class)) + .get() + .asInstanceOf(type(KeySecret.class)) + .extracting(KeySecret::key) + .isEqualTo(keySecret.key()); + soft.assertThat(secretsProvider.getSecret("basic", SecretType.BASIC, BasicCredentials.class)) + .get() + .asInstanceOf(type(BasicCredentials.class)) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly(basicCred.name(), basicCred.secret()); + soft.assertThat(secretsProvider.getSecret("tok", SecretType.EXPIRING_TOKEN, TokenSecret.class)) + .get() + .asInstanceOf(type(TokenSecret.class)) + .extracting(TokenSecret::token, TokenSecret::expiresAt) + .containsExactly(tokenSec.token(), tokenSec.expiresAt()); + + soft.assertThat(secretsProvider.getSecret("not-there", SecretType.KEY, KeySecret.class)) + .isEmpty(); + soft.assertThat(secretsProvider.getSecret("nope", SecretType.BASIC, BasicCredentials.class)) + .isEmpty(); + } +} diff --git a/catalog/secrets/vault/src/intTest/resources/logback-test.xml b/catalog/secrets/vault/src/intTest/resources/logback-test.xml new file mode 100644 index 00000000000..4f3fda900d7 --- /dev/null +++ b/catalog/secrets/vault/src/intTest/resources/logback-test.xml @@ -0,0 +1,30 @@ + + + + + + + %date{ISO8601} [%thread] [%X{nessie.events.subscription.id}] %-5level %logger{36} - + %msg%n + + + + + + diff --git a/catalog/secrets/vault/src/intTest/resources/org/projectnessie/catalog/secrets/vault/Dockerfile-vault-version b/catalog/secrets/vault/src/intTest/resources/org/projectnessie/catalog/secrets/vault/Dockerfile-vault-version new file mode 100644 index 00000000000..f019422d362 --- /dev/null +++ b/catalog/secrets/vault/src/intTest/resources/org/projectnessie/catalog/secrets/vault/Dockerfile-vault-version @@ -0,0 +1,3 @@ +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM docker.io/hashicorp/vault:1.17.0 diff --git a/catalog/secrets/vault/src/main/java/org/projectnessie/catalog/secrets/vault/VaultSecretsProvider.java b/catalog/secrets/vault/src/main/java/org/projectnessie/catalog/secrets/vault/VaultSecretsProvider.java new file mode 100644 index 00000000000..cefe18cbd2d --- /dev/null +++ b/catalog/secrets/vault/src/main/java/org/projectnessie/catalog/secrets/vault/VaultSecretsProvider.java @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets.vault; + +import io.quarkus.vault.VaultKVSecretReactiveEngine; +import io.quarkus.vault.client.VaultClientException; +import jakarta.annotation.Nonnull; +import java.time.Duration; +import java.util.Map; +import org.projectnessie.catalog.secrets.AbstractMapBasedSecretsProvider; + +public class VaultSecretsProvider extends AbstractMapBasedSecretsProvider { + private final VaultKVSecretReactiveEngine secretEngine; + private final String prefix; + private final Duration getSecretTimeout; + + public VaultSecretsProvider( + VaultKVSecretReactiveEngine secretEngine, String prefix, Duration getSecretTimeout) { + this.secretEngine = secretEngine; + this.prefix = prefix; + this.getSecretTimeout = getSecretTimeout; + } + + @Override + public Map resolveSecret(@Nonnull String name) { + return secretEngine + .readSecret(path(name)) + .onFailure( + e -> { + if (!(e instanceof VaultClientException)) { + return false; + } + VaultClientException ex = (VaultClientException) e; + return 404 == ex.getStatus(); + }) + .recoverWithNull() + .await() + .atMost(getSecretTimeout); + } + + private String path(String name) { + return prefix + name.replace('.', '/'); + } +} diff --git a/catalog/service/impl/build.gradle.kts b/catalog/service/impl/build.gradle.kts index 6e73c80e25c..c6b83a9af72 100644 --- a/catalog/service/impl/build.gradle.kts +++ b/catalog/service/impl/build.gradle.kts @@ -72,6 +72,7 @@ dependencies { testFixturesApi(libs.threeten.extra) testFixturesApi(libs.jakarta.ws.rs.api) + testImplementation(testFixtures(project(":nessie-catalog-secrets-api"))) testImplementation(platform(libs.awssdk.bom)) testImplementation("software.amazon.awssdk:s3") testImplementation("software.amazon.awssdk:url-connection-client") diff --git a/cli/cli/build.gradle.kts b/cli/cli/build.gradle.kts index d89ffa92dd4..1e4a0819833 100644 --- a/cli/cli/build.gradle.kts +++ b/cli/cli/build.gradle.kts @@ -93,6 +93,7 @@ dependencies { intTestImplementation(project(":nessie-keycloak-testcontainer")) intTestImplementation(project(":nessie-container-spec-helper")) intTestImplementation(project(":nessie-catalog-secrets-api")) + intTestImplementation(testFixtures(project(":nessie-catalog-secrets-api"))) nessieQuarkusServer(project(":nessie-quarkus", "quarkusRunner")) } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 03c6a135985..c8abc2da734 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -62,6 +62,7 @@ gatling-charts-highcharts = { module = "io.gatling.highcharts:gatling-charts-hig google-cloud-bigdataoss-gcs-connector = { module = "com.google.cloud.bigdataoss:gcs-connector", version.ref = "googleCloudBigdataoss" } google-cloud-bigdataoss-gcsio = { module = "com.google.cloud.bigdataoss:gcsio", version.ref = "googleCloudBigdataoss" } google-cloud-bigtable-bom = { module = "com.google.cloud:google-cloud-bigtable-bom", version = "2.43.0" } +google-cloud-secretmanager-bom = { module = "com.google.cloud:google-cloud-secretmanager-bom", version = "2.48.0" } google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.42.0" } google-cloud-libraries-bom = { module = "com.google.cloud:libraries-bom", version = "26.44.0" } google-java-format = { module = "com.google.googlejavaformat:google-java-format", version.ref = "googleJavaFormat" } @@ -99,6 +100,7 @@ junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api" } junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params" } junit-platform-reporting = { module = "org.junit.platform:junit-platform-reporting" } logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logback" } +lowkey-vault-testcontainers = { module = "com.github.nagyesta.lowkey-vault:lowkey-vault-testcontainers", version = "2.4.66" } keycloak-admin-client = { module = "org.keycloak:keycloak-admin-client", version.ref = "keycloak" } mariadb-java-client = { module = "org.mariadb.jdbc:mariadb-java-client", version = "3.4.1" } maven-resolver-supplier = { module = "org.apache.maven.resolver:maven-resolver-supplier", version.ref = "mavenResolver" } @@ -120,6 +122,7 @@ picocli-codegen = { module = "info.picocli:picocli-codegen", version.ref = "pico postgresql = { module = "org.postgresql:postgresql", version = "42.7.4" } protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" } quarkus-amazon-services-bom = { module = "io.quarkus.platform:quarkus-amazon-services-bom", version.ref = "quarkusPlatform" } +quarkus-azure-services-bom = { module = "io.quarkiverse.azureservices:quarkus-azure-services-bom", version = "1.0.5" } quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quarkusPlatform" } quarkus-cassandra-bom = { module = "io.quarkus.platform:quarkus-cassandra-bom", version.ref = "quarkusPlatform" } quarkus-google-cloud-services-bom = { module = "io.quarkus.platform:quarkus-google-cloud-services-bom", version.ref = "quarkusPlatform" } diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 592f9564023..36e49fef11e 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -101,3 +101,8 @@ nessie-catalog-service-rest=catalog/service/rest nessie-catalog-service-impl=catalog/service/impl nessie-catalog-service-transfer=catalog/service/transfer nessie-catalog-secrets-api=catalog/secrets/api +nessie-catalog-secrets-cache=catalog/secrets/cache +nessie-catalog-secrets-aws=catalog/secrets/aws +nessie-catalog-secrets-gcs=catalog/secrets/gcs +nessie-catalog-secrets-azure=catalog/secrets/azure +nessie-catalog-secrets-vault=catalog/secrets/vault