Add SecretsSupplier implementations for AWS, GCP, Vault

bom/build.gradle.kts

@@ -103,6 +103,10 @@ dependencies {
     api(project(":nessie-catalog-service-rest"))
     api(project(":nessie-catalog-service-impl"))
     api(project(":nessie-catalog-secrets-api"))
+    api(project(":nessie-catalog-secrets-cache"))
+    api(project(":nessie-catalog-secrets-aws"))
+    api(project(":nessie-catalog-secrets-gcs"))
+    api(project(":nessie-catalog-secrets-vault"))
 
     if (!isIncludedInNesQuEIT()) {
       api(project(":nessie-spark-antlr-runtime"))

catalog/secrets/aws/build.gradle.kts

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins {
+  id("nessie-conventions-server")
+  id("nessie-jacoco")
+}
+
+extra["maven.name"] = "Nessie - Catalog - Secrets AWS"
+
+dependencies {
+  implementation(project(":nessie-catalog-secrets-api"))
+  implementation(libs.guava)
+
+  implementation(platform(libs.awssdk.bom))
+  implementation("software.amazon.awssdk:apache-client") {
+    exclude("commons-logging", "commons-logging")
+  }
+  implementation("software.amazon.awssdk:secretsmanager")
+
+  compileOnly(project(":nessie-immutables"))
+  annotationProcessor(project(":nessie-immutables", configuration = "processor"))
+  // javax/jakarta
+  compileOnly(libs.jakarta.ws.rs.api)
+  compileOnly(libs.jakarta.enterprise.cdi.api)
+  compileOnly(libs.jakarta.validation.api)
+
+  compileOnly(libs.errorprone.annotations)
+  compileOnly(libs.microprofile.openapi)
+
+  testFixturesApi(platform(libs.junit.bom))
+  testFixturesApi(libs.bundles.junit.testing)
+
+  intTestCompileOnly(project(":nessie-immutables"))
+  intTestImplementation(platform(libs.testcontainers.bom))
+  intTestImplementation("org.testcontainers:testcontainers")
+  intTestImplementation("org.testcontainers:localstack")
+  intTestImplementation("org.testcontainers:junit-jupiter")
+  intTestImplementation(project(":nessie-container-spec-helper"))
+  intTestRuntimeOnly(libs.logback.classic)
+}

catalog/secrets/aws/src/intTest/java/org/projectnessie/catalog/secrets/aws/ITAwsSecretsSupplier.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.catalog.secrets.aws;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.testcontainers.containers.localstack.LocalStackContainer.Service.SECRETSMANAGER;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+import org.projectnessie.nessie.testing.containerspec.ContainerSpecHelper;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.localstack.LocalStackContainer;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;
+import software.amazon.awssdk.services.secretsmanager.model.CreateSecretRequest;
+
+@Testcontainers
+public class ITAwsSecretsSupplier {
+  @Container
+  static LocalStackContainer localstack =
+      new LocalStackContainer(
+              ContainerSpecHelper.builder()
+                  .name("localstack")
+                  .containerClass(ITAwsSecretsSupplier.class)
+                  .build()
+                  .dockerImageName(null)
+                  .asCompatibleSubstituteFor("localstack/localstack"))
+          .withLogConsumer(
+              new Slf4jLogConsumer(LoggerFactory.getLogger(ITAwsSecretsSupplier.class)))
+          .withServices(SECRETSMANAGER);
+
+  @Test
+  public void awsSecretsManager() {
+    URI secretsManagerEndpoint = localstack.getEndpointOverride(SECRETSMANAGER);
+    try (SecretsManagerClient client =
+        SecretsManagerClient.builder()
+            .endpointOverride(secretsManagerEndpoint)
+            .region(Region.of(localstack.getRegion()))
+            .credentialsProvider(
+                StaticCredentialsProvider.create(
+                    AwsBasicCredentials.create(
+                        localstack.getAccessKey(), localstack.getSecretKey())))
+            .build()) {
+      client.createSecret(
+          CreateSecretRequest.builder().name("foo").secretString("secret-foo").build());
+      client.createSecret(
+          CreateSecretRequest.builder()
+              .name("single")
+              .secretString("{\"bar\": \"secret-single\"}")
+              .build());
+      client.createSecret(
+          CreateSecretRequest.builder()
+              .name("multi")
+              .secretString("{\"name\": \"the-name\", \"value\": \"the-value\"}")
+              .build());
+
+      AwsSecretsSupplier awsSecretsSupplier = new AwsSecretsSupplier(client);
+
+      assertThat(awsSecretsSupplier.resolveSecrets(List.of("foo", "single", "multi")))
+          .containsEntry("foo", Map.of("value", "secret-foo"))
+          .containsEntry("single", Map.of("bar", "secret-single"))
+          .containsEntry("multi", Map.of("name", "the-name", "value", "the-value"))
+          .hasSize(3);
+    }
+  }
+}

catalog/secrets/aws/src/intTest/resources/logback-test.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+
+    Copyright (C) 2024 Dremio
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<configuration debug="true">
+  <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator"/>
+  <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder>
+      <pattern>%date{ISO8601} [%thread] [%X{nessie.events.subscription.id}] %-5level %logger{36} -
+        %msg%n</pattern>
+    </encoder>
+  </appender>
+  <root level="${test.log.level:-WARN}">
+    <appender-ref ref="console"/>
+  </root>
+</configuration>

catalog/secrets/aws/src/intTest/resources/org/projectnessie/catalog/secrets/aws/Dockerfile-localstack-version

@@ -0,0 +1,3 @@
+# Dockerfile to provide the image name and tag to a test.
+# Version is managed by Renovate - do not edit.
+FROM docker.io/localstack/localstack:3.4.0

catalog/secrets/aws/src/main/java/org/projectnessie/catalog/secrets/aws/AwsSecretsSupplier.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.catalog.secrets.aws;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.projectnessie.catalog.secrets.spi.SingleValueSecretsSupplier;
+import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;
+import software.amazon.awssdk.services.secretsmanager.model.BatchGetSecretValueRequest;
+import software.amazon.awssdk.services.secretsmanager.model.SecretValueEntry;
+
+public class AwsSecretsSupplier extends SingleValueSecretsSupplier {
+  private final SecretsManagerClient secretsManagerClient;
+
+  public AwsSecretsSupplier(SecretsManagerClient secretsManagerClient) {
+    this.secretsManagerClient = secretsManagerClient;
+  }
+
+  @Override
+  protected Map<String, String> resolveSingleValueSecrets(Collection<String> names) {
+    if (names.isEmpty()) {
+      return Map.of();
+    }
+
+    Map<String, String> secretIdToNameMap = new HashMap<>();
+
+    for (String name : names) {
+      String secretId = nameToSecretId(name);
+      secretIdToNameMap.put(secretId, name);
+    }
+
+    return secretsManagerClient
+        .batchGetSecretValue(
+            BatchGetSecretValueRequest.builder().secretIdList(secretIdToNameMap.keySet()).build())
+        .secretValues()
+        .stream()
+        .collect(
+            Collectors.toMap(
+                sv -> secretIdToNameMap.get(sv.name()), SecretValueEntry::secretString));
+  }
+
+  private String nameToSecretId(String name) {
+    // TODO can either return the name (as known to AWS Secrets Manager) or the fully qualified ARN
+    return name;
+  }
+}

catalog/secrets/cache/build.gradle.kts

@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins {
+  id("nessie-conventions-server")
+  id("nessie-jacoco")
+}
+
+extra["maven.name"] = "Nessie - Catalog - Secrets Cache"
+
+dependencies {
+  implementation(project(":nessie-catalog-secrets-api"))
+  implementation(libs.guava)
+  implementation(libs.caffeine)
+  implementation(libs.micrometer.core)
+
+  implementation(platform(libs.jackson.bom))
+  implementation("com.fasterxml.jackson.core:jackson-databind")
+  implementation("com.fasterxml.jackson.core:jackson-annotations")
+
+  compileOnly(project(":nessie-immutables"))
+  annotationProcessor(project(":nessie-immutables", configuration = "processor"))
+  // javax/jakarta
+  compileOnly(libs.jakarta.ws.rs.api)
+  compileOnly(libs.jakarta.enterprise.cdi.api)
+  compileOnly(libs.jakarta.validation.api)
+
+  compileOnly(libs.errorprone.annotations)
+  compileOnly(libs.microprofile.openapi)
+
+  testFixturesApi(platform(libs.junit.bom))
+  testFixturesApi(libs.bundles.junit.testing)
+}

catalog/secrets/cache/src/main/java/org/projectnessie/catalog/secrets/cache/CachingSecrets.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.catalog.secrets.cache;
+
+import org.projectnessie.catalog.secrets.spi.SecretsSupplier;
+
+public final class CachingSecrets {
+
+  private final CachingSecretsBackend backend;
+
+  public CachingSecrets(CachingSecretsBackend backend) {
+    this.backend = backend;
+  }
+
+  public SecretsSupplier forRepository(String repositoryId, SecretsSupplier secretsSupplier) {
+    return names -> backend.resolveSecrets(repositoryId, names, secretsSupplier);
+  }
+}