Add SecretsProvider implementations for AWS, GCP, Vault

bom/build.gradle.kts

@@ -122,6 +122,11 @@ dependencies {
     api(project(":nessie-catalog-service-transfer"))
     api(project(":nessie-catalog-secrets-api"))
     api(project(":nessie-catalog-secrets-smallrye"))
+    api(project(":nessie-catalog-secrets-cache"))
+    api(project(":nessie-catalog-secrets-aws"))
+    api(project(":nessie-catalog-secrets-gcs"))
+    api(project(":nessie-catalog-secrets-azure"))
+    api(project(":nessie-catalog-secrets-vault"))
 
     if (!isIncludedInNesQuEIT()) {
       api(project(":nessie-gc-iceberg"))

catalog/files/impl/build.gradle.kts

@@ -65,11 +65,13 @@ dependencies {
 
   testImplementation(platform(libs.cel.bom))
   testImplementation("org.projectnessie.cel:cel-standalone")
+
   testRuntimeOnly(libs.logback.classic)
 
   jmhImplementation(libs.jmh.core)
   jmhImplementation(project(":nessie-object-storage-mock"))
   jmhAnnotationProcessor(libs.jmh.generator.annprocess)
+  jmhImplementation(testFixtures(project(":nessie-catalog-secrets-api")))
 }
 
 tasks.named("processJmhJandexIndex").configure { enabled = false }

catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/AbstractStringBasedSecretsManager.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.catalog.secrets;
+
+import jakarta.annotation.Nonnull;
+import java.util.Map;
+
+public abstract class AbstractStringBasedSecretsManager extends AbstractMapBasedSecretsManager {
+  @Override
+  protected Map<String, String> resolveSecret(@Nonnull String name) {
+    String value = resolveSecretString(name);
+    return value != null ? parseOrSingle(value) : null;
+  }
+
+  protected abstract String resolveSecretString(String name);
+}

catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretJsonParser.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.catalog.secrets;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Map;
+
+final class SecretJsonParser {
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+
+  private SecretJsonParser() {}
+
+  @SuppressWarnings("unchecked")
+  static Map<String, String> parseOrSingle(String s) {
+    if (!s.trim().startsWith("{")) {
+      return Map.of("value", s);
+    }
+    try {
+      return MAPPER.readValue(s, Map.class);
+    } catch (JsonProcessingException e) {
+      return Map.of("value", s);
+    }
+  }
+}

catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/SecretType.java

@@ -17,6 +17,7 @@
 
 import static org.projectnessie.catalog.secrets.BasicCredentials.basicCredentials;
 import static org.projectnessie.catalog.secrets.KeySecret.keySecret;
+import static org.projectnessie.catalog.secrets.SecretJsonParser.parseOrSingle;
 import static org.projectnessie.catalog.secrets.TokenSecret.tokenSecret;
 
 import java.util.Map;
@@ -33,6 +34,16 @@ public Secret fromValueMap(Map<String, String> value) {
     public Secret fromValueMap(Map<String, String> value) {
       return keySecret(value);
     }
+
+    @Override
+    public Secret parse(String string) {
+      return keySecret(string);
+    }
+
+    @Override
+    public boolean singleValued() {
+      return true;
+    }
   },
   EXPIRING_TOKEN() {
     @Override
@@ -44,4 +55,12 @@ public Secret fromValueMap(Map<String, String> value) {
 
   /** Construct a {@link Secret} instance from its map representation. */
   public abstract Secret fromValueMap(Map<String, String> value);
+
+  public boolean singleValued() {
+    return false;
+  }
+
+  public Secret parse(String string) {
+    return fromValueMap(parseOrSingle(string));
+  }
 }

catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestSecretType.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.catalog.secrets;
+
+import java.time.Instant;
+import java.util.Map;
+import java.util.Optional;
+import org.assertj.core.api.SoftAssertions;
+import org.assertj.core.api.junit.jupiter.InjectSoftAssertions;
+import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(SoftAssertionsExtension.class)
+public class TestSecretType {
+  @InjectSoftAssertions protected SoftAssertions soft;
+
+  @Test
+  public void keySecret() {
+    soft.assertThat(SecretType.KEY.parse("foo"))
+        .extracting(KeySecret.class::cast)
+        .extracting(KeySecret::key)
+        .isEqualTo("foo");
+    soft.assertThat(SecretType.KEY.parse("{\"key\": \"foo\"}"))
+        .extracting(KeySecret.class::cast)
+        .extracting(KeySecret::key)
+        .isEqualTo("{\"key\": \"foo\"}");
+    soft.assertThat(SecretType.KEY.parse("{\"value\": \"foo\"}"))
+        .extracting(KeySecret.class::cast)
+        .extracting(KeySecret::key)
+        .isEqualTo("{\"value\": \"foo\"}");
+
+    soft.assertThat(SecretType.KEY.fromValueMap(Map.of("key", "foo")))
+        .extracting(KeySecret.class::cast)
+        .extracting(KeySecret::key)
+        .isEqualTo("foo");
+    soft.assertThat(SecretType.KEY.fromValueMap(Map.of("value", "foo")))
+        .extracting(KeySecret.class::cast)
+        .extracting(KeySecret::key)
+        .isEqualTo("foo");
+    soft.assertThat(SecretType.KEY.fromValueMap(Map.of("blah", "foo"))).isNull();
+    soft.assertThat(SecretType.KEY.fromValueMap(Map.of())).isNull();
+  }
+
+  @Test
+  public void tokenSecret() {
+    String instantStr = "2024-12-24T12:12:12Z";
+    Instant instant = Instant.parse(instantStr);
+
+    soft.assertThat(SecretType.EXPIRING_TOKEN.parse("tok"))
+        .extracting(TokenSecret.class::cast)
+        .extracting(TokenSecret::token, TokenSecret::expiresAt)
+        .containsExactly("tok", Optional.empty());
+    soft.assertThat(SecretType.EXPIRING_TOKEN.parse("{\"value\": \"tok\"}"))
+        .extracting(TokenSecret.class::cast)
+        .extracting(TokenSecret::token, TokenSecret::expiresAt)
+        .containsExactly("tok", Optional.empty());
+    soft.assertThat(
+            SecretType.EXPIRING_TOKEN.parse(
+                "{\"value\": \"tok\", \"expiresAt\": \"" + instantStr + "\"}"))
+        .extracting(TokenSecret.class::cast)
+        .extracting(TokenSecret::token, TokenSecret::expiresAt)
+        .containsExactly("tok", Optional.of(instant));
+    soft.assertThat(SecretType.EXPIRING_TOKEN.parse("{\"blah\": \"foo\"}")).isNull();
+    soft.assertThat(SecretType.EXPIRING_TOKEN.parse("{}")).isNull();
+
+    soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("token", "tok")))
+        .extracting(TokenSecret.class::cast)
+        .extracting(TokenSecret::token, TokenSecret::expiresAt)
+        .containsExactly("tok", Optional.empty());
+    soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("value", "tok")))
+        .extracting(TokenSecret.class::cast)
+        .extracting(TokenSecret::token, TokenSecret::expiresAt)
+        .containsExactly("tok", Optional.empty());
+    soft.assertThat(
+            SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("token", "tok", "expiresAt", instantStr)))
+        .extracting(TokenSecret.class::cast)
+        .extracting(TokenSecret::token, TokenSecret::expiresAt)
+        .containsExactly("tok", Optional.of(instant));
+    soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of("blah", "foo"))).isNull();
+    soft.assertThat(SecretType.EXPIRING_TOKEN.fromValueMap(Map.of())).isNull();
+  }
+
+  @Test
+  public void basicCredential() {
+    soft.assertThat(SecretType.BASIC.parse("tok")).isNull();
+    soft.assertThat(SecretType.BASIC.parse("{\"name\": \"user\", \"secret\": \"pass\"}"))
+        .extracting(BasicCredentials.class::cast)
+        .extracting(BasicCredentials::name, BasicCredentials::secret)
+        .containsExactly("user", "pass");
+    soft.assertThat(SecretType.BASIC.parse("{\"blah\": \"foo\"}")).isNull();
+    soft.assertThat(SecretType.BASIC.parse("{}")).isNull();
+
+    soft.assertThat(SecretType.BASIC.fromValueMap(Map.of("name", "user", "secret", "pass")))
+        .extracting(BasicCredentials.class::cast)
+        .extracting(BasicCredentials::name, BasicCredentials::secret)
+        .containsExactly("user", "pass");
+    soft.assertThat(SecretType.BASIC.fromValueMap(Map.of("name", "user"))).isNull();
+    soft.assertThat(SecretType.BASIC.fromValueMap(Map.of("blah", "foo"))).isNull();
+    soft.assertThat(SecretType.BASIC.fromValueMap(Map.of())).isNull();
+  }
+}

catalog/secrets/aws/build.gradle.kts

@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins { id("nessie-conventions-server") }
+
+extra["maven.name"] = "Nessie - Catalog - Secrets AWS"
+
+dependencies {
+  implementation(project(":nessie-catalog-secrets-api"))
+  implementation(libs.guava)
+
+  implementation(platform(libs.awssdk.bom))
+  implementation("software.amazon.awssdk:apache-client") {
+    exclude("commons-logging", "commons-logging")
+  }
+  implementation("software.amazon.awssdk:secretsmanager")
+
+  compileOnly(project(":nessie-immutables"))
+  annotationProcessor(project(":nessie-immutables", configuration = "processor"))
+  // javax/jakarta
+  compileOnly(libs.jakarta.ws.rs.api)
+  compileOnly(libs.jakarta.enterprise.cdi.api)
+  compileOnly(libs.jakarta.validation.api)
+
+  compileOnly(libs.errorprone.annotations)
+  compileOnly(libs.microprofile.openapi)
+
+  testFixturesApi(platform(libs.junit.bom))
+  testFixturesApi(libs.bundles.junit.testing)
+
+  intTestCompileOnly(project(":nessie-immutables"))
+  intTestImplementation(platform(libs.testcontainers.bom))
+  intTestImplementation("org.testcontainers:testcontainers")
+  intTestImplementation("org.testcontainers:localstack")
+  intTestImplementation("org.testcontainers:junit-jupiter")
+  intTestImplementation(project(":nessie-container-spec-helper"))
+  intTestRuntimeOnly(libs.logback.classic)
+}