diff --git a/CHANGELOG.md b/CHANGELOG.md index fc2a0ab5979..b6c0bb76d77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,16 @@ as necessary. Empty sections will not end in the release notes. ### Breaking changes - The deprecated JDBC configuration properties for `catalog` and `schema` have been removed. +- Catalog/Object store secrets: Secrets are now referenced via a URN as requirement to introduce support + for secret managers like Vault or those offered by cloud vendors. All secret reference URNs use the + pattern `urn:nessie-secret::`. + The currently supported provider is `quarkus`, the `` is the name of the Quarkus + configuration entry, which can also be an environment variable name. + Make sure to use the new helm chart. + See [Nessie Docs](https://projectnessie.org/nessie-latest/configuration/#secrets-manager-settings). +- Catalog/Object store secrets: secrets are now handled as immutable composites, which is important + to support secrets rotation with external secrets managers. + See [Nessie Docs](https://projectnessie.org/nessie-latest/configuration/#secrets-manager-settings). ### New Features diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index afe9ef3a8a3..44d6a8f03a0 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -121,6 +121,7 @@ dependencies { api(project(":nessie-catalog-service-impl")) api(project(":nessie-catalog-service-transfer")) api(project(":nessie-catalog-secrets-api")) + api(project(":nessie-catalog-secrets-smallrye")) if (!isIncludedInNesQuEIT()) { api(project(":nessie-gc-iceberg")) diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/ResolvingSecretsProvider.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/ResolvingSecretsProvider.java new file mode 100644 index 00000000000..dd6080073c3 --- /dev/null +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/ResolvingSecretsProvider.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets; + +import static com.google.common.base.Preconditions.checkArgument; + +import jakarta.annotation.Nonnull; +import java.net.URI; +import java.util.Map; +import java.util.Optional; +import org.projectnessie.nessie.immutables.NessieImmutable; + +@NessieImmutable +public abstract class ResolvingSecretsProvider implements SecretsProvider { + abstract Map secretsProviders(); + + @Override + public Optional getSecret( + @Nonnull URI name, @Nonnull SecretType secretType, @Nonnull Class secretJavaType) { + String scheme = name.getScheme(); + String next = name.getSchemeSpecificPart(); + checkArgument( + "urn".equals(scheme) && next != null, + "Invalid secret URI, must be in the form 'urn:nessie-secret::'"); + + int iNessieSecret = next.indexOf(':'); + checkArgument( + iNessieSecret > 0 && iNessieSecret != next.length() - 1, + "Invalid secret URI, must be in the form 'urn:nessie-secret::'"); + scheme = next.substring(0, iNessieSecret); + checkArgument( + "nessie-secret".equals(scheme), + "Invalid secret URI, must be in the form 'urn:nessie-secret::'"); + + int iProvider = next.indexOf(':', iNessieSecret + 1); + checkArgument( + iProvider > 0 && iProvider != next.length() - 1, + "Invalid secret URI, must be in the form 'urn:nessie-secret::'"); + String provider = next.substring(iNessieSecret + 1, iProvider); + checkArgument( + !provider.isBlank(), + "Invalid secret URI, must be in the form 'urn:nessie-secret::'"); + + next = next.substring(iProvider + 1); + checkArgument( + !next.isBlank() && next.charAt(0) != ':', + "Invalid secret URI, must be in the form 'urn:nessie-secret::'"); + name = URI.create(next); + + SecretsProvider secretsProvider = secretsProviders().get(provider); + if (secretsProvider == null) { + return Optional.empty(); + } + return secretsProvider.getSecret(name, secretType, secretJavaType); + } + + public static ImmutableResolvingSecretsProvider.Builder builder() { + return ImmutableResolvingSecretsProvider.builder(); + } +} diff --git a/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestResolvingSecretsProvider.java b/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestResolvingSecretsProvider.java new file mode 100644 index 00000000000..72c906b67da --- /dev/null +++ b/catalog/secrets/api/src/test/java/org/projectnessie/catalog/secrets/TestResolvingSecretsProvider.java @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.secrets; + +import static org.projectnessie.catalog.secrets.SecretType.BASIC; +import static org.projectnessie.catalog.secrets.SecretType.KEY; +import static org.projectnessie.catalog.secrets.UnsafePlainTextSecretsProvider.unsafePlainTextSecretsProvider; + +import java.net.URI; +import java.util.Map; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +@ExtendWith(SoftAssertionsExtension.class) +public class TestResolvingSecretsProvider { + @InjectSoftAssertions protected SoftAssertions soft; + + @Test + public void multipleProviders() { + SecretsProvider secretsProvider = + ResolvingSecretsProvider.builder() + .putSecretsProvider( + "one", + unsafePlainTextSecretsProvider( + Map.of( + URI.create("basic"), Map.of("name", "the-name", "secret", "the-secret")))) + .putSecretsProvider( + "two", + unsafePlainTextSecretsProvider( + Map.of(URI.create("key"), Map.of("key", "key-value")))) + .build(); + + soft.assertThat( + secretsProvider.getSecret( + URI.create("urn:nessie-secret:one:basic"), BASIC, BasicCredentials.class)) + .get() + .isInstanceOf(BasicCredentials.class) + .extracting(BasicCredentials::name, BasicCredentials::secret) + .containsExactly("the-name", "the-secret"); + soft.assertThat( + secretsProvider.getSecret( + URI.create("urn:nessie-secret:two:key"), KEY, KeySecret.class)) + .get() + .isInstanceOf(KeySecret.class) + .extracting(KeySecret::key) + .isEqualTo("key-value"); + soft.assertThat( + secretsProvider.getSecret( + URI.create("urn:nessie-secret:no-provider:key"), KEY, KeySecret.class)) + .isEmpty(); + } + + @ParameterizedTest + @CsvSource({ + "foo", + "foo:nessie-secret:provider:key", + "urn", + // "urn:", <-- illegal URI + "urn::", + "urn:::", + "urn:secret::", + "urn:secret:provider:key", + "urn:secret:provider:key", + "urn:nessie-secret", + "urn::nessie-secret", + "urn:nessie-secret:", + "urn:nessie-secret:provider", + "urn:nessie-secret::provider", + "urn::nessie-secret::provider", + "urn:nessie-secret:provider:", + "urn:nessie-secret:provider::key", + }) + public void illegal(String name) { + SecretsProvider secretsProvider = ResolvingSecretsProvider.builder().build(); + soft.assertThatIllegalArgumentException() + .isThrownBy( + () -> secretsProvider.getSecret(URI.create(name), BASIC, BasicCredentials.class)) + .withMessage( + "Invalid secret URI, must be in the form 'urn:nessie-secret::'"); + } +} diff --git a/catalog/secrets/smallrye/build.gradle.kts b/catalog/secrets/smallrye/build.gradle.kts new file mode 100644 index 00000000000..826079378e0 --- /dev/null +++ b/catalog/secrets/smallrye/build.gradle.kts @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2024 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id("nessie-conventions-server") } + +extra["maven.name"] = "Nessie - Catalog - Secrets Cache" + +dependencies { + implementation(project(":nessie-catalog-secrets-api")) + implementation(libs.guava) + implementation(libs.smallrye.config.core) + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-databind") + implementation("com.fasterxml.jackson.core:jackson-annotations") + + compileOnly(project(":nessie-immutables")) + annotationProcessor(project(":nessie-immutables", configuration = "processor")) + // javax/jakarta + compileOnly(libs.jakarta.ws.rs.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.validation.api) + + compileOnly(libs.errorprone.annotations) + compileOnly(libs.microprofile.openapi) + + testFixturesApi(platform(libs.junit.bom)) + testFixturesApi(libs.bundles.junit.testing) + testCompileOnly(libs.jakarta.annotation.api) +} diff --git a/servers/quarkus-secrets/src/main/java/org/projectnessie/server/catalog/secrets/SmallryeConfigSecretsProvider.java b/catalog/secrets/smallrye/src/main/java/org/projectnessie/catalog/secrets/smallrye/SmallryeConfigSecretsProvider.java similarity index 96% rename from servers/quarkus-secrets/src/main/java/org/projectnessie/server/catalog/secrets/SmallryeConfigSecretsProvider.java rename to catalog/secrets/smallrye/src/main/java/org/projectnessie/catalog/secrets/smallrye/SmallryeConfigSecretsProvider.java index eac0a3aa93c..a1be768b14c 100644 --- a/servers/quarkus-secrets/src/main/java/org/projectnessie/server/catalog/secrets/SmallryeConfigSecretsProvider.java +++ b/catalog/secrets/smallrye/src/main/java/org/projectnessie/catalog/secrets/smallrye/SmallryeConfigSecretsProvider.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.projectnessie.server.catalog.secrets; +package org.projectnessie.catalog.secrets.smallrye; import io.smallrye.config.SmallRyeConfig; import jakarta.annotation.Nonnull; diff --git a/servers/quarkus-secrets/src/test/java/org/projectnessie/server/catalog/secrets/TestQuarkusConfigSecretsSupplier.java b/catalog/secrets/smallrye/src/test/java/org/projectnessie/catalog/secrets/smallrye/TestSmallryeConfigSecretsProvider.java similarity index 91% rename from servers/quarkus-secrets/src/test/java/org/projectnessie/server/catalog/secrets/TestQuarkusConfigSecretsSupplier.java rename to catalog/secrets/smallrye/src/test/java/org/projectnessie/catalog/secrets/smallrye/TestSmallryeConfigSecretsProvider.java index 0ed9624b4d5..5c979f44819 100644 --- a/servers/quarkus-secrets/src/test/java/org/projectnessie/server/catalog/secrets/TestQuarkusConfigSecretsSupplier.java +++ b/catalog/secrets/smallrye/src/test/java/org/projectnessie/catalog/secrets/smallrye/TestSmallryeConfigSecretsProvider.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.projectnessie.server.catalog.secrets; +package org.projectnessie.catalog.secrets.smallrye; import io.smallrye.config.PropertiesConfigSource; import io.smallrye.config.SmallRyeConfig; @@ -30,14 +30,13 @@ import org.projectnessie.catalog.secrets.SecretType; import org.projectnessie.catalog.secrets.SecretsProvider; import org.projectnessie.catalog.secrets.TokenSecret; -import org.projectnessie.quarkus.config.QuarkusCatalogConfig; @ExtendWith(SoftAssertionsExtension.class) -public class TestQuarkusConfigSecretsSupplier { +public class TestSmallryeConfigSecretsProvider { @InjectSoftAssertions protected SoftAssertions soft; @Test - public void smallryeConfigSecretsSupplier() { + public void resolveSecrets() { Map configs = Map.of( "foo.a", "b", @@ -49,7 +48,6 @@ public void smallryeConfigSecretsSupplier() { new SmallRyeConfigBuilder() .setAddDefaultSources(false) .setAddDiscoveredSources(false) - .withMapping(QuarkusCatalogConfig.class) .withSources(new PropertiesConfigSource(configs, "configSource", 100)) .build(); @@ -61,7 +59,7 @@ public void smallryeConfigSecretsSupplier() { } @Test - public void smallryeConfigSecretsProvider() { + public void getSecrets() { Map configs = Map.of( "foo.name", @@ -79,7 +77,6 @@ public void smallryeConfigSecretsProvider() { new SmallRyeConfigBuilder() .setAddDefaultSources(false) .setAddDiscoveredSources(false) - .withMapping(QuarkusCatalogConfig.class) .withSources(new PropertiesConfigSource(configs, "configSource", 100)) .build(); diff --git a/cli/cli/src/intTest/java/org/projectnessie/nessie/cli/commands/WithNessie.java b/cli/cli/src/intTest/java/org/projectnessie/nessie/cli/commands/WithNessie.java index b80099ba5fd..86fcc87b61d 100644 --- a/cli/cli/src/intTest/java/org/projectnessie/nessie/cli/commands/WithNessie.java +++ b/cli/cli/src/intTest/java/org/projectnessie/nessie/cli/commands/WithNessie.java @@ -93,7 +93,8 @@ protected static void setupObjectStoreAndNessie( nessieProperties.put("nessie.catalog.service.s3.default-options.path-style-access", "true"); nessieProperties.put("nessie.catalog.service.s3.default-options.region", "eu-central-1"); nessieProperties.put( - "nessie.catalog.service.s3.default-options.access-key", "with-nessie-access-key"); + "nessie.catalog.service.s3.default-options.access-key", + "urn:nessie-secret:quarkus:with-nessie-access-key"); nessieProperties.put("with-nessie-access-key.name", "accessKey"); nessieProperties.put("with-nessie-access-key.secret", "secretKey"); nessieProperties.putAll(serverConfig); diff --git a/docker/catalog-auth-s3-otel-jdbc/docker-compose.yml b/docker/catalog-auth-s3-otel-jdbc/docker-compose.yml index 0a80881e11b..78fb15f4482 100644 --- a/docker/catalog-auth-s3-otel-jdbc/docker-compose.yml +++ b/docker/catalog-auth-s3-otel-jdbc/docker-compose.yml @@ -94,7 +94,7 @@ services: - nessie.catalog.warehouses.warehouse.location=s3://demobucket/ - nessie.catalog.service.s3.default-options.region=us-east-1 - nessie.catalog.service.s3.default-options.path-style-access=true - - nessie.catalog.service.s3.default-options.access-key=nessie.catalog.secrets.access-key + - nessie.catalog.service.s3.default-options.access-key=urn:nessie-secret:quarkus:nessie.catalog.secrets.access-key - nessie.catalog.secrets.access-key.name=minioadmin - nessie.catalog.secrets.access-key.secret=minioadmin # MinIO endpoint for Nessie server diff --git a/docker/catalog-auth-s3-otel/docker-compose.yml b/docker/catalog-auth-s3-otel/docker-compose.yml index f912e5cfb86..ae5bb3c8ac0 100644 --- a/docker/catalog-auth-s3-otel/docker-compose.yml +++ b/docker/catalog-auth-s3-otel/docker-compose.yml @@ -90,7 +90,7 @@ services: - nessie.catalog.warehouses.warehouse.location=s3://demobucket/ - nessie.catalog.service.s3.default-options.region=us-east-1 - nessie.catalog.service.s3.default-options.path-style-access=true - - nessie.catalog.service.s3.default-options.access-key=nessie.catalog.secrets.access-key + - nessie.catalog.service.s3.default-options.access-key=urn:nessie-secret:quarkus:nessie.catalog.secrets.access-key - nessie.catalog.secrets.access-key.name=minioadmin - nessie.catalog.secrets.access-key.secret=minioadmin # MinIO endpoint for Nessie server diff --git a/docker/catalog-auth-s3/docker-compose.yml b/docker/catalog-auth-s3/docker-compose.yml index 7d1a309fec2..bde081e626a 100644 --- a/docker/catalog-auth-s3/docker-compose.yml +++ b/docker/catalog-auth-s3/docker-compose.yml @@ -88,7 +88,7 @@ services: - nessie.catalog.warehouses.warehouse.location=s3://demobucket/ - nessie.catalog.service.s3.default-options.region=us-east-1 - nessie.catalog.service.s3.default-options.path-style-access=true - - nessie.catalog.service.s3.default-options.access-key=nessie.catalog.secrets.access-key + - nessie.catalog.service.s3.default-options.access-key=urn:nessie-secret:quarkus:nessie.catalog.secrets.access-key - nessie.catalog.secrets.access-key.name=minioadmin - nessie.catalog.secrets.access-key.secret=minioadmin # MinIO endpoint for Nessie server diff --git a/docker/catalog-nginx-https/docker-compose.yml b/docker/catalog-nginx-https/docker-compose.yml index 6aa4c0a2b96..4c42d5104db 100644 --- a/docker/catalog-nginx-https/docker-compose.yml +++ b/docker/catalog-nginx-https/docker-compose.yml @@ -64,7 +64,7 @@ services: - nessie.catalog.warehouses.warehouse.location=s3://demobucket/ - nessie.catalog.service.s3.default-options.region=us-east-1 - nessie.catalog.service.s3.default-options.path-style-access=true - - nessie.catalog.service.s3.default-options.access-key=nessie.catalog.secrets.access-key + - nessie.catalog.service.s3.default-options.access-key=urn:nessie-secret:quarkus:nessie.catalog.secrets.access-key - nessie.catalog.secrets.access-key.name=minioadmin - nessie.catalog.secrets.access-key.secret=minioadmin # MinIO endpoint for Nessie server diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 783385abfb6..217611cfff8 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -109,3 +109,4 @@ nessie-catalog-service-rest=catalog/service/rest nessie-catalog-service-impl=catalog/service/impl nessie-catalog-service-transfer=catalog/service/transfer nessie-catalog-secrets-api=catalog/secrets/api +nessie-catalog-secrets-smallrye=catalog/secrets/smallrye diff --git a/helm/nessie/templates/_helpers.tpl b/helm/nessie/templates/_helpers.tpl index 7cd6a5e5e4f..72ff81c4e86 100644 --- a/helm/nessie/templates/_helpers.tpl +++ b/helm/nessie/templates/_helpers.tpl @@ -347,7 +347,7 @@ config types know about that symbolic name and resolve it via a SecretsProvider, # {{ $midfix }} # - name: {{ (printf "nessie.catalog.service.%s" $midfix) | quote }} - value: {{ (printf "nessie-catalog-secrets.%s" $midfix) | quote }} + value: {{ (printf "urn:nessie-secret:quarkus:nessie-catalog-secrets.%s" $midfix) | quote }} {{- end }} - name: {{ (printf "nessie-catalog-secrets.%s.%s" $midfix $suffix) | quote }} valueFrom: diff --git a/integrations/spark-extensions/shared/src/intTest/java/org/projectnessie/spark/extensions/ITNessieStatementsViaIcebergRest.java b/integrations/spark-extensions/shared/src/intTest/java/org/projectnessie/spark/extensions/ITNessieStatementsViaIcebergRest.java index 3f61aeb2cf3..d3ea76638d9 100644 --- a/integrations/spark-extensions/shared/src/intTest/java/org/projectnessie/spark/extensions/ITNessieStatementsViaIcebergRest.java +++ b/integrations/spark-extensions/shared/src/intTest/java/org/projectnessie/spark/extensions/ITNessieStatementsViaIcebergRest.java @@ -42,7 +42,7 @@ public static void start() throws Exception { + objectStorage.getS3BaseUri().toString(), "-Dnessie.catalog.service.s3.default-options.path-style-access=true", "-Dnessie.catalog.service.s3.default-options.region=eu-central-1", - "-Dnessie.catalog.service.s3.default-options.access-key=nessie-catalog-secrets.s3-access-key", + "-Dnessie.catalog.service.s3.default-options.access-key=urn:nessie-secret:quarkus:nessie-catalog-secrets.s3-access-key", "-Dnessie-catalog-secrets.s3-access-key.name=accessKey", "-Dnessie-catalog-secrets.s3-access-key.secret=secretKey"); } diff --git a/servers/quarkus-secrets/build.gradle.kts b/servers/quarkus-secrets/build.gradle.kts index 9ed2427d747..71e30253416 100644 --- a/servers/quarkus-secrets/build.gradle.kts +++ b/servers/quarkus-secrets/build.gradle.kts @@ -27,6 +27,7 @@ configurations.all { exclude(group = "org.projectnessie.nessie", module = "nessi dependencies { implementation(project(":nessie-catalog-secrets-api")) + implementation(project(":nessie-catalog-secrets-smallrye")) implementation(project(":nessie-quarkus-config")) implementation(enforcedPlatform(libs.quarkus.bom)) diff --git a/servers/quarkus-secrets/src/main/java/org/projectnessie/server/catalog/secrets/SecretsProducers.java b/servers/quarkus-secrets/src/main/java/org/projectnessie/server/catalog/secrets/SecretsProducers.java index e74edd88385..e2556c1ce11 100644 --- a/servers/quarkus-secrets/src/main/java/org/projectnessie/server/catalog/secrets/SecretsProducers.java +++ b/servers/quarkus-secrets/src/main/java/org/projectnessie/server/catalog/secrets/SecretsProducers.java @@ -18,13 +18,18 @@ import io.smallrye.config.SmallRyeConfig; import jakarta.enterprise.inject.Produces; import jakarta.inject.Singleton; +import org.projectnessie.catalog.secrets.ResolvingSecretsProvider; import org.projectnessie.catalog.secrets.SecretsProvider; +import org.projectnessie.catalog.secrets.smallrye.SmallryeConfigSecretsProvider; public class SecretsProducers { @Produces @Singleton public SecretsProvider secretsProvider(SmallRyeConfig smallRyeConfig) { - return new SmallryeConfigSecretsProvider(smallRyeConfig); + // Reference secrets via `urn:nessie-secret:quarkus: + return ResolvingSecretsProvider.builder() + .putSecretsProvider("quarkus", new SmallryeConfigSecretsProvider(smallRyeConfig)) + .build(); } } diff --git a/servers/quarkus-server/src/main/resources/application.properties b/servers/quarkus-server/src/main/resources/application.properties index 3b7148a68a3..1e1d9a3293d 100644 --- a/servers/quarkus-server/src/main/resources/application.properties +++ b/servers/quarkus-server/src/main/resources/application.properties @@ -67,12 +67,12 @@ nessie.server.send-stacktrace-to-client=false #nessie.catalog.service.s3.default-options.endpoint=http://localhost:9000 #nessie.catalog.service.s3.default-options.path-style-access=false #nessie.catalog.service.s3.default-options.region=us-west-2 -#nessie.catalog.service.s3.default-options.access-key=my-secrets.s3-default +#nessie.catalog.service.s3.default-options.access-key=urn:nessie-secret:quarkus:my-secrets.s3-default #my-secrets.s3-default.name=awsAccessKeyId #my-secrets.s3-default.secret=awsSecretAccessKey # per-bucket S3 settings #nessie.catalog.service.s3.buckets.bucket1.endpoint=s3a://bucket1 -#nessie.catalog.service.s3.buckets.bucket1.access-key=my-secrets.s3-bucket +#nessie.catalog.service.s3.buckets.bucket1.access-key=urn:nessie-secret:quarkus:my-secrets.s3-bucket #nessie.catalog.service.s3.buckets.bucket1.region=us-east-1 #my-secrets.s3-bucket.name=awsAccessKeyId1 #my-secrets.s3-bucket.secret=awsSecretAccessKey1 @@ -82,27 +82,27 @@ nessie.server.send-stacktrace-to-client=false #nessie.catalog.service.gcs.default-options.host=http://localhost:4443 #nessie.catalog.service.gcs.default-options.project-id=nessie #nessie.catalog.service.gcs.default-options.auth-type=access_token -#nessie.catalog.service.gcs.default-options.oauth2-token=my-secrets.gcs-default +#nessie.catalog.service.gcs.default-options.oauth2-token=urn:nessie-secret:quarkus:my-secrets.gcs-default #my-secrets.gcs-default.token=tokenRef # per-bucket GCS settings #nessie.catalog.service.gcs.buckets.bucket1.host=http://localhost:4443 #nessie.catalog.service.gcs.buckets.bucket1.project-id=nessie #nessie.catalog.service.gcs.buckets.bucket1.auth-type=access_token -#nessie.catalog.service.gcs.buckets.bucket1.oauth2-token=my-secrets.gcs-bucket +#nessie.catalog.service.gcs.buckets.bucket1.oauth2-token=urn:nessie-secret:quarkus:my-secrets.gcs-bucket #my-secrets.gcs-bucket.token=tokenRef # ADLS settings #nessie.catalog.service.adls.default-options.endpoint=http://localhost/adlsgen2/bucket #nessie.catalog.service.adls.default-options.auth-type=none -#nessie.catalog.service.adls.default-options.account=my-secrets.adls-default +#nessie.catalog.service.adls.default-options.account=urn:nessie-secret:quarkus:my-secrets.adls-default #nessie.catalog.service.adls.default-options.configuration.propname=propvalue #my-secrets.adls-default.name=account #my-secrets.adls-default.secret=secret # per-file-system ADLS settings #nessie.catalog.service.adls.file-systems.bucket1.endpoint=http://localhost/adlsgen2/bucket #nessie.catalog.service.adls.file-systems.bucket1.auth-type=none -#nessie.catalog.service.adls.file-systems.bucket1.account=my-secrets.adls-fs +#nessie.catalog.service.adls.file-systems.bucket1.account=urn:nessie-secret:quarkus:my-secrets.adls-fs #nessie.catalog.service.adls.file-systems.bucket1.configuration.propname=propvalue #my-secrets.adls-fs.name=account #my-secrets.adls-fs.secret=secret diff --git a/servers/quarkus-server/src/test/java/org/projectnessie/server/catalog/s3/TestVendedS3CredentialsExpiry.java b/servers/quarkus-server/src/test/java/org/projectnessie/server/catalog/s3/TestVendedS3CredentialsExpiry.java index de5b2fcd0ad..3917e1346d2 100644 --- a/servers/quarkus-server/src/test/java/org/projectnessie/server/catalog/s3/TestVendedS3CredentialsExpiry.java +++ b/servers/quarkus-server/src/test/java/org/projectnessie/server/catalog/s3/TestVendedS3CredentialsExpiry.java @@ -124,7 +124,9 @@ public Map getConfigOverrides() { .putAll(super.getConfigOverrides()) .put("nessie.catalog.default-warehouse", WAREHOUSE_NAME) .put("nessie.catalog.service.s3.default-options.region", "us-west-2") - .put("nessie.catalog.service.s3.default-options.access-key", "test-access-key-secret") + .put( + "nessie.catalog.service.s3.default-options.access-key", + "urn:nessie-secret:quarkus:test-access-key-secret") .put("test-access-key-secret.name", "test-secret-key") .put("test-access-key-secret.secret", "test-secret-key") .put("nessie.catalog.service.s3.default-options.request-signing-enabled", "false") diff --git a/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/AzuriteTestResourceLifecycleManager.java b/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/AzuriteTestResourceLifecycleManager.java index 8b01a18877f..cda4c93fe00 100644 --- a/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/AzuriteTestResourceLifecycleManager.java +++ b/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/AzuriteTestResourceLifecycleManager.java @@ -40,7 +40,9 @@ public Map start() { + azurite.storageContainer()); return ImmutableMap.builder() .put("nessie.catalog.service.adls.default-options.auth-type", "STORAGE_SHARED_KEY") - .put("nessie.catalog.service.adls.default-options.account", "my-azurite-account") + .put( + "nessie.catalog.service.adls.default-options.account", + "urn:nessie-secret:quarkus:my-azurite-account") .put("my-azurite-account.name", azurite.account()) .put("my-azurite-account.secret", azurite.secretBase64()) .put("nessie.catalog.service.adls.default-options.endpoint", azurite.endpoint()) diff --git a/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/GcsEmulatorTestResourceLifecycleManager.java b/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/GcsEmulatorTestResourceLifecycleManager.java index 4d62c77431b..19c257edcef 100644 --- a/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/GcsEmulatorTestResourceLifecycleManager.java +++ b/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/GcsEmulatorTestResourceLifecycleManager.java @@ -36,7 +36,9 @@ public Map start() { .put("nessie.catalog.service.gcs.default-options.host", gcs.baseUri()) .put("nessie.catalog.service.gcs.default-options.project-id", gcs.projectId()) .put("nessie.catalog.service.gcs.default-options.auth-type", "ACCESS_TOKEN") - .put("nessie.catalog.service.gcs.default-options.oauth2-token", "my-gcs-token") + .put( + "nessie.catalog.service.gcs.default-options.oauth2-token", + "urn:nessie-secret:quarkus:my-gcs-token") .put("my-gcs-token.token", gcs.oauth2token()) .put("nessie.catalog.default-warehouse", "warehouse") .put("nessie.catalog.warehouses.warehouse.location", warehouseLocation.toString()) diff --git a/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/MinioTestResourceLifecycleManager.java b/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/MinioTestResourceLifecycleManager.java index 3ab1bffa33c..aa89e1a40d8 100644 --- a/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/MinioTestResourceLifecycleManager.java +++ b/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/MinioTestResourceLifecycleManager.java @@ -44,7 +44,9 @@ public Map start() { .put("nessie.catalog.service.s3.default-options.path-style-access", "true") .put("nessie.catalog.service.s3.default-options.sts-endpoint", minio.s3endpoint()) .put("nessie.catalog.service.s3.default-options.region", TEST_REGION) - .put("nessie.catalog.service.s3.default-options.access-key", "minio-access-key") + .put( + "nessie.catalog.service.s3.default-options.access-key", + "urn:nessie-secret:quarkus:minio-access-key") .put("minio-access-key.name", minio.accessKey()) .put("minio-access-key.secret", minio.secretKey()) .put("nessie.catalog.default-warehouse", "warehouse") diff --git a/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/ObjectStorageMockTestResourceLifecycleManager.java b/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/ObjectStorageMockTestResourceLifecycleManager.java index 29016e7264f..ad31a0102ad 100644 --- a/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/ObjectStorageMockTestResourceLifecycleManager.java +++ b/servers/quarkus-server/src/testFixtures/java/org/projectnessie/server/catalog/ObjectStorageMockTestResourceLifecycleManager.java @@ -77,7 +77,9 @@ public Map start() { .put("nessie.catalog.service.s3.buckets.mock-bucket.endpoint", s3Endpoint) .put("nessie.catalog.service.s3.buckets.mock-bucket.region", "us-east-1") .put("nessie.catalog.service.s3.buckets.mock-bucket.path-style-access", "true") - .put("nessie.catalog.service.s3.buckets.mock-bucket.access-key", "mock-bucket-access-key") + .put( + "nessie.catalog.service.s3.buckets.mock-bucket.access-key", + "urn:nessie-secret:quarkus:mock-bucket-access-key") .put("mock-bucket-access-key.name", "accessKey") .put("mock-bucket-access-key.secret", "secretKey") // GCS @@ -88,7 +90,9 @@ public Map start() { // ADLS .put("nessie.catalog.service.adls.file-systems.mock-fs.name", BUCKET) .put("nessie.catalog.service.adls.file-systems.mock-fs.endpoint", adlsEndpoint) - .put("nessie.catalog.service.adls.file-systems.mock-fs.sas-token", "sas-token") + .put( + "nessie.catalog.service.adls.file-systems.mock-fs.sas-token", + "urn:nessie-secret:quarkus:sas-token") .put("sas-token.key", "sas-token") .put("nessie.catalog.service.adls.file-systems.mock-fs.auth-type", "SAS_TOKEN") .build(); diff --git a/site/in-dev/configuration.md b/site/in-dev/configuration.md index 1947d92bea3..50d07f40783 100644 --- a/site/in-dev/configuration.md +++ b/site/in-dev/configuration.md @@ -183,6 +183,50 @@ Related Quarkus settings: {% include './generated-docs/smallrye-nessie_catalog_service.md' %} +#### Secrets manager settings + +Secrets for object stores are strictly separated from the actual configuration entries. This enables +the use of external secrets managers. Secrets are referenced using a URN notation. + +The URN notation for Nessie secrets is `urn:nessie-secret::`. `` +references the name of the provider, for example `quarkus` to resolve secrets via the +[Quarkus configuration](#quarkus-configuration-incl-environment-variables). `` is the +secrets manager specific name for the secret to resolve. + +##### Types of Secrets + +* **Basic credentials** are composites of a `name` attribute and a `secret` attribute. + AWS credentials are managed as basic credentials, where the `name` represents the access key ID and + the `secret` represents the secret access key. +* **Tokens** are composites of a `token` attribute and an optional `expiresAt` attribute, latter + represented as an instant. +* **Keys** consist of a single `key` attribute. + +##### Quarkus configuration (incl environment variables) + +Object store secrets managed via Quarkus' configuration mechanism (SmallRye Config) resolve components of +the secret types (basic credentials, tokens, keys) via individual configuration keys. + +The Quarkus configuration key prefix (or environment variable name) is specified for the secret using the +URN notation `urn:nessie-secret:quarkus:.`. + +The following example illustrates the Quarkus configuration entries to define the default S3 access-key and +secret-access-key: +```properties +# +# Prefix of the Quarkus configuration keys for this secret ---+ +# | +# The URN for Quarkus secrets ---+ | +# | | +# |------------------------- |-------------------- +nessie.catalog.service.s3.default-options.access-key=urn:nessie-secret:quarkus:my-secrets.s3-default +# The AWS access-key and secret-access-key are referenced via the "secret-part" name, +# see 'Types of Secrets' above. +# `my-secrets.s3-default` is the `secret-part` as in the last part of the above property +my-secrets.s3-default.name=awsAccessKeyId +my-secrets.s3-default.secret=awsSecretAccessKey +``` + ### Version Store Settings {% include './generated-docs/smallrye-nessie_version_store.md' %}