Skip to content

Commit

Permalink
Introduce urn:nessie-secret:...
Browse files Browse the repository at this point in the history
  • Loading branch information
snazy committed Sep 6, 2024
1 parent 51ffe73 commit c1dc43a
Show file tree
Hide file tree
Showing 24 changed files with 315 additions and 28 deletions.
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,16 @@ as necessary. Empty sections will not end in the release notes.
### Breaking changes

- The deprecated JDBC configuration properties for `catalog` and `schema` have been removed.
- Catalog/Object store secrets: Secrets are now referenced via a URN as requirement to introduce support
for secret managers like Vault or those offered by cloud vendors. All secret reference URNs use the
pattern `urn:nessie-secret:<provider>:<secret-name>`.
The currently supported provider is `quarkus`, the `<secret-name>` is the name of the Quarkus
configuration entry, which can also be an environment variable name.
Make sure to use the new helm chart.
See [Nessie Docs](https://projectnessie.org/nessie-latest/configuration/#secrets-manager-settings).
- Catalog/Object store secrets: secrets are now handled as immutable composites, which is important
to support secrets rotation with external secrets managers.
See [Nessie Docs](https://projectnessie.org/nessie-latest/configuration/#secrets-manager-settings).

### New Features

Expand Down
1 change: 1 addition & 0 deletions bom/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ dependencies {
api(project(":nessie-catalog-service-impl"))
api(project(":nessie-catalog-service-transfer"))
api(project(":nessie-catalog-secrets-api"))
api(project(":nessie-catalog-secrets-smallrye"))

if (!isIncludedInNesQuEIT()) {
api(project(":nessie-gc-iceberg"))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
* Copyright (C) 2024 Dremio
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.projectnessie.catalog.secrets;

import static com.google.common.base.Preconditions.checkArgument;

import jakarta.annotation.Nonnull;
import java.net.URI;
import java.util.Map;
import java.util.Optional;
import org.projectnessie.nessie.immutables.NessieImmutable;

@NessieImmutable
public abstract class ResolvingSecretsProvider implements SecretsProvider {
abstract Map<String, SecretsProvider> secretsProviders();

@Override
public <S extends Secret> Optional<S> getSecret(
@Nonnull URI name, @Nonnull SecretType secretType, @Nonnull Class<S> secretJavaType) {
String scheme = name.getScheme();
String next = name.getSchemeSpecificPart();
checkArgument(
"urn".equals(scheme) && next != null,
"Invalid secret URI, must be in the form 'urn:nessie-secret:<provider>:<secret-name>'");

int iNessieSecret = next.indexOf(':');
checkArgument(
iNessieSecret > 0 && iNessieSecret != next.length() - 1,
"Invalid secret URI, must be in the form 'urn:nessie-secret:<provider>:<secret-name>'");
scheme = next.substring(0, iNessieSecret);
checkArgument(
"nessie-secret".equals(scheme),
"Invalid secret URI, must be in the form 'urn:nessie-secret:<provider>:<secret-name>'");

int iProvider = next.indexOf(':', iNessieSecret + 1);
checkArgument(
iProvider > 0 && iProvider != next.length() - 1,
"Invalid secret URI, must be in the form 'urn:nessie-secret:<provider>:<secret-name>'");
String provider = next.substring(iNessieSecret + 1, iProvider);
checkArgument(
!provider.isBlank(),
"Invalid secret URI, must be in the form 'urn:nessie-secret:<provider>:<secret-name>'");

next = next.substring(iProvider + 1);
checkArgument(
!next.isBlank() && next.charAt(0) != ':',
"Invalid secret URI, must be in the form 'urn:nessie-secret:<provider>:<secret-name>'");
name = URI.create(next);

SecretsProvider secretsProvider = secretsProviders().get(provider);
if (secretsProvider == null) {
return Optional.empty();
}
return secretsProvider.getSecret(name, secretType, secretJavaType);
}

public static ImmutableResolvingSecretsProvider.Builder builder() {
return ImmutableResolvingSecretsProvider.builder();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/*
* Copyright (C) 2024 Dremio
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.projectnessie.catalog.secrets;

import static org.projectnessie.catalog.secrets.SecretType.BASIC;
import static org.projectnessie.catalog.secrets.SecretType.KEY;
import static org.projectnessie.catalog.secrets.UnsafePlainTextSecretsProvider.unsafePlainTextSecretsProvider;

import java.net.URI;
import java.util.Map;
import org.assertj.core.api.SoftAssertions;
import org.assertj.core.api.junit.jupiter.InjectSoftAssertions;
import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;

@ExtendWith(SoftAssertionsExtension.class)
public class TestResolvingSecretsProvider {
@InjectSoftAssertions protected SoftAssertions soft;

@Test
public void multipleProviders() {
SecretsProvider secretsProvider =
ResolvingSecretsProvider.builder()
.putSecretsProvider(
"one",
unsafePlainTextSecretsProvider(
Map.of(
URI.create("basic"), Map.of("name", "the-name", "secret", "the-secret"))))
.putSecretsProvider(
"two",
unsafePlainTextSecretsProvider(
Map.of(URI.create("key"), Map.of("key", "key-value"))))
.build();

soft.assertThat(
secretsProvider.getSecret(
URI.create("urn:nessie-secret:one:basic"), BASIC, BasicCredentials.class))
.get()
.isInstanceOf(BasicCredentials.class)
.extracting(BasicCredentials::name, BasicCredentials::secret)
.containsExactly("the-name", "the-secret");
soft.assertThat(
secretsProvider.getSecret(
URI.create("urn:nessie-secret:two:key"), KEY, KeySecret.class))
.get()
.isInstanceOf(KeySecret.class)
.extracting(KeySecret::key)
.isEqualTo("key-value");
soft.assertThat(
secretsProvider.getSecret(
URI.create("urn:nessie-secret:no-provider:key"), KEY, KeySecret.class))
.isEmpty();
}

@ParameterizedTest
@CsvSource({
"foo",
"foo:nessie-secret:provider:key",
"urn",
// "urn:", <-- illegal URI
"urn::",
"urn:::",
"urn:secret::",
"urn:secret:provider:key",
"urn:secret:provider:key",
"urn:nessie-secret",
"urn::nessie-secret",
"urn:nessie-secret:",
"urn:nessie-secret:provider",
"urn:nessie-secret::provider",
"urn::nessie-secret::provider",
"urn:nessie-secret:provider:",
"urn:nessie-secret:provider::key",
})
public void illegal(String name) {
SecretsProvider secretsProvider = ResolvingSecretsProvider.builder().build();
soft.assertThatIllegalArgumentException()
.isThrownBy(
() -> secretsProvider.getSecret(URI.create(name), BASIC, BasicCredentials.class))
.withMessage(
"Invalid secret URI, must be in the form 'urn:nessie-secret:<provider>:<secret-name>'");
}
}
43 changes: 43 additions & 0 deletions catalog/secrets/smallrye/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Copyright (C) 2024 Dremio
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

plugins { id("nessie-conventions-server") }

extra["maven.name"] = "Nessie - Catalog - Secrets Cache"

dependencies {
implementation(project(":nessie-catalog-secrets-api"))
implementation(libs.guava)
implementation(libs.smallrye.config.core)

implementation(platform(libs.jackson.bom))
implementation("com.fasterxml.jackson.core:jackson-databind")
implementation("com.fasterxml.jackson.core:jackson-annotations")

compileOnly(project(":nessie-immutables"))
annotationProcessor(project(":nessie-immutables", configuration = "processor"))
// javax/jakarta
compileOnly(libs.jakarta.ws.rs.api)
compileOnly(libs.jakarta.enterprise.cdi.api)
compileOnly(libs.jakarta.validation.api)

compileOnly(libs.errorprone.annotations)
compileOnly(libs.microprofile.openapi)

testFixturesApi(platform(libs.junit.bom))
testFixturesApi(libs.bundles.junit.testing)
testCompileOnly(libs.jakarta.annotation.api)
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.projectnessie.server.catalog.secrets;
package org.projectnessie.catalog.secrets.smallrye;

import io.smallrye.config.SmallRyeConfig;
import jakarta.annotation.Nonnull;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.projectnessie.server.catalog.secrets;
package org.projectnessie.catalog.secrets.smallrye;

import io.smallrye.config.PropertiesConfigSource;
import io.smallrye.config.SmallRyeConfig;
Expand All @@ -30,14 +30,13 @@
import org.projectnessie.catalog.secrets.SecretType;
import org.projectnessie.catalog.secrets.SecretsProvider;
import org.projectnessie.catalog.secrets.TokenSecret;
import org.projectnessie.quarkus.config.QuarkusCatalogConfig;

@ExtendWith(SoftAssertionsExtension.class)
public class TestQuarkusConfigSecretsSupplier {
public class TestSmallryeConfigSecretsProvider {
@InjectSoftAssertions protected SoftAssertions soft;

@Test
public void smallryeConfigSecretsSupplier() {
public void resolveSecrets() {
Map<String, String> configs =
Map.of(
"foo.a", "b",
Expand All @@ -49,7 +48,6 @@ public void smallryeConfigSecretsSupplier() {
new SmallRyeConfigBuilder()
.setAddDefaultSources(false)
.setAddDiscoveredSources(false)
.withMapping(QuarkusCatalogConfig.class)
.withSources(new PropertiesConfigSource(configs, "configSource", 100))
.build();

Expand All @@ -61,7 +59,7 @@ public void smallryeConfigSecretsSupplier() {
}

@Test
public void smallryeConfigSecretsProvider() {
public void getSecrets() {
Map<String, String> configs =
Map.of(
"foo.name",
Expand All @@ -79,7 +77,6 @@ public void smallryeConfigSecretsProvider() {
new SmallRyeConfigBuilder()
.setAddDefaultSources(false)
.setAddDiscoveredSources(false)
.withMapping(QuarkusCatalogConfig.class)
.withSources(new PropertiesConfigSource(configs, "configSource", 100))
.build();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,8 @@ protected static void setupObjectStoreAndNessie(
nessieProperties.put("nessie.catalog.service.s3.default-options.path-style-access", "true");
nessieProperties.put("nessie.catalog.service.s3.default-options.region", "eu-central-1");
nessieProperties.put(
"nessie.catalog.service.s3.default-options.access-key", "with-nessie-access-key");
"nessie.catalog.service.s3.default-options.access-key",
"urn:nessie-secret:quarkus:with-nessie-access-key");
nessieProperties.put("with-nessie-access-key.name", "accessKey");
nessieProperties.put("with-nessie-access-key.secret", "secretKey");
nessieProperties.putAll(serverConfig);
Expand Down
2 changes: 1 addition & 1 deletion docker/catalog-auth-s3-otel-jdbc/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ services:
- nessie.catalog.warehouses.warehouse.location=s3://demobucket/
- nessie.catalog.service.s3.default-options.region=us-east-1
- nessie.catalog.service.s3.default-options.path-style-access=true
- nessie.catalog.service.s3.default-options.access-key=nessie.catalog.secrets.access-key
- nessie.catalog.service.s3.default-options.access-key=urn:nessie-secret:quarkus:nessie.catalog.secrets.access-key
- nessie.catalog.secrets.access-key.name=minioadmin
- nessie.catalog.secrets.access-key.secret=minioadmin
# MinIO endpoint for Nessie server
Expand Down
2 changes: 1 addition & 1 deletion docker/catalog-auth-s3-otel/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ services:
- nessie.catalog.warehouses.warehouse.location=s3://demobucket/
- nessie.catalog.service.s3.default-options.region=us-east-1
- nessie.catalog.service.s3.default-options.path-style-access=true
- nessie.catalog.service.s3.default-options.access-key=nessie.catalog.secrets.access-key
- nessie.catalog.service.s3.default-options.access-key=urn:nessie-secret:quarkus:nessie.catalog.secrets.access-key
- nessie.catalog.secrets.access-key.name=minioadmin
- nessie.catalog.secrets.access-key.secret=minioadmin
# MinIO endpoint for Nessie server
Expand Down
2 changes: 1 addition & 1 deletion docker/catalog-auth-s3/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ services:
- nessie.catalog.warehouses.warehouse.location=s3://demobucket/
- nessie.catalog.service.s3.default-options.region=us-east-1
- nessie.catalog.service.s3.default-options.path-style-access=true
- nessie.catalog.service.s3.default-options.access-key=nessie.catalog.secrets.access-key
- nessie.catalog.service.s3.default-options.access-key=urn:nessie-secret:quarkus:nessie.catalog.secrets.access-key
- nessie.catalog.secrets.access-key.name=minioadmin
- nessie.catalog.secrets.access-key.secret=minioadmin
# MinIO endpoint for Nessie server
Expand Down
2 changes: 1 addition & 1 deletion docker/catalog-nginx-https/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ services:
- nessie.catalog.warehouses.warehouse.location=s3://demobucket/
- nessie.catalog.service.s3.default-options.region=us-east-1
- nessie.catalog.service.s3.default-options.path-style-access=true
- nessie.catalog.service.s3.default-options.access-key=nessie.catalog.secrets.access-key
- nessie.catalog.service.s3.default-options.access-key=urn:nessie-secret:quarkus:nessie.catalog.secrets.access-key
- nessie.catalog.secrets.access-key.name=minioadmin
- nessie.catalog.secrets.access-key.secret=minioadmin
# MinIO endpoint for Nessie server
Expand Down
1 change: 1 addition & 0 deletions gradle/projects.main.properties
Original file line number Diff line number Diff line change
Expand Up @@ -109,3 +109,4 @@ nessie-catalog-service-rest=catalog/service/rest
nessie-catalog-service-impl=catalog/service/impl
nessie-catalog-service-transfer=catalog/service/transfer
nessie-catalog-secrets-api=catalog/secrets/api
nessie-catalog-secrets-smallrye=catalog/secrets/smallrye
2 changes: 1 addition & 1 deletion helm/nessie/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,7 @@ config types know about that symbolic name and resolve it via a SecretsProvider,
# {{ $midfix }}
#
- name: {{ (printf "nessie.catalog.service.%s" $midfix) | quote }}
value: {{ (printf "nessie-catalog-secrets.%s" $midfix) | quote }}
value: {{ (printf "urn:nessie-secret:quarkus:nessie-catalog-secrets.%s" $midfix) | quote }}
{{- end }}
- name: {{ (printf "nessie-catalog-secrets.%s.%s" $midfix $suffix) | quote }}
valueFrom:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public static void start() throws Exception {
+ objectStorage.getS3BaseUri().toString(),
"-Dnessie.catalog.service.s3.default-options.path-style-access=true",
"-Dnessie.catalog.service.s3.default-options.region=eu-central-1",
"-Dnessie.catalog.service.s3.default-options.access-key=nessie-catalog-secrets.s3-access-key",
"-Dnessie.catalog.service.s3.default-options.access-key=urn:nessie-secret:quarkus:nessie-catalog-secrets.s3-access-key",
"-Dnessie-catalog-secrets.s3-access-key.name=accessKey",
"-Dnessie-catalog-secrets.s3-access-key.secret=secretKey");
}
Expand Down
1 change: 1 addition & 0 deletions servers/quarkus-secrets/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ configurations.all { exclude(group = "org.projectnessie.nessie", module = "nessi

dependencies {
implementation(project(":nessie-catalog-secrets-api"))
implementation(project(":nessie-catalog-secrets-smallrye"))
implementation(project(":nessie-quarkus-config"))

implementation(enforcedPlatform(libs.quarkus.bom))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,18 @@
import io.smallrye.config.SmallRyeConfig;
import jakarta.enterprise.inject.Produces;
import jakarta.inject.Singleton;
import org.projectnessie.catalog.secrets.ResolvingSecretsProvider;
import org.projectnessie.catalog.secrets.SecretsProvider;
import org.projectnessie.catalog.secrets.smallrye.SmallryeConfigSecretsProvider;

public class SecretsProducers {

@Produces
@Singleton
public SecretsProvider secretsProvider(SmallRyeConfig smallRyeConfig) {
return new SmallryeConfigSecretsProvider(smallRyeConfig);
// Reference secrets via `urn:nessie-secret:quarkus:<secret-name>
return ResolvingSecretsProvider.builder()
.putSecretsProvider("quarkus", new SmallryeConfigSecretsProvider(smallRyeConfig))
.build();
}
}
Loading

0 comments on commit c1dc43a

Please sign in to comment.