From d570f1bcc1454fbd99b566ee68d01f13c4b561c1 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Thu, 21 Sep 2023 09:43:28 +0200 Subject: [PATCH] Bump Nessie to 0.71.0, adjust to Nessie client-builder updates --- gradle/libs.versions.toml | 2 +- .../apache/iceberg/nessie/NessieCatalog.java | 15 +++-- .../nessie/TestCustomNessieClient.java | 60 +++++++++---------- ...g.projectnessie.client.NessieClientBuilder | 16 +++++ 4 files changed, 56 insertions(+), 37 deletions(-) create mode 100644 nessie/src/test/resources/META-INF/services/org.projectnessie.client.NessieClientBuilder diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d562ff7e799b..2deb6c3c2d6f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -47,7 +47,7 @@ kryo-shaded = "4.0.3" microprofile-openapi-api = "3.1.1" mockito = "4.11.0" mockserver = "5.15.0" -nessie = "0.67.0" +nessie = "0.71.0" netty-buffer = "4.1.97.Final" netty-buffer-compat = "4.1.68.Final" object-client-bundle = "3.3.2" diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java index b02df3d7596d..bacc4a485393 100644 --- a/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java +++ b/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java @@ -18,6 +18,8 @@ */ package org.apache.iceberg.nessie; +import static org.projectnessie.client.config.NessieClientConfigSources.mapConfigSource; + import java.io.IOException; import java.util.List; import java.util.Map; @@ -46,6 +48,7 @@ import org.projectnessie.client.NessieConfigConstants; import org.projectnessie.client.api.NessieApiV1; import org.projectnessie.client.api.NessieApiV2; +import org.projectnessie.client.config.NessieClientConfigSource; import org.projectnessie.client.http.HttpClientBuilder; import org.projectnessie.model.ContentKey; import org.projectnessie.model.TableReference; @@ -94,10 +97,10 @@ public void initialize(String name, Map options) { String requestedHash = options.get(removePrefix.apply(NessieConfigConstants.CONF_NESSIE_REF_HASH)); - NessieClientBuilder nessieClientBuilder = - createNessieClientBuilder( - options.get(NessieConfigConstants.CONF_NESSIE_CLIENT_BUILDER_IMPL)) - .fromConfig(x -> options.get(removePrefix.apply(x))); + NessieClientConfigSource configSource = + mapConfigSource(options).fallbackTo(x -> options.get(removePrefix.apply(x))); + NessieClientBuilder nessieClientBuilder = + NessieClientBuilder.createClientBuilderFromSystemSettings(configSource); // default version is set to v1. final String apiVersion = options.getOrDefault(removePrefix.apply(NessieUtil.CLIENT_API_VERSION), "1"); @@ -182,8 +185,8 @@ private String validateWarehouseLocation(String name, Map catalo return warehouseLocation; } - private static NessieClientBuilder createNessieClientBuilder(String customBuilder) { - NessieClientBuilder clientBuilder; + private static NessieClientBuilder createNessieClientBuilder(String customBuilder) { + NessieClientBuilder clientBuilder; if (customBuilder != null) { try { clientBuilder = diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java index 40341358073f..5738d1386e2a 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java @@ -20,17 +20,14 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; -import java.net.URI; -import java.util.function.Function; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.TestCatalogUtil; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; -import org.projectnessie.client.NessieClientBuilder; +import org.projectnessie.client.NessieClientBuilder.AbstractNessieClientBuilder; import org.projectnessie.client.NessieConfigConstants; import org.projectnessie.client.api.NessieApi; -import org.projectnessie.client.auth.NessieAuthentication; import org.projectnessie.client.http.HttpClientBuilder; public class TestCustomNessieClient extends BaseTestIceberg { @@ -71,7 +68,6 @@ public void testUnnecessaryDefaultCustomClient() { @Test public void testNonExistentCustomClient() { - String nonExistingClass = "non.existent.ClientBuilderImpl"; assertThatThrownBy( () -> { NessieCatalog catalog = new NessieCatalog(); @@ -83,14 +79,14 @@ public void testNonExistentCustomClient() { CatalogProperties.URI, uri, NessieConfigConstants.CONF_NESSIE_CLIENT_BUILDER_IMPL, - nonExistingClass)); + "non.existent.ClientBuilderImpl")); }) .isInstanceOf(RuntimeException.class) - .hasMessageContaining(nonExistingClass); + .hasMessageContaining("Cannot load Nessie client builder implementation class"); } @Test - public void testCustomClient() { + public void testCustomClientByImpl() { assertThatThrownBy( () -> { NessieCatalog catalog = new NessieCatalog(); @@ -108,6 +104,25 @@ public void testCustomClient() { .hasMessage("BUILD CALLED"); } + @Test + public void testCustomClientByName() { + assertThatThrownBy( + () -> { + NessieCatalog catalog = new NessieCatalog(); + catalog.initialize( + "nessie", + ImmutableMap.of( + CatalogProperties.WAREHOUSE_LOCATION, + temp.toUri().toString(), + CatalogProperties.URI, + uri, + NessieConfigConstants.CONF_NESSIE_CLIENT_NAME, + "Dummy")); + }) + .isInstanceOf(RuntimeException.class) + .hasMessage("BUILD CALLED"); + } + @Test public void testAlternativeInitializeWithNulls() { NessieCatalog catalog = new NessieCatalog(); @@ -128,7 +143,7 @@ public void testAlternativeInitializeWithNulls() { } @SuppressWarnings("rawtypes") - public static final class DummyClientBuilderImpl implements NessieClientBuilder { + public static final class DummyClientBuilderImpl extends AbstractNessieClientBuilder { @SuppressWarnings("unused") public static DummyClientBuilderImpl builder() { @@ -136,33 +151,18 @@ public static DummyClientBuilderImpl builder() { } @Override - public NessieClientBuilder fromSystemProperties() { - return this; - } - - @Override - public NessieClientBuilder withAuthentication(NessieAuthentication authentication) { - return this; - } - - @Override - public NessieClientBuilder withUri(URI uri) { - return this; - } - - @Override - public NessieClientBuilder withAuthenticationFromConfig(Function configuration) { - return this; + public API build(Class apiContract) { + throw new RuntimeException("BUILD CALLED"); } @Override - public NessieClientBuilder fromConfig(Function configuration) { - return this; + public String name() { + return "Dummy"; } @Override - public NessieApi build(Class apiContract) { - throw new RuntimeException("BUILD CALLED"); + public int priority() { + return 42; } } } diff --git a/nessie/src/test/resources/META-INF/services/org.projectnessie.client.NessieClientBuilder b/nessie/src/test/resources/META-INF/services/org.projectnessie.client.NessieClientBuilder new file mode 100644 index 000000000000..289944f550ea --- /dev/null +++ b/nessie/src/test/resources/META-INF/services/org.projectnessie.client.NessieClientBuilder @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.iceberg.nessie.TestCustomNessieClient$DummyClientBuilderImpl