From ec1d5dd35b3161a86fb1dad5d7dc1553858d76b0 Mon Sep 17 00:00:00 2001 From: ajantha-bhat Date: Tue, 9 Jan 2024 15:46:54 +0530 Subject: [PATCH] Bump Iceberg and Nessie versions - Bump Iceberg to 1.5.0 and Nessie to 0.77.1 - Remove deprecated usage - Fix regression when reading manifest file by overriding newInputfile and passing in the known length Co-authored-by: amogh-jahagirdar --- plugin/trino-iceberg/pom.xml | 4 +-- .../nessie/IcebergNessieCatalogModule.java | 4 +-- .../nessie/IcebergNessieTableOperations.java | 23 +++++++++--- .../iceberg/fileio/ForwardingFileIo.java | 35 +++++++++++++++++++ .../nessie/TestTrinoNessieCatalog.java | 6 ++-- ...ergTrinoRestCatalogConnectorSmokeTest.java | 8 ----- .../iceberg/containers/NessieContainer.java | 4 +-- .../jdbc/TestingTrinoIcebergJdbcUtil.java | 4 +-- pom.xml | 2 +- .../EnvSinglenodeSparkIcebergNessie.java | 6 ++-- 10 files changed, 69 insertions(+), 27 deletions(-) diff --git a/plugin/trino-iceberg/pom.xml b/plugin/trino-iceberg/pom.xml index 6bf4f21137bc..bc95e44086ac 100644 --- a/plugin/trino-iceberg/pom.xml +++ b/plugin/trino-iceberg/pom.xml @@ -16,7 +16,7 @@ ${project.parent.basedir} - 0.71.1 + 0.77.1 @@ -338,7 +338,7 @@ org.apache.httpcomponents.client5 httpclient5 - 5.2.3 + 5.3.1 runtime diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieCatalogModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieCatalogModule.java index 7a2008dfc0c4..a80eb8a1125c 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieCatalogModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieCatalogModule.java @@ -22,9 +22,9 @@ import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; import org.apache.iceberg.nessie.NessieIcebergClient; +import org.projectnessie.client.NessieClientBuilder; import org.projectnessie.client.api.NessieApiV1; import org.projectnessie.client.auth.BearerAuthenticationProvider; -import org.projectnessie.client.http.HttpClientBuilder; import static io.airlift.configuration.ConfigBinder.configBinder; import static java.lang.Math.toIntExact; @@ -47,7 +47,7 @@ protected void setup(Binder binder) @Singleton public static NessieIcebergClient createNessieIcebergClient(IcebergNessieCatalogConfig icebergNessieCatalogConfig) { - HttpClientBuilder builder = HttpClientBuilder.builder() + NessieClientBuilder builder = NessieClientBuilder.createClientBuilderFromSystemSettings() .withUri(icebergNessieCatalogConfig.getServerUri()) .withDisableCompression(!icebergNessieCatalogConfig.isCompressionEnabled()) .withReadTimeout(toIntExact(icebergNessieCatalogConfig.getReadTimeout().toMillis())) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieTableOperations.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieTableOperations.java index 3fe3c5b33dc4..16a9e3eef7c5 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieTableOperations.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieTableOperations.java @@ -108,8 +108,14 @@ protected String getRefreshedLocation(boolean invalidateCaches) protected void commitNewTable(TableMetadata metadata) { verify(version.isEmpty(), "commitNewTable called on a table which already exists"); + String contentId = table == null ? null : table.getId(); try { - nessieClient.commitTable(null, metadata, writeNewMetadata(metadata, 0), table, toKey(new SchemaTableName(database, this.tableName))); + nessieClient.commitTable( + null, + metadata, + writeNewMetadata(metadata, 0), + contentId, + toKey(database, tableName)); } catch (NessieNotFoundException e) { throw new TrinoException(ICEBERG_COMMIT_ERROR, format("Cannot commit: ref '%s' no longer exists", nessieClient.refName()), e); @@ -125,8 +131,16 @@ protected void commitNewTable(TableMetadata metadata) protected void commitToExistingTable(TableMetadata base, TableMetadata metadata) { verify(version.orElseThrow() >= 0, "commitToExistingTable called on a new table"); + if (table == null) { + table = nessieClient.table(toIdentifier(new SchemaTableName(database, tableName))); + } try { - nessieClient.commitTable(base, metadata, writeNewMetadata(metadata, version.getAsInt() + 1), table, toKey(new SchemaTableName(database, this.tableName))); + nessieClient.commitTable( + base, + metadata, + writeNewMetadata(metadata, version.getAsInt() + 1), + table.getId(), + toKey(database, tableName)); } catch (NessieNotFoundException e) { throw new TrinoException(ICEBERG_COMMIT_ERROR, format("Cannot commit: ref '%s' no longer exists", nessieClient.refName()), e); @@ -144,8 +158,9 @@ protected void commitMaterializedViewRefresh(TableMetadata base, TableMetadata m throw new UnsupportedOperationException(); } - private static ContentKey toKey(SchemaTableName tableName) + private static ContentKey toKey(String databaseName, String tableName) { - return ContentKey.of(Namespace.parse(tableName.getSchemaName()), tableName.getTableName()); + SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName); + return ContentKey.of(Namespace.parse(schemaTableName.getSchemaName()), schemaTableName.getTableName()); } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/fileio/ForwardingFileIo.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/fileio/ForwardingFileIo.java index cfb511c3fccb..343de1853e34 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/fileio/ForwardingFileIo.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/fileio/ForwardingFileIo.java @@ -17,6 +17,9 @@ import com.google.common.collect.Iterables; import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.ManifestFile; import org.apache.iceberg.io.BulkDeletionFailureException; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.io.OutputFile; @@ -28,6 +31,7 @@ import java.util.Map; import java.util.stream.Stream; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; @@ -88,6 +92,37 @@ public void deleteFiles(Iterable pathsToDelete) partitions.forEach(this::deleteBatch); } + // TODO: remove below workarounds after https://github.com/apache/iceberg/pull/9953 + @Override + public InputFile newInputFile(ManifestFile manifest) + { + checkArgument( + manifest.keyMetadata() == null, + "Cannot decrypt manifest: %s (use EncryptingFileIO)", + manifest.path()); + return newInputFile(manifest.path(), manifest.length()); + } + + @Override + public InputFile newInputFile(DataFile file) + { + checkArgument( + file.keyMetadata() == null, + "Cannot decrypt data file: %s (use EncryptingFileIO)", + file.path()); + return newInputFile(file.path().toString(), file.fileSizeInBytes()); + } + + @Override + public InputFile newInputFile(DeleteFile file) + { + checkArgument( + file.keyMetadata() == null, + "Cannot decrypt delete file: %s (use EncryptingFileIO)", + file.path()); + return newInputFile(file.path().toString(), file.fileSizeInBytes()); + } + private void deleteBatch(List filesToDelete) { try { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java index b9a64fe3eba8..b9c7fe2568f5 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java @@ -36,8 +36,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.parallel.Execution; +import org.projectnessie.client.NessieClientBuilder; import org.projectnessie.client.api.NessieApiV1; -import org.projectnessie.client.http.HttpClientBuilder; import java.io.File; import java.io.IOException; @@ -94,7 +94,7 @@ protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) TrinoFileSystemFactory fileSystemFactory = new HdfsFileSystemFactory(HDFS_ENVIRONMENT, HDFS_FILE_SYSTEM_STATS); IcebergNessieCatalogConfig icebergNessieCatalogConfig = new IcebergNessieCatalogConfig() .setServerUri(URI.create(nessieContainer.getRestApiUri())); - NessieApiV1 nessieApi = HttpClientBuilder.builder() + NessieApiV1 nessieApi = NessieClientBuilder.createClientBuilderFromSystemSettings() .withUri(nessieContainer.getRestApiUri()) .build(NessieApiV1.class); NessieIcebergClient nessieClient = new NessieIcebergClient(nessieApi, icebergNessieCatalogConfig.getDefaultReferenceName(), null, ImmutableMap.of()); @@ -118,7 +118,7 @@ public void testDefaultLocation() IcebergNessieCatalogConfig icebergNessieCatalogConfig = new IcebergNessieCatalogConfig() .setDefaultWarehouseDir(tmpDirectory.toAbsolutePath().toString()) .setServerUri(URI.create(nessieContainer.getRestApiUri())); - NessieApiV1 nessieApi = HttpClientBuilder.builder() + NessieApiV1 nessieApi = NessieClientBuilder.createClientBuilderFromSystemSettings() .withUri(nessieContainer.getRestApiUri()) .build(NessieApiV1.class); NessieIcebergClient nessieClient = new NessieIcebergClient(nessieApi, icebergNessieCatalogConfig.getDefaultReferenceName(), null, ImmutableMap.of()); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergTrinoRestCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergTrinoRestCatalogConnectorSmokeTest.java index 1176aeb9d967..81b50106f966 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergTrinoRestCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergTrinoRestCatalogConnectorSmokeTest.java @@ -183,14 +183,6 @@ public void testDropTableWithMissingManifestListFile() .hasMessageContaining("Table location should not exist"); } - @Test - @Override - public void testDropTableWithMissingDataFile() - { - assertThatThrownBy(super::testDropTableWithMissingDataFile) - .hasMessageContaining("Table location should not exist"); - } - @Test @Override public void testDropTableWithNonExistentTableLocation() diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java index db79a4e19d62..e633455d92ce 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java @@ -28,9 +28,9 @@ public class NessieContainer { private static final Logger log = Logger.get(NessieContainer.class); - public static final String DEFAULT_IMAGE = "projectnessie/nessie:0.71.1"; + public static final String DEFAULT_IMAGE = "ghcr.io/projectnessie/nessie:0.77.1"; public static final String DEFAULT_HOST_NAME = "nessie"; - public static final String VERSION_STORE_TYPE = "INMEMORY"; + public static final String VERSION_STORE_TYPE = "IN_MEMORY"; public static final int PORT = 19121; diff --git a/plugin/trino-iceberg/src/test/java/org/apache/iceberg/jdbc/TestingTrinoIcebergJdbcUtil.java b/plugin/trino-iceberg/src/test/java/org/apache/iceberg/jdbc/TestingTrinoIcebergJdbcUtil.java index f15d86b24b66..4967d9708188 100644 --- a/plugin/trino-iceberg/src/test/java/org/apache/iceberg/jdbc/TestingTrinoIcebergJdbcUtil.java +++ b/plugin/trino-iceberg/src/test/java/org/apache/iceberg/jdbc/TestingTrinoIcebergJdbcUtil.java @@ -16,8 +16,8 @@ public final class TestingTrinoIcebergJdbcUtil { - public static final String CREATE_CATALOG_TABLE = JdbcUtil.CREATE_CATALOG_TABLE; - public static final String CREATE_NAMESPACE_PROPERTIES_TABLE = JdbcUtil.CREATE_NAMESPACE_PROPERTIES_TABLE; + public static final String CREATE_CATALOG_TABLE = JdbcUtil.V0_CREATE_CATALOG_SQL; + public static final String CREATE_NAMESPACE_PROPERTIES_TABLE = JdbcUtil.CREATE_NAMESPACE_PROPERTIES_TABLE_SQL; private TestingTrinoIcebergJdbcUtil() {} } diff --git a/pom.xml b/pom.xml index fac15c1e8628..9aa39d2623aa 100644 --- a/pom.xml +++ b/pom.xml @@ -191,7 +191,7 @@ 2.25.0 10.9.0 1.44.1 - 1.4.3 + 1.5.0 5.14.0 2.12.7 0.12.5 diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java index b2c8634c79e4..7908428b5b00 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java @@ -43,7 +43,7 @@ public class EnvSinglenodeSparkIcebergNessie private static final int SPARK_THRIFT_PORT = 10213; private static final int NESSIE_PORT = 19120; - private static final String NESSIE_VERSION = "0.71.1"; + private static final String NESSIE_VERSION = "0.77.1"; private static final String SPARK = "spark"; private final DockerFiles dockerFiles; @@ -99,8 +99,8 @@ private DockerContainer createSparkContainer() private DockerContainer createNessieContainer() { - DockerContainer container = new DockerContainer("projectnessie/nessie:" + NESSIE_VERSION, "nessie-server") - .withEnv("NESSIE_VERSION_STORE_TYPE", "INMEMORY") + DockerContainer container = new DockerContainer("ghcr.io/projectnessie/nessie:" + NESSIE_VERSION, "nessie-server") + .withEnv("NESSIE_VERSION_STORE_TYPE", "IN_MEMORY") .withEnv("QUARKUS_HTTP_PORT", Integer.valueOf(NESSIE_PORT).toString()) .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) .waitingFor(forSelectedPorts(NESSIE_PORT));