From 2a0d9f0a6cce1e6dd2feb1dd7374b4363c609fdb Mon Sep 17 00:00:00 2001 From: Ajantha Bhat Date: Thu, 18 Jan 2024 21:40:49 +0530 Subject: [PATCH] Core: Remove deprecated operations method from BaseMetadataTable (#9298) --- .palantir/revapi.yml | 71 +++++++++++++++++++ .../org/apache/iceberg/BaseMetadataTable.java | 12 +--- .../org/apache/iceberg/SerializableTable.java | 2 + .../spark/BaseFileRewriteCoordinator.java | 12 +--- .../iceberg/spark/ScanTaskSetManager.java | 11 +-- .../org/apache/iceberg/spark/Spark3Util.java | 13 ++++ .../spark/BaseFileRewriteCoordinator.java | 12 +--- .../iceberg/spark/ScanTaskSetManager.java | 11 +-- .../org/apache/iceberg/spark/Spark3Util.java | 13 ++++ .../spark/BaseFileRewriteCoordinator.java | 12 +--- .../iceberg/spark/ScanTaskSetManager.java | 11 +-- .../org/apache/iceberg/spark/Spark3Util.java | 13 ++++ 12 files changed, 126 insertions(+), 67 deletions(-) diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 990c9ba31afd..c5c92dba0d29 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -874,6 +874,74 @@ acceptedBreaks: justification: "Static utility class - should not have public constructor" "1.4.0": org.apache.iceberg:iceberg-core: + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.AllDataFilesTable" + new: "class org.apache.iceberg.AllDataFilesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.AllDeleteFilesTable" + new: "class org.apache.iceberg.AllDeleteFilesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.AllEntriesTable" + new: "class org.apache.iceberg.AllEntriesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.AllFilesTable" + new: "class org.apache.iceberg.AllFilesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.AllManifestsTable" + new: "class org.apache.iceberg.AllManifestsTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.BaseMetadataTable" + new: "class org.apache.iceberg.BaseMetadataTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.DataFilesTable" + new: "class org.apache.iceberg.DataFilesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.DeleteFilesTable" + new: "class org.apache.iceberg.DeleteFilesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.FilesTable" + new: "class org.apache.iceberg.FilesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.HistoryTable" + new: "class org.apache.iceberg.HistoryTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.ManifestEntriesTable" + new: "class org.apache.iceberg.ManifestEntriesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.ManifestsTable" + new: "class org.apache.iceberg.ManifestsTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.MetadataLogEntriesTable" + new: "class org.apache.iceberg.MetadataLogEntriesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.PartitionsTable" + new: "class org.apache.iceberg.PartitionsTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.PositionDeletesTable" + new: "class org.apache.iceberg.PositionDeletesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.RefsTable" + new: "class org.apache.iceberg.RefsTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.SnapshotsTable" + new: "class org.apache.iceberg.SnapshotsTable" + justification: "Removing deprecated code" - code: "java.class.defaultSerializationChanged" old: "class org.apache.iceberg.mapping.NameMapping" new: "class org.apache.iceberg.mapping.NameMapping" @@ -890,6 +958,9 @@ acceptedBreaks: - code: "java.field.serialVersionUIDChanged" new: "field org.apache.iceberg.util.SerializableMap.serialVersionUID" justification: "Serialization is not be used" + - code: "java.method.removed" + old: "method org.apache.iceberg.TableOperations org.apache.iceberg.BaseMetadataTable::operations()" + justification: "Removing deprecated code" apache-iceberg-0.14.0: org.apache.iceberg:iceberg-api: - code: "java.class.defaultSerializationChanged" diff --git a/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java b/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java index 28e3b1ead69d..57a6386093d6 100644 --- a/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java +++ b/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java @@ -38,8 +38,7 @@ * the metadata table using a {@link StaticTableOperations}. This way no Catalog related calls are * needed when reading the table data after deserialization. */ -public abstract class BaseMetadataTable extends BaseReadOnlyTable - implements HasTableOperations, Serializable { +public abstract class BaseMetadataTable extends BaseReadOnlyTable implements Serializable { private final PartitionSpec spec = PartitionSpec.unpartitioned(); private final SortOrder sortOrder = SortOrder.unsorted(); private final BaseTable table; @@ -101,17 +100,10 @@ static Map transformSpecs( abstract MetadataTableType metadataTableType(); - protected BaseTable table() { + public BaseTable table() { return table; } - /** @deprecated will be removed in 1.4.0; do not use metadata table TableOperations */ - @Override - @Deprecated - public TableOperations operations() { - return table.operations(); - } - @Override public String name() { return name; diff --git a/core/src/main/java/org/apache/iceberg/SerializableTable.java b/core/src/main/java/org/apache/iceberg/SerializableTable.java index 5a98ddbaf993..42e7af72d4e8 100644 --- a/core/src/main/java/org/apache/iceberg/SerializableTable.java +++ b/core/src/main/java/org/apache/iceberg/SerializableTable.java @@ -105,6 +105,8 @@ private String metadataFileLocation(Table table) { if (table instanceof HasTableOperations) { TableOperations ops = ((HasTableOperations) table).operations(); return ops.current().metadataFileLocation(); + } else if (table instanceof BaseMetadataTable) { + return ((BaseMetadataTable) table).table().operations().current().metadataFileLocation(); } else { return null; } diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java index 45c46f1a3e12..5c95475d3302 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java @@ -22,9 +22,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.iceberg.ContentFile; -import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.Table; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.Pair; @@ -72,18 +70,12 @@ public void clearRewrite(Table table, String fileSetId) { public Set fetchSetIds(Table table) { return resultMap.keySet().stream() - .filter(e -> e.first().equals(tableUUID(table))) + .filter(e -> e.first().equals(Spark3Util.baseTableUUID(table))) .map(Pair::second) .collect(Collectors.toSet()); } private Pair toId(Table table, String setId) { - String tableUUID = tableUUID(table); - return Pair.of(tableUUID, setId); - } - - private String tableUUID(Table table) { - TableOperations ops = ((HasTableOperations) table).operations(); - return ops.current().uuid(); + return Pair.of(Spark3Util.baseTableUUID(table), setId); } } diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java index e8cd7decce51..cab40d103171 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java @@ -22,10 +22,8 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ScanTask; import org.apache.iceberg.Table; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.Pair; @@ -64,17 +62,12 @@ public List removeTasks(Table table, String setId) { public Set fetchSetIds(Table table) { return tasksMap.keySet().stream() - .filter(e -> e.first().equals(tableUUID(table))) + .filter(e -> e.first().equals(Spark3Util.baseTableUUID(table))) .map(Pair::second) .collect(Collectors.toSet()); } - private String tableUUID(Table table) { - TableOperations ops = ((HasTableOperations) table).operations(); - return ops.current().uuid(); - } - private Pair toId(Table table, String setId) { - return Pair.of(tableUUID(table), setId); + return Pair.of(Spark3Util.baseTableUUID(table), setId); } } diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index d7717e2bfd49..10b856a7acf0 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -29,6 +29,8 @@ import java.util.stream.Stream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.iceberg.BaseMetadataTable; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.NullOrder; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; @@ -945,6 +947,17 @@ public static org.apache.spark.sql.catalyst.TableIdentifier toV1TableIdentifier( return org.apache.spark.sql.catalyst.TableIdentifier.apply(table, database); } + static String baseTableUUID(org.apache.iceberg.Table table) { + if (table instanceof HasTableOperations) { + TableOperations ops = ((HasTableOperations) table).operations(); + return ops.current().uuid(); + } else if (table instanceof BaseMetadataTable) { + return ((BaseMetadataTable) table).table().operations().current().uuid(); + } else { + throw new UnsupportedOperationException("Cannot retrieve UUID for table " + table.name()); + } + } + private static class DescribeSortOrderVisitor implements SortOrderVisitor { private static final DescribeSortOrderVisitor INSTANCE = new DescribeSortOrderVisitor(); diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java index 45c46f1a3e12..5c95475d3302 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java @@ -22,9 +22,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.iceberg.ContentFile; -import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.Table; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.Pair; @@ -72,18 +70,12 @@ public void clearRewrite(Table table, String fileSetId) { public Set fetchSetIds(Table table) { return resultMap.keySet().stream() - .filter(e -> e.first().equals(tableUUID(table))) + .filter(e -> e.first().equals(Spark3Util.baseTableUUID(table))) .map(Pair::second) .collect(Collectors.toSet()); } private Pair toId(Table table, String setId) { - String tableUUID = tableUUID(table); - return Pair.of(tableUUID, setId); - } - - private String tableUUID(Table table) { - TableOperations ops = ((HasTableOperations) table).operations(); - return ops.current().uuid(); + return Pair.of(Spark3Util.baseTableUUID(table), setId); } } diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java index e8cd7decce51..cab40d103171 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java @@ -22,10 +22,8 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ScanTask; import org.apache.iceberg.Table; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.Pair; @@ -64,17 +62,12 @@ public List removeTasks(Table table, String setId) { public Set fetchSetIds(Table table) { return tasksMap.keySet().stream() - .filter(e -> e.first().equals(tableUUID(table))) + .filter(e -> e.first().equals(Spark3Util.baseTableUUID(table))) .map(Pair::second) .collect(Collectors.toSet()); } - private String tableUUID(Table table) { - TableOperations ops = ((HasTableOperations) table).operations(); - return ops.current().uuid(); - } - private Pair toId(Table table, String setId) { - return Pair.of(tableUUID(table), setId); + return Pair.of(Spark3Util.baseTableUUID(table), setId); } } diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 62301e9676b8..a81877c98b3d 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -29,6 +29,8 @@ import java.util.stream.Stream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.iceberg.BaseMetadataTable; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.NullOrder; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; @@ -948,6 +950,17 @@ public static org.apache.spark.sql.catalyst.TableIdentifier toV1TableIdentifier( return org.apache.spark.sql.catalyst.TableIdentifier.apply(table, database); } + static String baseTableUUID(org.apache.iceberg.Table table) { + if (table instanceof HasTableOperations) { + TableOperations ops = ((HasTableOperations) table).operations(); + return ops.current().uuid(); + } else if (table instanceof BaseMetadataTable) { + return ((BaseMetadataTable) table).table().operations().current().uuid(); + } else { + throw new UnsupportedOperationException("Cannot retrieve UUID for table " + table.name()); + } + } + private static class DescribeSortOrderVisitor implements SortOrderVisitor { private static final DescribeSortOrderVisitor INSTANCE = new DescribeSortOrderVisitor(); diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java index 45c46f1a3e12..5c95475d3302 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java @@ -22,9 +22,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.iceberg.ContentFile; -import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.Table; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.Pair; @@ -72,18 +70,12 @@ public void clearRewrite(Table table, String fileSetId) { public Set fetchSetIds(Table table) { return resultMap.keySet().stream() - .filter(e -> e.first().equals(tableUUID(table))) + .filter(e -> e.first().equals(Spark3Util.baseTableUUID(table))) .map(Pair::second) .collect(Collectors.toSet()); } private Pair toId(Table table, String setId) { - String tableUUID = tableUUID(table); - return Pair.of(tableUUID, setId); - } - - private String tableUUID(Table table) { - TableOperations ops = ((HasTableOperations) table).operations(); - return ops.current().uuid(); + return Pair.of(Spark3Util.baseTableUUID(table), setId); } } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java index e8cd7decce51..cab40d103171 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java @@ -22,10 +22,8 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ScanTask; import org.apache.iceberg.Table; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.Pair; @@ -64,17 +62,12 @@ public List removeTasks(Table table, String setId) { public Set fetchSetIds(Table table) { return tasksMap.keySet().stream() - .filter(e -> e.first().equals(tableUUID(table))) + .filter(e -> e.first().equals(Spark3Util.baseTableUUID(table))) .map(Pair::second) .collect(Collectors.toSet()); } - private String tableUUID(Table table) { - TableOperations ops = ((HasTableOperations) table).operations(); - return ops.current().uuid(); - } - private Pair toId(Table table, String setId) { - return Pair.of(tableUUID(table), setId); + return Pair.of(Spark3Util.baseTableUUID(table), setId); } } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index cfcc3941c748..39518ffbb945 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -28,6 +28,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.fs.Path; +import org.apache.iceberg.BaseMetadataTable; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.NullOrder; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; @@ -948,6 +950,17 @@ public static org.apache.spark.sql.catalyst.TableIdentifier toV1TableIdentifier( return org.apache.spark.sql.catalyst.TableIdentifier.apply(table, database); } + static String baseTableUUID(org.apache.iceberg.Table table) { + if (table instanceof HasTableOperations) { + TableOperations ops = ((HasTableOperations) table).operations(); + return ops.current().uuid(); + } else if (table instanceof BaseMetadataTable) { + return ((BaseMetadataTable) table).table().operations().current().uuid(); + } else { + throw new UnsupportedOperationException("Cannot retrieve UUID for table " + table.name()); + } + } + private static class DescribeSortOrderVisitor implements SortOrderVisitor { private static final DescribeSortOrderVisitor INSTANCE = new DescribeSortOrderVisitor();