From 005ecf8395efe062e86cd29b5deec02e37d36405 Mon Sep 17 00:00:00 2001 From: ajantha-bhat Date: Thu, 14 Dec 2023 18:26:12 +0530 Subject: [PATCH] Core: Remove deprecated method from BaseMetadataTable --- .palantir/revapi.yml | 71 +++++++++++++++++++ .../org/apache/iceberg/BaseMetadataTable.java | 12 +--- .../org/apache/iceberg/SerializableTable.java | 2 + .../spark/BaseFileRewriteCoordinator.java | 12 +--- .../iceberg/spark/ScanTaskSetManager.java | 11 +-- .../org/apache/iceberg/spark/Spark3Util.java | 13 ++++ .../spark/BaseFileRewriteCoordinator.java | 12 +--- .../iceberg/spark/ScanTaskSetManager.java | 11 +-- .../org/apache/iceberg/spark/Spark3Util.java | 13 ++++ .../spark/BaseFileRewriteCoordinator.java | 12 +--- .../iceberg/spark/ScanTaskSetManager.java | 11 +-- .../org/apache/iceberg/spark/Spark3Util.java | 13 ++++ 12 files changed, 126 insertions(+), 67 deletions(-) diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 226714261460..11b557653489 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -874,6 +874,74 @@ acceptedBreaks: justification: "Static utility class - should not have public constructor" "1.4.0": org.apache.iceberg:iceberg-core: + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.AllDataFilesTable" + new: "class org.apache.iceberg.AllDataFilesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.AllDeleteFilesTable" + new: "class org.apache.iceberg.AllDeleteFilesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.AllEntriesTable" + new: "class org.apache.iceberg.AllEntriesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.AllFilesTable" + new: "class org.apache.iceberg.AllFilesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.AllManifestsTable" + new: "class org.apache.iceberg.AllManifestsTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.BaseMetadataTable" + new: "class org.apache.iceberg.BaseMetadataTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.DataFilesTable" + new: "class org.apache.iceberg.DataFilesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.DeleteFilesTable" + new: "class org.apache.iceberg.DeleteFilesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.FilesTable" + new: "class org.apache.iceberg.FilesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.HistoryTable" + new: "class org.apache.iceberg.HistoryTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.ManifestEntriesTable" + new: "class org.apache.iceberg.ManifestEntriesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.ManifestsTable" + new: "class org.apache.iceberg.ManifestsTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.MetadataLogEntriesTable" + new: "class org.apache.iceberg.MetadataLogEntriesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.PartitionsTable" + new: "class org.apache.iceberg.PartitionsTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.PositionDeletesTable" + new: "class org.apache.iceberg.PositionDeletesTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.RefsTable" + new: "class org.apache.iceberg.RefsTable" + justification: "Removing deprecated code" + - code: "java.class.noLongerImplementsInterface" + old: "class org.apache.iceberg.SnapshotsTable" + new: "class org.apache.iceberg.SnapshotsTable" + justification: "Removing deprecated code" - code: "java.class.removed" old: "class org.apache.iceberg.actions.BinPackStrategy" justification: "Removing deprecated code" @@ -886,6 +954,9 @@ acceptedBreaks: - code: "java.field.serialVersionUIDChanged" new: "field org.apache.iceberg.util.SerializableMap.serialVersionUID" justification: "Serialization is not be used" + - code: "java.method.removed" + old: "method org.apache.iceberg.TableOperations org.apache.iceberg.BaseMetadataTable::operations()" + justification: "Removing deprecated code" apache-iceberg-0.14.0: org.apache.iceberg:iceberg-api: - code: "java.class.defaultSerializationChanged" diff --git a/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java b/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java index 124115496b3e..c6ec9cb798b1 100644 --- a/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java +++ b/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java @@ -38,8 +38,7 @@ * the metadata table using a {@link StaticTableOperations}. This way no Catalog related calls are * needed when reading the table data after deserialization. */ -public abstract class BaseMetadataTable extends BaseReadOnlyTable - implements HasTableOperations, Serializable { +public abstract class BaseMetadataTable extends BaseReadOnlyTable implements Serializable { private final PartitionSpec spec = PartitionSpec.unpartitioned(); private final SortOrder sortOrder = SortOrder.unsorted(); private final BaseTable table; @@ -99,17 +98,10 @@ static Map transformSpecs( abstract MetadataTableType metadataTableType(); - protected BaseTable table() { + public BaseTable table() { return table; } - /** @deprecated will be removed in 1.4.0; do not use metadata table TableOperations */ - @Override - @Deprecated - public TableOperations operations() { - return table.operations(); - } - @Override public String name() { return name; diff --git a/core/src/main/java/org/apache/iceberg/SerializableTable.java b/core/src/main/java/org/apache/iceberg/SerializableTable.java index 778a993c5144..c08b4f04aafa 100644 --- a/core/src/main/java/org/apache/iceberg/SerializableTable.java +++ b/core/src/main/java/org/apache/iceberg/SerializableTable.java @@ -105,6 +105,8 @@ private String metadataFileLocation(Table table) { if (table instanceof HasTableOperations) { TableOperations ops = ((HasTableOperations) table).operations(); return ops.current().metadataFileLocation(); + } else if (table instanceof BaseMetadataTable) { + return ((BaseMetadataTable) table).table().operations().current().metadataFileLocation(); } else { return null; } diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java index 45c46f1a3e12..6f683df2cdf9 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java @@ -22,9 +22,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.iceberg.ContentFile; -import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.Table; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.Pair; @@ -72,18 +70,12 @@ public void clearRewrite(Table table, String fileSetId) { public Set fetchSetIds(Table table) { return resultMap.keySet().stream() - .filter(e -> e.first().equals(tableUUID(table))) + .filter(e -> e.first().equals(Spark3Util.tableUUID(table))) .map(Pair::second) .collect(Collectors.toSet()); } private Pair toId(Table table, String setId) { - String tableUUID = tableUUID(table); - return Pair.of(tableUUID, setId); - } - - private String tableUUID(Table table) { - TableOperations ops = ((HasTableOperations) table).operations(); - return ops.current().uuid(); + return Pair.of(Spark3Util.tableUUID(table), setId); } } diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java index e8cd7decce51..b89be92c3a6b 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java @@ -22,10 +22,8 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ScanTask; import org.apache.iceberg.Table; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.Pair; @@ -64,17 +62,12 @@ public List removeTasks(Table table, String setId) { public Set fetchSetIds(Table table) { return tasksMap.keySet().stream() - .filter(e -> e.first().equals(tableUUID(table))) + .filter(e -> e.first().equals(Spark3Util.tableUUID(table))) .map(Pair::second) .collect(Collectors.toSet()); } - private String tableUUID(Table table) { - TableOperations ops = ((HasTableOperations) table).operations(); - return ops.current().uuid(); - } - private Pair toId(Table table, String setId) { - return Pair.of(tableUUID(table), setId); + return Pair.of(Spark3Util.tableUUID(table), setId); } } diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index d7717e2bfd49..48d22fc8961c 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -29,6 +29,8 @@ import java.util.stream.Stream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.iceberg.BaseMetadataTable; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.NullOrder; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; @@ -945,6 +947,17 @@ public static org.apache.spark.sql.catalyst.TableIdentifier toV1TableIdentifier( return org.apache.spark.sql.catalyst.TableIdentifier.apply(table, database); } + static String tableUUID(org.apache.iceberg.Table table) { + if (table instanceof HasTableOperations) { + TableOperations ops = ((HasTableOperations) table).operations(); + return ops.current().uuid(); + } else if (table instanceof BaseMetadataTable) { + return ((BaseMetadataTable) table).table().operations().current().uuid(); + } else { + return null; + } + } + private static class DescribeSortOrderVisitor implements SortOrderVisitor { private static final DescribeSortOrderVisitor INSTANCE = new DescribeSortOrderVisitor(); diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java index 45c46f1a3e12..6f683df2cdf9 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java @@ -22,9 +22,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.iceberg.ContentFile; -import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.Table; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.Pair; @@ -72,18 +70,12 @@ public void clearRewrite(Table table, String fileSetId) { public Set fetchSetIds(Table table) { return resultMap.keySet().stream() - .filter(e -> e.first().equals(tableUUID(table))) + .filter(e -> e.first().equals(Spark3Util.tableUUID(table))) .map(Pair::second) .collect(Collectors.toSet()); } private Pair toId(Table table, String setId) { - String tableUUID = tableUUID(table); - return Pair.of(tableUUID, setId); - } - - private String tableUUID(Table table) { - TableOperations ops = ((HasTableOperations) table).operations(); - return ops.current().uuid(); + return Pair.of(Spark3Util.tableUUID(table), setId); } } diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java index e8cd7decce51..b89be92c3a6b 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java @@ -22,10 +22,8 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ScanTask; import org.apache.iceberg.Table; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.Pair; @@ -64,17 +62,12 @@ public List removeTasks(Table table, String setId) { public Set fetchSetIds(Table table) { return tasksMap.keySet().stream() - .filter(e -> e.first().equals(tableUUID(table))) + .filter(e -> e.first().equals(Spark3Util.tableUUID(table))) .map(Pair::second) .collect(Collectors.toSet()); } - private String tableUUID(Table table) { - TableOperations ops = ((HasTableOperations) table).operations(); - return ops.current().uuid(); - } - private Pair toId(Table table, String setId) { - return Pair.of(tableUUID(table), setId); + return Pair.of(Spark3Util.tableUUID(table), setId); } } diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 62301e9676b8..c512e5781c7f 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -29,6 +29,8 @@ import java.util.stream.Stream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.iceberg.BaseMetadataTable; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.NullOrder; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; @@ -948,6 +950,17 @@ public static org.apache.spark.sql.catalyst.TableIdentifier toV1TableIdentifier( return org.apache.spark.sql.catalyst.TableIdentifier.apply(table, database); } + static String tableUUID(org.apache.iceberg.Table table) { + if (table instanceof HasTableOperations) { + TableOperations ops = ((HasTableOperations) table).operations(); + return ops.current().uuid(); + } else if (table instanceof BaseMetadataTable) { + return ((BaseMetadataTable) table).table().operations().current().uuid(); + } else { + return null; + } + } + private static class DescribeSortOrderVisitor implements SortOrderVisitor { private static final DescribeSortOrderVisitor INSTANCE = new DescribeSortOrderVisitor(); diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java index 45c46f1a3e12..6f683df2cdf9 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/BaseFileRewriteCoordinator.java @@ -22,9 +22,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.iceberg.ContentFile; -import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.Table; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.Pair; @@ -72,18 +70,12 @@ public void clearRewrite(Table table, String fileSetId) { public Set fetchSetIds(Table table) { return resultMap.keySet().stream() - .filter(e -> e.first().equals(tableUUID(table))) + .filter(e -> e.first().equals(Spark3Util.tableUUID(table))) .map(Pair::second) .collect(Collectors.toSet()); } private Pair toId(Table table, String setId) { - String tableUUID = tableUUID(table); - return Pair.of(tableUUID, setId); - } - - private String tableUUID(Table table) { - TableOperations ops = ((HasTableOperations) table).operations(); - return ops.current().uuid(); + return Pair.of(Spark3Util.tableUUID(table), setId); } } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java index e8cd7decce51..b89be92c3a6b 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/ScanTaskSetManager.java @@ -22,10 +22,8 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ScanTask; import org.apache.iceberg.Table; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.Pair; @@ -64,17 +62,12 @@ public List removeTasks(Table table, String setId) { public Set fetchSetIds(Table table) { return tasksMap.keySet().stream() - .filter(e -> e.first().equals(tableUUID(table))) + .filter(e -> e.first().equals(Spark3Util.tableUUID(table))) .map(Pair::second) .collect(Collectors.toSet()); } - private String tableUUID(Table table) { - TableOperations ops = ((HasTableOperations) table).operations(); - return ops.current().uuid(); - } - private Pair toId(Table table, String setId) { - return Pair.of(tableUUID(table), setId); + return Pair.of(Spark3Util.tableUUID(table), setId); } } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index cfcc3941c748..62c0bff329b5 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -28,6 +28,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.fs.Path; +import org.apache.iceberg.BaseMetadataTable; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.NullOrder; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; @@ -948,6 +950,17 @@ public static org.apache.spark.sql.catalyst.TableIdentifier toV1TableIdentifier( return org.apache.spark.sql.catalyst.TableIdentifier.apply(table, database); } + static String tableUUID(org.apache.iceberg.Table table) { + if (table instanceof HasTableOperations) { + TableOperations ops = ((HasTableOperations) table).operations(); + return ops.current().uuid(); + } else if (table instanceof BaseMetadataTable) { + return ((BaseMetadataTable) table).table().operations().current().uuid(); + } else { + return null; + } + } + private static class DescribeSortOrderVisitor implements SortOrderVisitor { private static final DescribeSortOrderVisitor INSTANCE = new DescribeSortOrderVisitor();