From 7c4640dec1dd6f25fe68fc574f4cf7018a015b98 Mon Sep 17 00:00:00 2001 From: roryqi Date: Sat, 2 Sep 2023 18:29:13 +0800 Subject: [PATCH 01/15] Test: Add a test utility method to programmatically create expected partition specs --- .../java/org/apache/iceberg/TestHelpers.java | 39 +++++++++++ .../TestAlterTablePartitionFields.java | 66 ++++++------------- .../source/TestForwardCompatibility.java | 18 +++-- ...tMetadataTablesWithPartitionEvolution.java | 10 +-- .../source/TestSparkMetadataColumns.java | 10 +-- 5 files changed, 83 insertions(+), 60 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/TestHelpers.java b/api/src/test/java/org/apache/iceberg/TestHelpers.java index 153e2de7ea9a..5dce78c342eb 100644 --- a/api/src/test/java/org/apache/iceberg/TestHelpers.java +++ b/api/src/test/java/org/apache/iceberg/TestHelpers.java @@ -667,4 +667,43 @@ public List splitOffsets() { return null; } } + + public static PartitionSpecBuilder newPartitionSpecBuilder() { + return new PartitionSpecBuilder(); + } + + public static class PartitionSpecBuilder { + private final UnboundPartitionSpec.Builder unboundPartitionSpecBuilder; + private Schema schema; + + private PartitionSpecBuilder() { + this.unboundPartitionSpecBuilder = UnboundPartitionSpec.builder(); + } + + public PartitionSpecBuilder withSchema(Schema newSchema) { + this.schema = newSchema; + return this; + } + + public PartitionSpecBuilder withSpecId(int newSpecId) { + unboundPartitionSpecBuilder.withSpecId(newSpecId); + return this; + } + + public PartitionSpecBuilder addField( + String transformAsString, int sourceId, int partitionId, String name) { + unboundPartitionSpecBuilder.addField(transformAsString, sourceId, partitionId, name); + return this; + } + + public PartitionSpecBuilder addField(String transformAsString, int sourceId, String name) { + unboundPartitionSpecBuilder.addField(transformAsString, sourceId, name); + return this; + } + + public PartitionSpec build() { + Preconditions.checkNotNull(schema, "Field schema is missing"); + return unboundPartitionSpecBuilder.build().bind(schema); + } + } } diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index a43f2a041b97..65c0e1260d56 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -19,9 +19,9 @@ package org.apache.iceberg.spark.extensions; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.spark.source.SparkTable; import org.apache.spark.sql.connector.catalog.CatalogManager; @@ -315,17 +315,11 @@ public void testReplacePartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -354,17 +348,11 @@ public void testReplacePartitionAndRename() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -393,17 +381,11 @@ public void testReplaceNamedPartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -432,17 +414,11 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index 73e572ecaecd..bc3646f9bf5e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -34,7 +34,6 @@ import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.ManifestWriter; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; @@ -75,14 +74,19 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("zero", 1, "id_zero") + .build(); + // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"identity\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("identity", 1, "id_zero") + .build(); @Rule public TemporaryFolder temp = new TemporaryFolder(); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index b4b60dc02dd5..64171b9e55d8 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -37,10 +37,10 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; @@ -627,9 +627,11 @@ public void testMetadataTablesWithUnknownTransforms() { Table table = validationCatalog.loadTable(tableIdent); PartitionSpec unknownSpec = - PartitionSpecParser.fromJson( - table.schema(), - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); // replace the table spec to include an unknown transform TableOperations ops = ((HasTableOperations) table).operations(); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 8a2fae811750..6cc7290bdb36 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -34,11 +34,11 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -75,9 +75,11 @@ public class TestSparkMetadataColumns extends SparkTestBase { Types.NestedField.optional(3, "data", Types.StringType.get())); private static final PartitionSpec SPEC = PartitionSpec.unpartitioned(); private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); @Parameterized.Parameters(name = "fileFormat = {0}, vectorized = {1}, formatVersion = {2}") public static Object[][] parameters() { From 97ca713e69df05d452947853ec1210d79636a3c8 Mon Sep 17 00:00:00 2001 From: roryqi Date: Sat, 2 Sep 2023 18:37:43 +0800 Subject: [PATCH 02/15] Add Spark 3.3 --- .../TestAlterTablePartitionFields.java | 68 ++++++------------- .../source/TestForwardCompatibility.java | 17 +++-- ...tMetadataTablesWithPartitionEvolution.java | 8 ++- .../source/TestSparkMetadataColumns.java | 8 ++- 4 files changed, 41 insertions(+), 60 deletions(-) diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index a43f2a041b97..9bbc10a2dc03 100644 --- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -314,18 +314,11 @@ public void testReplacePartition() { .hour("ts") .build(); } else { - expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + expected = TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -353,18 +346,11 @@ public void testReplacePartitionAndRename() { .hour("ts", "hour_col") .build(); } else { - expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + expected = TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -392,18 +378,11 @@ public void testReplaceNamedPartition() { .hour("ts") .build(); } else { - expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + expected = TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -431,18 +410,11 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .hour("ts", "hour_col") .build(); } else { - expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + expected = TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index fe440235901c..9d31c615860d 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -75,14 +75,19 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("zero", 1, "id_zero") + .build(); + // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"identity\", \"source-id\": 1 } ] }"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("zero", 1, "id_zero") + .build(); @Rule public TemporaryFolder temp = new TemporaryFolder(); diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index a62199181a3a..3c54a73ec7c9 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -627,9 +627,11 @@ public void testMetadataTablesWithUnknownTransforms() { Table table = validationCatalog.loadTable(tableIdent); PartitionSpec unknownSpec = - PartitionSpecParser.fromJson( - table.schema(), - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); // replace the table spec to include an unknown transform TableOperations ops = ((HasTableOperations) table).operations(); diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 79b755872dc3..53bf32264538 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -75,9 +75,11 @@ public class TestSparkMetadataColumns extends SparkTestBase { Types.NestedField.optional(3, "data", Types.StringType.get())); private static final PartitionSpec SPEC = PartitionSpec.unpartitioned(); private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); @Parameterized.Parameters(name = "fileFormat = {0}, vectorized = {1}, formatVersion = {2}") public static Object[][] parameters() { From b090d27ca6ec8a0fa11e25f3e1d246fe57c0e201 Mon Sep 17 00:00:00 2001 From: roryqi Date: Sat, 2 Sep 2023 18:44:50 +0800 Subject: [PATCH 03/15] Add Spark 3.2 --- .../TestAlterTablePartitionFields.java | 64 ++++++------------- .../source/TestForwardCompatibility.java | 16 +++-- ...tMetadataTablesWithPartitionEvolution.java | 8 ++- .../source/TestSparkMetadataColumns.java | 8 ++- 4 files changed, 40 insertions(+), 56 deletions(-) diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 9b4bd12ec1bf..7be92bf0ee86 100644 --- a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -315,17 +315,11 @@ public void testReplacePartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -354,17 +348,11 @@ public void testReplacePartitionAndRename() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -393,17 +381,11 @@ public void testReplaceNamedPartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -432,17 +414,11 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index fe440235901c..02621699d096 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -75,14 +75,18 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("zero", 1, "id_zero") + .build(); // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"identity\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("identity", 1, "id_zero") + .build(); @Rule public TemporaryFolder temp = new TemporaryFolder(); diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index a62199181a3a..3c54a73ec7c9 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -627,9 +627,11 @@ public void testMetadataTablesWithUnknownTransforms() { Table table = validationCatalog.loadTable(tableIdent); PartitionSpec unknownSpec = - PartitionSpecParser.fromJson( - table.schema(), - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); // replace the table spec to include an unknown transform TableOperations ops = ((HasTableOperations) table).operations(); diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 79b755872dc3..53bf32264538 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -75,9 +75,11 @@ public class TestSparkMetadataColumns extends SparkTestBase { Types.NestedField.optional(3, "data", Types.StringType.get())); private static final PartitionSpec SPEC = PartitionSpec.unpartitioned(); private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); @Parameterized.Parameters(name = "fileFormat = {0}, vectorized = {1}, formatVersion = {2}") public static Object[][] parameters() { From 0340c8bf7f9fcba4a22b18f001d265d9a1eedca2 Mon Sep 17 00:00:00 2001 From: roryqi Date: Sat, 2 Sep 2023 18:49:26 +0800 Subject: [PATCH 04/15] Add Spark 3.1 --- .../TestAlterTablePartitionFields.java | 64 ++++++------------- .../source/TestForwardCompatibility.java | 17 +++-- ...tMetadataTablesWithPartitionEvolution.java | 8 ++- .../source/TestSparkMetadataColumns.java | 8 ++- 4 files changed, 41 insertions(+), 56 deletions(-) diff --git a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 5102da844242..2c0d4389da8c 100644 --- a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -315,17 +315,11 @@ public void testReplacePartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -354,17 +348,11 @@ public void testReplacePartitionAndRename() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -393,17 +381,11 @@ public void testReplaceNamedPartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -432,17 +414,11 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index 9ca073ecf338..d3ec2d19aab0 100644 --- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -75,14 +75,19 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("zero", 1, "id_zero") + .build(); + // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"identity\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("identity", 1, "id_zero") + .build(); @Rule public TemporaryFolder temp = new TemporaryFolder(); diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index 1245e1fc1fe9..de57a18c9673 100644 --- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -256,9 +256,11 @@ public void testMetadataTablesWithUnknownTransforms() { Table table = validationCatalog.loadTable(tableIdent); PartitionSpec unknownSpec = - PartitionSpecParser.fromJson( - table.schema(), - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); // replace the table spec to include an unknown transform TableOperations ops = ((HasTableOperations) table).operations(); diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 1b7098827f9f..31542f3e934a 100644 --- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -65,9 +65,11 @@ public class TestSparkMetadataColumns extends SparkTestBase { Types.NestedField.optional(3, "data", Types.StringType.get())); private static final PartitionSpec SPEC = PartitionSpec.unpartitioned(); private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); @Parameterized.Parameters(name = "fileFormat = {0}, vectorized = {1}, formatVersion = {2}") public static Object[][] parameters() { From 6e0643276010bab1590632a9323193edcb06e3c6 Mon Sep 17 00:00:00 2001 From: roryqi Date: Sat, 2 Sep 2023 19:08:56 +0800 Subject: [PATCH 05/15] Fix code styles --- .../iceberg/spark/extensions/TestAlterTablePartitionFields.java | 1 - .../apache/iceberg/spark/source/TestForwardCompatibility.java | 1 - .../spark/source/TestMetadataTablesWithPartitionEvolution.java | 1 - .../apache/iceberg/spark/source/TestSparkMetadataColumns.java | 1 - .../iceberg/spark/extensions/TestAlterTablePartitionFields.java | 1 - .../apache/iceberg/spark/source/TestForwardCompatibility.java | 1 - .../spark/source/TestMetadataTablesWithPartitionEvolution.java | 1 - .../apache/iceberg/spark/source/TestSparkMetadataColumns.java | 1 - .../iceberg/spark/extensions/TestAlterTablePartitionFields.java | 1 - .../apache/iceberg/spark/source/TestForwardCompatibility.java | 1 - .../spark/source/TestMetadataTablesWithPartitionEvolution.java | 1 - .../apache/iceberg/spark/source/TestSparkMetadataColumns.java | 1 - 12 files changed, 12 deletions(-) diff --git a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 2c0d4389da8c..eab30f3572ee 100644 --- a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -19,7 +19,6 @@ package org.apache.iceberg.spark.extensions; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.spark.SparkCatalogConfig; diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index d3ec2d19aab0..486955612149 100644 --- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -35,7 +35,6 @@ import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.ManifestWriter; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index de57a18c9673..ed684f9e0069 100644 --- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -38,7 +38,6 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 31542f3e934a..5ffec7a97e58 100644 --- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -31,7 +31,6 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 7be92bf0ee86..7b1a78f135ba 100644 --- a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -19,7 +19,6 @@ package org.apache.iceberg.spark.extensions; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.spark.SparkCatalogConfig; diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index 02621699d096..5c230cd693ff 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -35,7 +35,6 @@ import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.ManifestWriter; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index 3c54a73ec7c9..fe5393701110 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -38,7 +38,6 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 53bf32264538..bb9bdd15dece 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -35,7 +35,6 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 9bbc10a2dc03..521f0ded7bca 100644 --- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -19,7 +19,6 @@ package org.apache.iceberg.spark.extensions; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.spark.SparkCatalogConfig; diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index 9d31c615860d..0ef6b4f0cbcc 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -35,7 +35,6 @@ import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.ManifestWriter; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index 3c54a73ec7c9..fe5393701110 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -38,7 +38,6 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 53bf32264538..bb9bdd15dece 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -35,7 +35,6 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; From 1d542fd0b0f9c3b26e02dc73ce30fb4487e8db93 Mon Sep 17 00:00:00 2001 From: roryqi Date: Sat, 2 Sep 2023 19:30:37 +0800 Subject: [PATCH 06/15] Fix import issues --- .../extensions/TestAlterTablePartitionFields.java | 1 + .../TestMetadataTablesWithPartitionEvolution.java | 1 + .../spark/source/TestSparkMetadataColumns.java | 1 + .../extensions/TestAlterTablePartitionFields.java | 1 + .../TestMetadataTablesWithPartitionEvolution.java | 1 + .../spark/source/TestSparkMetadataColumns.java | 1 + .../extensions/TestAlterTablePartitionFields.java | 1 + .../spark/source/TestForwardCompatibility.java | 13 +++++++------ .../TestMetadataTablesWithPartitionEvolution.java | 1 + .../spark/source/TestSparkMetadataColumns.java | 1 + 10 files changed, 16 insertions(+), 6 deletions(-) diff --git a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index eab30f3572ee..7bcfa374df19 100644 --- a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -21,6 +21,7 @@ import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.spark.source.SparkTable; import org.apache.spark.sql.connector.catalog.CatalogManager; diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index ed684f9e0069..fa775c1c42f4 100644 --- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -41,6 +41,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 5ffec7a97e58..7143fe188158 100644 --- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -35,6 +35,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 7b1a78f135ba..80a1aba46c6b 100644 --- a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -21,6 +21,7 @@ import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.spark.source.SparkTable; import org.apache.spark.sql.connector.catalog.CatalogManager; diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index fe5393701110..f45b67dc4613 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -41,6 +41,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index bb9bdd15dece..ecd0088f477f 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -39,6 +39,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 521f0ded7bca..ba2d35816866 100644 --- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -21,6 +21,7 @@ import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.spark.source.SparkTable; import org.apache.spark.sql.connector.catalog.CatalogManager; diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index 0ef6b4f0cbcc..3bd914a5b3c7 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -74,7 +74,7 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - TestHelpers.newPartitionSpecBuilder() + org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() .withSchema(SCHEMA) .withSpecId(0) .addField("zero", 1, "id_zero") @@ -82,11 +82,12 @@ public class TestForwardCompatibility { // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - TestHelpers.newPartitionSpecBuilder() - .withSchema(SCHEMA) - .withSpecId(0) - .addField("zero", 1, "id_zero") - .build(); + org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("identity", 1, "id_zero") + .build(); + @Rule public TemporaryFolder temp = new TemporaryFolder(); diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index fe5393701110..f45b67dc4613 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -41,6 +41,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index bb9bdd15dece..ecd0088f477f 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -39,6 +39,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; From a6cec23c043738fb45e61cf42b37219acae6b376 Mon Sep 17 00:00:00 2001 From: roryqi Date: Sat, 2 Sep 2023 19:38:31 +0800 Subject: [PATCH 07/15] fix --- .../iceberg/spark/source/TestForwardCompatibility.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index 3bd914a5b3c7..7c018c0a6709 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -83,11 +83,9 @@ public class TestForwardCompatibility { // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() - .withSchema(SCHEMA) - .withSpecId(0) - .addField("identity", 1, "id_zero") - .build(); - + .withSchema(SCHEMA) + .withSpecId(0) + .addField("identity", 1, "id_zero").build(); @Rule public TemporaryFolder temp = new TemporaryFolder(); From 4f5ba1360abf436f69ac89803bbb956a065cbeff Mon Sep 17 00:00:00 2001 From: roryqi Date: Sat, 2 Sep 2023 19:42:33 +0800 Subject: [PATCH 08/15] fix --- .../apache/iceberg/spark/source/TestForwardCompatibility.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index 7c018c0a6709..4a335686294c 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -85,7 +85,8 @@ public class TestForwardCompatibility { org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() .withSchema(SCHEMA) .withSpecId(0) - .addField("identity", 1, "id_zero").build(); + .addField("identity", 1, "id_zero") + .build(); @Rule public TemporaryFolder temp = new TemporaryFolder(); From 52d5440d7f7398a81f13e0cac6d046e4ca696336 Mon Sep 17 00:00:00 2001 From: roryqi Date: Sat, 2 Sep 2023 19:56:51 +0800 Subject: [PATCH 09/15] fix --- .../TestAlterTablePartitionFields.java | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index ba2d35816866..f74d3e7b0cc6 100644 --- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -314,11 +314,13 @@ public void testReplacePartition() { .hour("ts") .build(); } else { - expected = TestHelpers.newPartitionSpecBuilder() - .withSchema(table.schema()) - .withSpecId(2) - .addField("hour", 3, 1001, "ts_hour") - .build(); + expected = + TestHelpers + .newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -346,11 +348,13 @@ public void testReplacePartitionAndRename() { .hour("ts", "hour_col") .build(); } else { - expected = TestHelpers.newPartitionSpecBuilder() - .withSchema(table.schema()) - .withSpecId(2) - .addField("hour", 3, 1001, "hour_col") - .build(); + expected = + TestHelpers + .newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -378,11 +382,13 @@ public void testReplaceNamedPartition() { .hour("ts") .build(); } else { - expected = TestHelpers.newPartitionSpecBuilder() - .withSchema(table.schema()) - .withSpecId(2) - .addField("hour", 3, 1001, "ts_hour") - .build(); + expected = + TestHelpers + .newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -410,11 +416,13 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .hour("ts", "hour_col") .build(); } else { - expected = TestHelpers.newPartitionSpecBuilder() - .withSchema(table.schema()) - .withSpecId(2) - .addField("hour", 3, 1001, "hour_col") - .build(); + expected = + TestHelpers + .newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); From 156bde4b3e4c1975d514ef81e07777d4f3aa8262 Mon Sep 17 00:00:00 2001 From: roryqi Date: Sat, 2 Sep 2023 19:58:49 +0800 Subject: [PATCH 10/15] fix --- .../extensions/TestAlterTablePartitionFields.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index f74d3e7b0cc6..65c0e1260d56 100644 --- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -315,8 +315,7 @@ public void testReplacePartition() { .build(); } else { expected = - TestHelpers - .newPartitionSpecBuilder() + TestHelpers.newPartitionSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "ts_hour") @@ -349,8 +348,7 @@ public void testReplacePartitionAndRename() { .build(); } else { expected = - TestHelpers - .newPartitionSpecBuilder() + TestHelpers.newPartitionSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "hour_col") @@ -383,8 +381,7 @@ public void testReplaceNamedPartition() { .build(); } else { expected = - TestHelpers - .newPartitionSpecBuilder() + TestHelpers.newPartitionSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "ts_hour") @@ -417,8 +414,7 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .build(); } else { expected = - TestHelpers - .newPartitionSpecBuilder() + TestHelpers.newPartitionSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "hour_col") From 0e460e221a055bc69be2f7ab8c1c2dee21b40fa4 Mon Sep 17 00:00:00 2001 From: roryqi Date: Sat, 2 Sep 2023 21:49:43 +0800 Subject: [PATCH 11/15] fix --- api/src/test/java/org/apache/iceberg/TestHelpers.java | 1 + 1 file changed, 1 insertion(+) diff --git a/api/src/test/java/org/apache/iceberg/TestHelpers.java b/api/src/test/java/org/apache/iceberg/TestHelpers.java index 5dce78c342eb..126730da722e 100644 --- a/api/src/test/java/org/apache/iceberg/TestHelpers.java +++ b/api/src/test/java/org/apache/iceberg/TestHelpers.java @@ -674,6 +674,7 @@ public static PartitionSpecBuilder newPartitionSpecBuilder() { public static class PartitionSpecBuilder { private final UnboundPartitionSpec.Builder unboundPartitionSpecBuilder; + private Schema schema; private PartitionSpecBuilder() { From 8bc77faf4156f2a0fc2c1b3a09a30e40529c4829 Mon Sep 17 00:00:00 2001 From: roryqi Date: Sat, 2 Sep 2023 21:53:30 +0800 Subject: [PATCH 12/15] fix --- api/src/test/java/org/apache/iceberg/TestHelpers.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/TestHelpers.java b/api/src/test/java/org/apache/iceberg/TestHelpers.java index 126730da722e..536cdc1a556c 100644 --- a/api/src/test/java/org/apache/iceberg/TestHelpers.java +++ b/api/src/test/java/org/apache/iceberg/TestHelpers.java @@ -265,6 +265,10 @@ public static void serialize(final Serializable obj, final OutputStream outputSt } } + public static PartitionSpecBuilder newPartitionSpecBuilder() { + return new PartitionSpecBuilder(); + } + public static class KryoHelpers { private KryoHelpers() {} @@ -668,10 +672,6 @@ public List splitOffsets() { } } - public static PartitionSpecBuilder newPartitionSpecBuilder() { - return new PartitionSpecBuilder(); - } - public static class PartitionSpecBuilder { private final UnboundPartitionSpec.Builder unboundPartitionSpecBuilder; From 85a1b87d3352107b01fc4fdf2c2723b1b5f9b76a Mon Sep 17 00:00:00 2001 From: roryqi Date: Tue, 19 Sep 2023 16:51:54 +0800 Subject: [PATCH 13/15] Add Spark 3.5 --- .../TestAlterTablePartitionFields.java | 66 ++++++------------- .../source/TestForwardCompatibility.java | 17 +++-- ...tMetadataTablesWithPartitionEvolution.java | 10 +-- .../source/TestSparkMetadataColumns.java | 10 +-- 4 files changed, 43 insertions(+), 60 deletions(-) diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index a43f2a041b97..65c0e1260d56 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -19,9 +19,9 @@ package org.apache.iceberg.spark.extensions; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.spark.source.SparkTable; import org.apache.spark.sql.connector.catalog.CatalogManager; @@ -315,17 +315,11 @@ public void testReplacePartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -354,17 +348,11 @@ public void testReplacePartitionAndRename() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -393,17 +381,11 @@ public void testReplaceNamedPartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -432,17 +414,11 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index 73e572ecaecd..616b93c6f4eb 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -34,7 +34,6 @@ import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.ManifestWriter; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; @@ -75,14 +74,18 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("zero", 1, "id_zero") + .build(); // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"identity\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("identity", 1, "id_zero") + .build(); @Rule public TemporaryFolder temp = new TemporaryFolder(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index b4b60dc02dd5..64171b9e55d8 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -37,10 +37,10 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; @@ -627,9 +627,11 @@ public void testMetadataTablesWithUnknownTransforms() { Table table = validationCatalog.loadTable(tableIdent); PartitionSpec unknownSpec = - PartitionSpecParser.fromJson( - table.schema(), - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(table.schema()) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); // replace the table spec to include an unknown transform TableOperations ops = ((HasTableOperations) table).operations(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 8a2fae811750..6cc7290bdb36 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -34,11 +34,11 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -75,9 +75,11 @@ public class TestSparkMetadataColumns extends SparkTestBase { Types.NestedField.optional(3, "data", Types.StringType.get())); private static final PartitionSpec SPEC = PartitionSpec.unpartitioned(); private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newPartitionSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); @Parameterized.Parameters(name = "fileFormat = {0}, vectorized = {1}, formatVersion = {2}") public static Object[][] parameters() { From d2b047228b07801cec008886e81b0aa33e2e509f Mon Sep 17 00:00:00 2001 From: qqqttt Date: Tue, 31 Oct 2023 10:13:26 +0800 Subject: [PATCH 14/15] rename to ExpectedSpecBuilder --- .../java/org/apache/iceberg/TestHelpers.java | 16 ++++++++-------- .../TestAlterTablePartitionFields.java | 8 ++++---- .../spark/source/TestForwardCompatibility.java | 4 ++-- ...TestMetadataTablesWithPartitionEvolution.java | 2 +- .../spark/source/TestSparkMetadataColumns.java | 2 +- .../TestAlterTablePartitionFields.java | 8 ++++---- .../spark/source/TestForwardCompatibility.java | 4 ++-- ...TestMetadataTablesWithPartitionEvolution.java | 2 +- .../spark/source/TestSparkMetadataColumns.java | 2 +- .../TestAlterTablePartitionFields.java | 8 ++++---- .../spark/source/TestForwardCompatibility.java | 4 ++-- ...TestMetadataTablesWithPartitionEvolution.java | 2 +- .../spark/source/TestSparkMetadataColumns.java | 2 +- .../TestAlterTablePartitionFields.java | 8 ++++---- .../spark/source/TestForwardCompatibility.java | 4 ++-- ...TestMetadataTablesWithPartitionEvolution.java | 2 +- .../spark/source/TestSparkMetadataColumns.java | 2 +- 17 files changed, 40 insertions(+), 40 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/TestHelpers.java b/api/src/test/java/org/apache/iceberg/TestHelpers.java index 536cdc1a556c..890ae8abd474 100644 --- a/api/src/test/java/org/apache/iceberg/TestHelpers.java +++ b/api/src/test/java/org/apache/iceberg/TestHelpers.java @@ -265,8 +265,8 @@ public static void serialize(final Serializable obj, final OutputStream outputSt } } - public static PartitionSpecBuilder newPartitionSpecBuilder() { - return new PartitionSpecBuilder(); + public static ExpectedSpecBuilder newExpectedSpecBuilder() { + return new ExpectedSpecBuilder(); } public static class KryoHelpers { @@ -672,32 +672,32 @@ public List splitOffsets() { } } - public static class PartitionSpecBuilder { + public static class ExpectedSpecBuilder { private final UnboundPartitionSpec.Builder unboundPartitionSpecBuilder; private Schema schema; - private PartitionSpecBuilder() { + private ExpectedSpecBuilder() { this.unboundPartitionSpecBuilder = UnboundPartitionSpec.builder(); } - public PartitionSpecBuilder withSchema(Schema newSchema) { + public ExpectedSpecBuilder withSchema(Schema newSchema) { this.schema = newSchema; return this; } - public PartitionSpecBuilder withSpecId(int newSpecId) { + public ExpectedSpecBuilder withSpecId(int newSpecId) { unboundPartitionSpecBuilder.withSpecId(newSpecId); return this; } - public PartitionSpecBuilder addField( + public ExpectedSpecBuilder addField( String transformAsString, int sourceId, int partitionId, String name) { unboundPartitionSpecBuilder.addField(transformAsString, sourceId, partitionId, name); return this; } - public PartitionSpecBuilder addField(String transformAsString, int sourceId, String name) { + public ExpectedSpecBuilder addField(String transformAsString, int sourceId, String name) { unboundPartitionSpecBuilder.addField(transformAsString, sourceId, name); return this; } diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 61764af11342..042e87c729a5 100644 --- a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -392,7 +392,7 @@ public void testReplacePartition() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "ts_hour") @@ -425,7 +425,7 @@ public void testReplacePartitionAndRename() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "hour_col") @@ -458,7 +458,7 @@ public void testReplaceNamedPartition() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "ts_hour") @@ -491,7 +491,7 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "hour_col") diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index 5c230cd693ff..96bebf7c8868 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -74,14 +74,14 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() .withSchema(SCHEMA) .withSpecId(0) .addField("zero", 1, "id_zero") .build(); // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() .withSchema(SCHEMA) .withSpecId(0) .addField("identity", 1, "id_zero") diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index f45b67dc4613..0baaef1374d4 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -627,7 +627,7 @@ public void testMetadataTablesWithUnknownTransforms() { Table table = validationCatalog.loadTable(tableIdent); PartitionSpec unknownSpec = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(1) .addField("zero", 1, "id_zero") diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index ecd0088f477f..5c7929112fb9 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -75,7 +75,7 @@ public class TestSparkMetadataColumns extends SparkTestBase { Types.NestedField.optional(3, "data", Types.StringType.get())); private static final PartitionSpec SPEC = PartitionSpec.unpartitioned(); private static final PartitionSpec UNKNOWN_SPEC = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(SCHEMA) .withSpecId(1) .addField("zero", 1, "id_zero") diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 5a070b71d44d..948fc462de99 100644 --- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -392,7 +392,7 @@ public void testReplacePartition() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "ts_hour") @@ -425,7 +425,7 @@ public void testReplacePartitionAndRename() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "hour_col") @@ -458,7 +458,7 @@ public void testReplaceNamedPartition() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "ts_hour") @@ -491,7 +491,7 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "hour_col") diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index 4a335686294c..6ab9e57949e3 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -74,7 +74,7 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() .withSchema(SCHEMA) .withSpecId(0) .addField("zero", 1, "id_zero") @@ -82,7 +82,7 @@ public class TestForwardCompatibility { // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() .withSchema(SCHEMA) .withSpecId(0) .addField("identity", 1, "id_zero") diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index f45b67dc4613..0baaef1374d4 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -627,7 +627,7 @@ public void testMetadataTablesWithUnknownTransforms() { Table table = validationCatalog.loadTable(tableIdent); PartitionSpec unknownSpec = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(1) .addField("zero", 1, "id_zero") diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index ecd0088f477f..5c7929112fb9 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -75,7 +75,7 @@ public class TestSparkMetadataColumns extends SparkTestBase { Types.NestedField.optional(3, "data", Types.StringType.get())); private static final PartitionSpec SPEC = PartitionSpec.unpartitioned(); private static final PartitionSpec UNKNOWN_SPEC = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(SCHEMA) .withSpecId(1) .addField("zero", 1, "id_zero") diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 5a070b71d44d..948fc462de99 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -392,7 +392,7 @@ public void testReplacePartition() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "ts_hour") @@ -425,7 +425,7 @@ public void testReplacePartitionAndRename() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "hour_col") @@ -458,7 +458,7 @@ public void testReplaceNamedPartition() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "ts_hour") @@ -491,7 +491,7 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "hour_col") diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index bc3646f9bf5e..80a8196f8a09 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -74,7 +74,7 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() .withSchema(SCHEMA) .withSpecId(0) .addField("zero", 1, "id_zero") @@ -82,7 +82,7 @@ public class TestForwardCompatibility { // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() .withSchema(SCHEMA) .withSpecId(0) .addField("identity", 1, "id_zero") diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index 64171b9e55d8..ea65fead10fc 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -627,7 +627,7 @@ public void testMetadataTablesWithUnknownTransforms() { Table table = validationCatalog.loadTable(tableIdent); PartitionSpec unknownSpec = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(1) .addField("zero", 1, "id_zero") diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 6cc7290bdb36..b2361c188c17 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -75,7 +75,7 @@ public class TestSparkMetadataColumns extends SparkTestBase { Types.NestedField.optional(3, "data", Types.StringType.get())); private static final PartitionSpec SPEC = PartitionSpec.unpartitioned(); private static final PartitionSpec UNKNOWN_SPEC = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(SCHEMA) .withSpecId(1) .addField("zero", 1, "id_zero") diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 5a070b71d44d..948fc462de99 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -392,7 +392,7 @@ public void testReplacePartition() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "ts_hour") @@ -425,7 +425,7 @@ public void testReplacePartitionAndRename() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "hour_col") @@ -458,7 +458,7 @@ public void testReplaceNamedPartition() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "ts_hour") @@ -491,7 +491,7 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .build(); } else { expected = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) .addField("hour", 3, 1001, "hour_col") diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index 616b93c6f4eb..446989d1af3e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -74,14 +74,14 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() .withSchema(SCHEMA) .withSpecId(0) .addField("zero", 1, "id_zero") .build(); // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - org.apache.iceberg.TestHelpers.newPartitionSpecBuilder() + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() .withSchema(SCHEMA) .withSpecId(0) .addField("identity", 1, "id_zero") diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index 64171b9e55d8..ea65fead10fc 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -627,7 +627,7 @@ public void testMetadataTablesWithUnknownTransforms() { Table table = validationCatalog.loadTable(tableIdent); PartitionSpec unknownSpec = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(1) .addField("zero", 1, "id_zero") diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 6cc7290bdb36..b2361c188c17 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -75,7 +75,7 @@ public class TestSparkMetadataColumns extends SparkTestBase { Types.NestedField.optional(3, "data", Types.StringType.get())); private static final PartitionSpec SPEC = PartitionSpec.unpartitioned(); private static final PartitionSpec UNKNOWN_SPEC = - TestHelpers.newPartitionSpecBuilder() + TestHelpers.newExpectedSpecBuilder() .withSchema(SCHEMA) .withSpecId(1) .addField("zero", 1, "id_zero") From 170470fde8761365778e72a66e271c84bc05c545 Mon Sep 17 00:00:00 2001 From: qqqttt Date: Tue, 31 Oct 2023 10:22:41 +0800 Subject: [PATCH 15/15] trigger