From e8bb8b502c981f6e5dea27cc6f833655bf79f4c6 Mon Sep 17 00:00:00 2001 From: roryqi Date: Wed, 8 Nov 2023 04:55:41 +0800 Subject: [PATCH] Test: Add a test utility method to programmatically create expected partition specs (#8467) --- .../java/org/apache/iceberg/TestHelpers.java | 40 +++++++++++ .../TestAlterTablePartitionFields.java | 66 ++++++------------- .../source/TestForwardCompatibility.java | 17 +++-- ...tMetadataTablesWithPartitionEvolution.java | 10 +-- .../source/TestSparkMetadataColumns.java | 10 +-- .../TestAlterTablePartitionFields.java | 66 ++++++------------- .../source/TestForwardCompatibility.java | 18 +++-- ...tMetadataTablesWithPartitionEvolution.java | 10 +-- .../source/TestSparkMetadataColumns.java | 10 +-- .../TestAlterTablePartitionFields.java | 66 ++++++------------- .../source/TestForwardCompatibility.java | 18 +++-- ...tMetadataTablesWithPartitionEvolution.java | 10 +-- .../source/TestSparkMetadataColumns.java | 10 +-- .../TestAlterTablePartitionFields.java | 66 ++++++------------- .../source/TestForwardCompatibility.java | 17 +++-- ...tMetadataTablesWithPartitionEvolution.java | 10 +-- .../source/TestSparkMetadataColumns.java | 10 +-- 17 files changed, 214 insertions(+), 240 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/TestHelpers.java b/api/src/test/java/org/apache/iceberg/TestHelpers.java index 153e2de7ea9a..890ae8abd474 100644 --- a/api/src/test/java/org/apache/iceberg/TestHelpers.java +++ b/api/src/test/java/org/apache/iceberg/TestHelpers.java @@ -265,6 +265,10 @@ public static void serialize(final Serializable obj, final OutputStream outputSt } } + public static ExpectedSpecBuilder newExpectedSpecBuilder() { + return new ExpectedSpecBuilder(); + } + public static class KryoHelpers { private KryoHelpers() {} @@ -667,4 +671,40 @@ public List splitOffsets() { return null; } } + + public static class ExpectedSpecBuilder { + private final UnboundPartitionSpec.Builder unboundPartitionSpecBuilder; + + private Schema schema; + + private ExpectedSpecBuilder() { + this.unboundPartitionSpecBuilder = UnboundPartitionSpec.builder(); + } + + public ExpectedSpecBuilder withSchema(Schema newSchema) { + this.schema = newSchema; + return this; + } + + public ExpectedSpecBuilder withSpecId(int newSpecId) { + unboundPartitionSpecBuilder.withSpecId(newSpecId); + return this; + } + + public ExpectedSpecBuilder addField( + String transformAsString, int sourceId, int partitionId, String name) { + unboundPartitionSpecBuilder.addField(transformAsString, sourceId, partitionId, name); + return this; + } + + public ExpectedSpecBuilder addField(String transformAsString, int sourceId, String name) { + unboundPartitionSpecBuilder.addField(transformAsString, sourceId, name); + return this; + } + + public PartitionSpec build() { + Preconditions.checkNotNull(schema, "Field schema is missing"); + return unboundPartitionSpecBuilder.build().bind(schema); + } + } } diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 2ecf6b0c4ca7..042e87c729a5 100644 --- a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -19,9 +19,9 @@ package org.apache.iceberg.spark.extensions; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.spark.source.SparkTable; import org.apache.spark.sql.connector.catalog.CatalogManager; @@ -392,17 +392,11 @@ public void testReplacePartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -431,17 +425,11 @@ public void testReplacePartitionAndRename() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -470,17 +458,11 @@ public void testReplaceNamedPartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -509,17 +491,11 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index fe440235901c..96bebf7c8868 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -35,7 +35,6 @@ import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.ManifestWriter; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; @@ -75,14 +74,18 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("zero", 1, "id_zero") + .build(); // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"identity\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("identity", 1, "id_zero") + .build(); @Rule public TemporaryFolder temp = new TemporaryFolder(); diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index a62199181a3a..0baaef1374d4 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -38,10 +38,10 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; @@ -627,9 +627,11 @@ public void testMetadataTablesWithUnknownTransforms() { Table table = validationCatalog.loadTable(tableIdent); PartitionSpec unknownSpec = - PartitionSpecParser.fromJson( - table.schema(), - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); // replace the table spec to include an unknown transform TableOperations ops = ((HasTableOperations) table).operations(); diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 79b755872dc3..5c7929112fb9 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -35,11 +35,11 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -75,9 +75,11 @@ public class TestSparkMetadataColumns extends SparkTestBase { Types.NestedField.optional(3, "data", Types.StringType.get())); private static final PartitionSpec SPEC = PartitionSpec.unpartitioned(); private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); @Parameterized.Parameters(name = "fileFormat = {0}, vectorized = {1}, formatVersion = {2}") public static Object[][] parameters() { diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 0e978e52e570..948fc462de99 100644 --- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -19,9 +19,9 @@ package org.apache.iceberg.spark.extensions; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.spark.source.SparkTable; import org.apache.spark.sql.connector.catalog.CatalogManager; @@ -392,17 +392,11 @@ public void testReplacePartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -431,17 +425,11 @@ public void testReplacePartitionAndRename() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -470,17 +458,11 @@ public void testReplaceNamedPartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -509,17 +491,11 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index fe440235901c..6ab9e57949e3 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -35,7 +35,6 @@ import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.ManifestWriter; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; @@ -75,14 +74,19 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("zero", 1, "id_zero") + .build(); + // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"identity\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("identity", 1, "id_zero") + .build(); @Rule public TemporaryFolder temp = new TemporaryFolder(); diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index a62199181a3a..0baaef1374d4 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -38,10 +38,10 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; @@ -627,9 +627,11 @@ public void testMetadataTablesWithUnknownTransforms() { Table table = validationCatalog.loadTable(tableIdent); PartitionSpec unknownSpec = - PartitionSpecParser.fromJson( - table.schema(), - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); // replace the table spec to include an unknown transform TableOperations ops = ((HasTableOperations) table).operations(); diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 79b755872dc3..5c7929112fb9 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -35,11 +35,11 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -75,9 +75,11 @@ public class TestSparkMetadataColumns extends SparkTestBase { Types.NestedField.optional(3, "data", Types.StringType.get())); private static final PartitionSpec SPEC = PartitionSpec.unpartitioned(); private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); @Parameterized.Parameters(name = "fileFormat = {0}, vectorized = {1}, formatVersion = {2}") public static Object[][] parameters() { diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 0e978e52e570..948fc462de99 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -19,9 +19,9 @@ package org.apache.iceberg.spark.extensions; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.spark.source.SparkTable; import org.apache.spark.sql.connector.catalog.CatalogManager; @@ -392,17 +392,11 @@ public void testReplacePartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -431,17 +425,11 @@ public void testReplacePartitionAndRename() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -470,17 +458,11 @@ public void testReplaceNamedPartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -509,17 +491,11 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index 73e572ecaecd..80a8196f8a09 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -34,7 +34,6 @@ import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.ManifestWriter; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; @@ -75,14 +74,19 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("zero", 1, "id_zero") + .build(); + // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"identity\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("identity", 1, "id_zero") + .build(); @Rule public TemporaryFolder temp = new TemporaryFolder(); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index b4b60dc02dd5..ea65fead10fc 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -37,10 +37,10 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; @@ -627,9 +627,11 @@ public void testMetadataTablesWithUnknownTransforms() { Table table = validationCatalog.loadTable(tableIdent); PartitionSpec unknownSpec = - PartitionSpecParser.fromJson( - table.schema(), - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); // replace the table spec to include an unknown transform TableOperations ops = ((HasTableOperations) table).operations(); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 8a2fae811750..b2361c188c17 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -34,11 +34,11 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -75,9 +75,11 @@ public class TestSparkMetadataColumns extends SparkTestBase { Types.NestedField.optional(3, "data", Types.StringType.get())); private static final PartitionSpec SPEC = PartitionSpec.unpartitioned(); private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); @Parameterized.Parameters(name = "fileFormat = {0}, vectorized = {1}, formatVersion = {2}") public static Object[][] parameters() { diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 0e978e52e570..948fc462de99 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -19,9 +19,9 @@ package org.apache.iceberg.spark.extensions; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.spark.source.SparkTable; import org.apache.spark.sql.connector.catalog.CatalogManager; @@ -392,17 +392,11 @@ public void testReplacePartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -431,17 +425,11 @@ public void testReplacePartitionAndRename() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -470,17 +458,11 @@ public void testReplaceNamedPartition() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"ts_hour\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "ts_hour") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); @@ -509,17 +491,11 @@ public void testReplaceNamedPartitionAndRenameDifferently() { .build(); } else { expected = - PartitionSpecParser.fromJson( - table.schema(), - "{\n" - + " \"spec-id\" : 2,\n" - + " \"fields\" : [ {\n" - + " \"name\" : \"hour_col\",\n" - + " \"transform\" : \"hour\",\n" - + " \"source-id\" : 3,\n" - + " \"field-id\" : 1001\n" - + " } ]\n" - + "}"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(2) + .addField("hour", 3, 1001, "hour_col") + .build(); } Assert.assertEquals( "Should changed from daily to hourly partitioned field", expected, table.spec()); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index 73e572ecaecd..446989d1af3e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -34,7 +34,6 @@ import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.ManifestWriter; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; @@ -75,14 +74,18 @@ public class TestForwardCompatibility { // create a spec for the schema that uses a "zero" transform that produces all 0s private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("zero", 1, "id_zero") + .build(); // create a fake spec to use to write table metadata private static final PartitionSpec FAKE_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"identity\", \"source-id\": 1 } ] }"); + org.apache.iceberg.TestHelpers.newExpectedSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(0) + .addField("identity", 1, "id_zero") + .build(); @Rule public TemporaryFolder temp = new TemporaryFolder(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index b4b60dc02dd5..ea65fead10fc 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -37,10 +37,10 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; @@ -627,9 +627,11 @@ public void testMetadataTablesWithUnknownTransforms() { Table table = validationCatalog.loadTable(tableIdent); PartitionSpec unknownSpec = - PartitionSpecParser.fromJson( - table.schema(), - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(table.schema()) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); // replace the table spec to include an unknown transform TableOperations ops = ((HasTableOperations) table).operations(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java index 8a2fae811750..b2361c188c17 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java @@ -34,11 +34,11 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TestHelpers; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -75,9 +75,11 @@ public class TestSparkMetadataColumns extends SparkTestBase { Types.NestedField.optional(3, "data", Types.StringType.get())); private static final PartitionSpec SPEC = PartitionSpec.unpartitioned(); private static final PartitionSpec UNKNOWN_SPEC = - PartitionSpecParser.fromJson( - SCHEMA, - "{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }"); + TestHelpers.newExpectedSpecBuilder() + .withSchema(SCHEMA) + .withSpecId(1) + .addField("zero", 1, "id_zero") + .build(); @Parameterized.Parameters(name = "fileFormat = {0}, vectorized = {1}, formatVersion = {2}") public static Object[][] parameters() {