Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test: Add a test utility method to programmatically create expected partition specs #8467

Merged
merged 17 commits into from
Nov 7, 2023
Merged

Test: Add a test utility method to programmatically create expected partition specs #8467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
7c4640d
Test: Add a test utility method to programmatically create expected p…
jerqi Sep 2, 2023
97ca713
Add Spark 3.3
jerqi Sep 2, 2023
b090d27
Add Spark 3.2
jerqi Sep 2, 2023
0340c8b
Add Spark 3.1
jerqi Sep 2, 2023
6e06432
Fix code styles
jerqi Sep 2, 2023
1d542fd
Fix import issues
jerqi Sep 2, 2023
a6cec23
fix
jerqi Sep 2, 2023
4f5ba13
fix
jerqi Sep 2, 2023
52d5440
fix
jerqi Sep 2, 2023
156bde4
fix
jerqi Sep 2, 2023
0e460e2
fix
jerqi Sep 2, 2023
8bc77fa
fix
jerqi Sep 2, 2023
a08d61a
Merge remote-tracking branch 'github/master' into ICEBERG-8434
jerqi Sep 19, 2023
85a1b87
Add Spark 3.5
jerqi Sep 19, 2023
11dd7d8
Merge remote-tracking branch 'github/master' into ICEBERG-8434
jerqi Oct 7, 2023
d2b0472
rename to ExpectedSpecBuilder
Oct 31, 2023
170470f
trigger
Oct 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions api/src/test/java/org/apache/iceberg/TestHelpers.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,10 @@ public static void serialize(final Serializable obj, final OutputStream outputSt
}
}

public static ExpectedSpecBuilder newExpectedSpecBuilder() {
return new ExpectedSpecBuilder();
}

public static class KryoHelpers {
private KryoHelpers() {}

Expand Down Expand Up @@ -667,4 +671,40 @@ public List<Long> splitOffsets() {
return null;
}
}

public static class ExpectedSpecBuilder {
private final UnboundPartitionSpec.Builder unboundPartitionSpecBuilder;

private Schema schema;

private ExpectedSpecBuilder() {
this.unboundPartitionSpecBuilder = UnboundPartitionSpec.builder();
}

public ExpectedSpecBuilder withSchema(Schema newSchema) {
this.schema = newSchema;
return this;
}

public ExpectedSpecBuilder withSpecId(int newSpecId) {
unboundPartitionSpecBuilder.withSpecId(newSpecId);
return this;
}

public ExpectedSpecBuilder addField(
String transformAsString, int sourceId, int partitionId, String name) {
unboundPartitionSpecBuilder.addField(transformAsString, sourceId, partitionId, name);
return this;
}

public ExpectedSpecBuilder addField(String transformAsString, int sourceId, String name) {
unboundPartitionSpecBuilder.addField(transformAsString, sourceId, name);
return this;
}

public PartitionSpec build() {
Preconditions.checkNotNull(schema, "Field schema is missing");
return unboundPartitionSpecBuilder.build().bind(schema);
}
}
}
66 changes: 21 additions & 45 deletions ...ions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@
package org.apache.iceberg.spark.extensions;

import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.PartitionSpecParser;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.spark.SparkCatalogConfig;
import org.apache.iceberg.spark.source.SparkTable;
import org.apache.spark.sql.connector.catalog.CatalogManager;
Expand Down Expand Up @@ -392,17 +392,11 @@ public void testReplacePartition() {
.build();
} else {
expected =
PartitionSpecParser.fromJson(
table.schema(),
"{\n"
+ " \"spec-id\" : 2,\n"
+ " \"fields\" : [ {\n"
+ " \"name\" : \"ts_hour\",\n"
+ " \"transform\" : \"hour\",\n"
+ " \"source-id\" : 3,\n"
+ " \"field-id\" : 1001\n"
+ " } ]\n"
+ "}");
TestHelpers.newExpectedSpecBuilder()
.withSchema(table.schema())
.withSpecId(2)
.addField("hour", 3, 1001, "ts_hour")
.build();
}
Assert.assertEquals(
"Should changed from daily to hourly partitioned field", expected, table.spec());
Expand Down Expand Up @@ -431,17 +425,11 @@ public void testReplacePartitionAndRename() {
.build();
} else {
expected =
PartitionSpecParser.fromJson(
table.schema(),
"{\n"
+ " \"spec-id\" : 2,\n"
+ " \"fields\" : [ {\n"
+ " \"name\" : \"hour_col\",\n"
+ " \"transform\" : \"hour\",\n"
+ " \"source-id\" : 3,\n"
+ " \"field-id\" : 1001\n"
+ " } ]\n"
+ "}");
TestHelpers.newExpectedSpecBuilder()
.withSchema(table.schema())
.withSpecId(2)
.addField("hour", 3, 1001, "hour_col")
.build();
}
Assert.assertEquals(
"Should changed from daily to hourly partitioned field", expected, table.spec());
Expand Down Expand Up @@ -470,17 +458,11 @@ public void testReplaceNamedPartition() {
.build();
} else {
expected =
PartitionSpecParser.fromJson(
table.schema(),
"{\n"
+ " \"spec-id\" : 2,\n"
+ " \"fields\" : [ {\n"
+ " \"name\" : \"ts_hour\",\n"
+ " \"transform\" : \"hour\",\n"
+ " \"source-id\" : 3,\n"
+ " \"field-id\" : 1001\n"
+ " } ]\n"
+ "}");
TestHelpers.newExpectedSpecBuilder()
.withSchema(table.schema())
.withSpecId(2)
.addField("hour", 3, 1001, "ts_hour")
.build();
}
Assert.assertEquals(
"Should changed from daily to hourly partitioned field", expected, table.spec());
Expand Down Expand Up @@ -509,17 +491,11 @@ public void testReplaceNamedPartitionAndRenameDifferently() {
.build();
} else {
expected =
PartitionSpecParser.fromJson(
table.schema(),
"{\n"
+ " \"spec-id\" : 2,\n"
+ " \"fields\" : [ {\n"
+ " \"name\" : \"hour_col\",\n"
+ " \"transform\" : \"hour\",\n"
+ " \"source-id\" : 3,\n"
+ " \"field-id\" : 1001\n"
+ " } ]\n"
+ "}");
TestHelpers.newExpectedSpecBuilder()
.withSchema(table.schema())
.withSpecId(2)
.addField("hour", 3, 1001, "hour_col")
.build();
}
Assert.assertEquals(
"Should changed from daily to hourly partitioned field", expected, table.spec());
Expand Down
17 changes: 10 additions & 7 deletions spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import org.apache.iceberg.ManifestFiles;
import org.apache.iceberg.ManifestWriter;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.PartitionSpecParser;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableProperties;
Expand Down Expand Up @@ -75,14 +74,18 @@ public class TestForwardCompatibility {

// create a spec for the schema that uses a "zero" transform that produces all 0s
private static final PartitionSpec UNKNOWN_SPEC =
PartitionSpecParser.fromJson(
SCHEMA,
"{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }");
org.apache.iceberg.TestHelpers.newExpectedSpecBuilder()
.withSchema(SCHEMA)
.withSpecId(0)
.addField("zero", 1, "id_zero")
.build();
// create a fake spec to use to write table metadata
private static final PartitionSpec FAKE_SPEC =
PartitionSpecParser.fromJson(
SCHEMA,
"{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"identity\", \"source-id\": 1 } ] }");
org.apache.iceberg.TestHelpers.newExpectedSpecBuilder()
.withSchema(SCHEMA)
.withSpecId(0)
.addField("identity", 1, "id_zero")
.build();

@Rule public TemporaryFolder temp = new TemporaryFolder();

Expand Down
10 changes: 6 additions & 4 deletions ...c/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@
import org.apache.iceberg.HasTableOperations;
import org.apache.iceberg.MetadataTableType;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.PartitionSpecParser;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.TableOperations;
import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
Expand Down Expand Up @@ -627,9 +627,11 @@ public void testMetadataTablesWithUnknownTransforms() {
Table table = validationCatalog.loadTable(tableIdent);

PartitionSpec unknownSpec =
PartitionSpecParser.fromJson(
table.schema(),
"{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }");
TestHelpers.newExpectedSpecBuilder()
.withSchema(table.schema())
.withSpecId(1)
.addField("zero", 1, "id_zero")
.build();

// replace the table spec to include an unknown transform
TableOperations ops = ((HasTableOperations) table).operations();
Expand Down
10 changes: 6 additions & 4 deletions spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@
import org.apache.iceberg.HasTableOperations;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.PartitionSpecParser;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.TableOperations;
import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
Expand Down Expand Up @@ -75,9 +75,11 @@ public class TestSparkMetadataColumns extends SparkTestBase {
Types.NestedField.optional(3, "data", Types.StringType.get()));
private static final PartitionSpec SPEC = PartitionSpec.unpartitioned();
private static final PartitionSpec UNKNOWN_SPEC =
PartitionSpecParser.fromJson(
SCHEMA,
"{ \"spec-id\": 1, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }");
TestHelpers.newExpectedSpecBuilder()
.withSchema(SCHEMA)
.withSpecId(1)
.addField("zero", 1, "id_zero")
.build();

@Parameterized.Parameters(name = "fileFormat = {0}, vectorized = {1}, formatVersion = {2}")
public static Object[][] parameters() {
Expand Down
66 changes: 21 additions & 45 deletions ...ions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@
package org.apache.iceberg.spark.extensions;

import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.PartitionSpecParser;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.spark.SparkCatalogConfig;
import org.apache.iceberg.spark.source.SparkTable;
import org.apache.spark.sql.connector.catalog.CatalogManager;
Expand Down Expand Up @@ -392,17 +392,11 @@ public void testReplacePartition() {
.build();
} else {
expected =
PartitionSpecParser.fromJson(
table.schema(),
"{\n"
+ " \"spec-id\" : 2,\n"
+ " \"fields\" : [ {\n"
+ " \"name\" : \"ts_hour\",\n"
+ " \"transform\" : \"hour\",\n"
+ " \"source-id\" : 3,\n"
+ " \"field-id\" : 1001\n"
+ " } ]\n"
+ "}");
TestHelpers.newExpectedSpecBuilder()
.withSchema(table.schema())
.withSpecId(2)
.addField("hour", 3, 1001, "ts_hour")
.build();
}
Assert.assertEquals(
"Should changed from daily to hourly partitioned field", expected, table.spec());
Expand Down Expand Up @@ -431,17 +425,11 @@ public void testReplacePartitionAndRename() {
.build();
} else {
expected =
PartitionSpecParser.fromJson(
table.schema(),
"{\n"
+ " \"spec-id\" : 2,\n"
+ " \"fields\" : [ {\n"
+ " \"name\" : \"hour_col\",\n"
+ " \"transform\" : \"hour\",\n"
+ " \"source-id\" : 3,\n"
+ " \"field-id\" : 1001\n"
+ " } ]\n"
+ "}");
TestHelpers.newExpectedSpecBuilder()
.withSchema(table.schema())
.withSpecId(2)
.addField("hour", 3, 1001, "hour_col")
.build();
}
Assert.assertEquals(
"Should changed from daily to hourly partitioned field", expected, table.spec());
Expand Down Expand Up @@ -470,17 +458,11 @@ public void testReplaceNamedPartition() {
.build();
} else {
expected =
PartitionSpecParser.fromJson(
table.schema(),
"{\n"
+ " \"spec-id\" : 2,\n"
+ " \"fields\" : [ {\n"
+ " \"name\" : \"ts_hour\",\n"
+ " \"transform\" : \"hour\",\n"
+ " \"source-id\" : 3,\n"
+ " \"field-id\" : 1001\n"
+ " } ]\n"
+ "}");
TestHelpers.newExpectedSpecBuilder()
.withSchema(table.schema())
.withSpecId(2)
.addField("hour", 3, 1001, "ts_hour")
.build();
}
Assert.assertEquals(
"Should changed from daily to hourly partitioned field", expected, table.spec());
Expand Down Expand Up @@ -509,17 +491,11 @@ public void testReplaceNamedPartitionAndRenameDifferently() {
.build();
} else {
expected =
PartitionSpecParser.fromJson(
table.schema(),
"{\n"
+ " \"spec-id\" : 2,\n"
+ " \"fields\" : [ {\n"
+ " \"name\" : \"hour_col\",\n"
+ " \"transform\" : \"hour\",\n"
+ " \"source-id\" : 3,\n"
+ " \"field-id\" : 1001\n"
+ " } ]\n"
+ "}");
TestHelpers.newExpectedSpecBuilder()
.withSchema(table.schema())
.withSpecId(2)
.addField("hour", 3, 1001, "hour_col")
.build();
}
Assert.assertEquals(
"Should changed from daily to hourly partitioned field", expected, table.spec());
Expand Down
18 changes: 11 additions & 7 deletions spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import org.apache.iceberg.ManifestFiles;
import org.apache.iceberg.ManifestWriter;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.PartitionSpecParser;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableProperties;
Expand Down Expand Up @@ -75,14 +74,19 @@ public class TestForwardCompatibility {

// create a spec for the schema that uses a "zero" transform that produces all 0s
private static final PartitionSpec UNKNOWN_SPEC =
PartitionSpecParser.fromJson(
SCHEMA,
"{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"zero\", \"source-id\": 1 } ] }");
org.apache.iceberg.TestHelpers.newExpectedSpecBuilder()
.withSchema(SCHEMA)
.withSpecId(0)
.addField("zero", 1, "id_zero")
.build();

// create a fake spec to use to write table metadata
private static final PartitionSpec FAKE_SPEC =
PartitionSpecParser.fromJson(
SCHEMA,
"{ \"spec-id\": 0, \"fields\": [ { \"name\": \"id_zero\", \"transform\": \"identity\", \"source-id\": 1 } ] }");
org.apache.iceberg.TestHelpers.newExpectedSpecBuilder()
.withSchema(SCHEMA)
.withSpecId(0)
.addField("identity", 1, "id_zero")
.build();

@Rule public TemporaryFolder temp = new TemporaryFolder();

Expand Down
Loading