diff --git a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 5102da844242..5589b1b05c4b 100644 --- a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -27,6 +27,7 @@ import org.apache.spark.sql.connector.catalog.CatalogManager; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -179,6 +180,82 @@ public void testAddHoursPartition() { Assert.assertEquals("Should have new spec field", expected, table.spec()); } + @Test + public void testAddYearPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddMonthPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddDayPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddHourPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + @Test public void testAddNamedPartition() { createTable("id bigint NOT NULL, category string, ts timestamp, data string"); diff --git a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 2bfd0aaf8da7..3e0452d94ac5 100644 --- a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -374,14 +374,18 @@ public static Term toIcebergTerm(Transform transform) { return org.apache.iceberg.expressions.Expressions.ref(colName); case "bucket": return org.apache.iceberg.expressions.Expressions.bucket(colName, findWidth(transform)); + case "year": case "years": return org.apache.iceberg.expressions.Expressions.year(colName); + case "month": case "months": return org.apache.iceberg.expressions.Expressions.month(colName); case "date": + case "day": case "days": return org.apache.iceberg.expressions.Expressions.day(colName); case "date_hour": + case "hour": case "hours": return org.apache.iceberg.expressions.Expressions.hour(colName); case "truncate": @@ -417,17 +421,21 @@ public static PartitionSpec toPartitionSpec(Schema schema, Transform[] partition case "bucket": builder.bucket(colName, findWidth(transform)); break; + case "year": case "years": builder.year(colName); break; + case "month": case "months": builder.month(colName); break; case "date": + case "day": case "days": builder.day(colName); break; case "date_hour": + case "hour": case "hours": builder.hour(colName); break; diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java index 1411c83ddc65..a6256afcdf65 100644 --- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java +++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java @@ -63,6 +63,26 @@ public void testTransformIgnoreCase() { Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent)); } + @Test + public void testTransformSingularForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hour(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + + @Test + public void testTransformPluralForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hours(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + @Test public void testCreateTable() { Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 9b4bd12ec1bf..2ecf6b0c4ca7 100644 --- a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -27,6 +27,7 @@ import org.apache.spark.sql.connector.catalog.CatalogManager; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -179,6 +180,82 @@ public void testAddHoursPartition() { Assert.assertEquals("Should have new spec field", expected, table.spec()); } + @Test + public void testAddYearPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddMonthPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddDayPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddHourPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + @Test public void testAddNamedPartition() { createTable("id bigint NOT NULL, category string, ts timestamp, data string"); diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index cdd8a45af691..c5c7ee4d53d7 100644 --- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -342,14 +342,18 @@ public static Term toIcebergTerm(Expression expr) { return org.apache.iceberg.expressions.Expressions.ref(colName); case "bucket": return org.apache.iceberg.expressions.Expressions.bucket(colName, findWidth(transform)); + case "year": case "years": return org.apache.iceberg.expressions.Expressions.year(colName); + case "month": case "months": return org.apache.iceberg.expressions.Expressions.month(colName); case "date": + case "day": case "days": return org.apache.iceberg.expressions.Expressions.day(colName); case "date_hour": + case "hour": case "hours": return org.apache.iceberg.expressions.Expressions.hour(colName); case "truncate": @@ -399,17 +403,21 @@ public static PartitionSpec toPartitionSpec(Schema schema, Transform[] partition case "bucket": builder.bucket(colName, findWidth(transform)); break; + case "year": case "years": builder.year(colName); break; + case "month": case "months": builder.month(colName); break; case "date": + case "day": case "days": builder.day(colName); break; case "date_hour": + case "hour": case "hours": builder.hour(colName); break; diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java index 1411c83ddc65..a6256afcdf65 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java @@ -63,6 +63,26 @@ public void testTransformIgnoreCase() { Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent)); } + @Test + public void testTransformSingularForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hour(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + + @Test + public void testTransformPluralForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hours(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + @Test public void testCreateTable() { Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index a43f2a041b97..0e978e52e570 100644 --- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -27,6 +27,7 @@ import org.apache.spark.sql.connector.catalog.CatalogManager; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -179,6 +180,82 @@ public void testAddHoursPartition() { Assert.assertEquals("Should have new spec field", expected, table.spec()); } + @Test + public void testAddYearPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddMonthPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddDayPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddHourPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + @Test public void testAddNamedPartition() { createTable("id bigint NOT NULL, category string, ts timestamp, data string"); diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 23a53ea9e8c3..d7717e2bfd49 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -359,14 +359,18 @@ public static Term toIcebergTerm(Expression expr) { return org.apache.iceberg.expressions.Expressions.ref(colName); case "bucket": return org.apache.iceberg.expressions.Expressions.bucket(colName, findWidth(transform)); + case "year": case "years": return org.apache.iceberg.expressions.Expressions.year(colName); + case "month": case "months": return org.apache.iceberg.expressions.Expressions.month(colName); case "date": + case "day": case "days": return org.apache.iceberg.expressions.Expressions.day(colName); case "date_hour": + case "hour": case "hours": return org.apache.iceberg.expressions.Expressions.hour(colName); case "truncate": @@ -416,17 +420,21 @@ public static PartitionSpec toPartitionSpec(Schema schema, Transform[] partition case "bucket": builder.bucket(colName, findWidth(transform)); break; + case "year": case "years": builder.year(colName); break; + case "month": case "months": builder.month(colName); break; case "date": + case "day": case "days": builder.day(colName); break; case "date_hour": + case "hour": case "hours": builder.hour(colName); break; diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java index 1411c83ddc65..a6256afcdf65 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java @@ -63,6 +63,26 @@ public void testTransformIgnoreCase() { Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent)); } + @Test + public void testTransformSingularForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hour(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + + @Test + public void testTransformPluralForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hours(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + @Test public void testCreateTable() { Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index a43f2a041b97..0e978e52e570 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -27,6 +27,7 @@ import org.apache.spark.sql.connector.catalog.CatalogManager; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -179,6 +180,82 @@ public void testAddHoursPartition() { Assert.assertEquals("Should have new spec field", expected, table.spec()); } + @Test + public void testAddYearPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddMonthPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddDayPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddHourPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + @Test public void testAddNamedPartition() { createTable("id bigint NOT NULL, category string, ts timestamp, data string"); diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index bbd7986b26d1..62301e9676b8 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -368,14 +368,18 @@ public static Term toIcebergTerm(Expression expr) { return org.apache.iceberg.expressions.Expressions.ref(colName); case "bucket": return org.apache.iceberg.expressions.Expressions.bucket(colName, findWidth(transform)); + case "year": case "years": return org.apache.iceberg.expressions.Expressions.year(colName); + case "month": case "months": return org.apache.iceberg.expressions.Expressions.month(colName); case "date": + case "day": case "days": return org.apache.iceberg.expressions.Expressions.day(colName); case "date_hour": + case "hour": case "hours": return org.apache.iceberg.expressions.Expressions.hour(colName); case "truncate": @@ -425,17 +429,21 @@ public static PartitionSpec toPartitionSpec(Schema schema, Transform[] partition case "bucket": builder.bucket(colName, findWidth(transform)); break; + case "year": case "years": builder.year(colName); break; + case "month": case "months": builder.month(colName); break; case "date": + case "day": case "days": builder.day(colName); break; case "date_hour": + case "hour": case "hours": builder.hour(colName); break; diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java index 01b8b99062a0..ecfd6759b900 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java @@ -63,6 +63,26 @@ public void testTransformIgnoreCase() { Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent)); } + @Test + public void testTransformSingularForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hour(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + + @Test + public void testTransformPluralForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hours(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + @Test public void testCreateTable() { Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index a43f2a041b97..0e978e52e570 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -27,6 +27,7 @@ import org.apache.spark.sql.connector.catalog.CatalogManager; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -179,6 +180,82 @@ public void testAddHoursPartition() { Assert.assertEquals("Should have new spec field", expected, table.spec()); } + @Test + public void testAddYearPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddMonthPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddDayPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + + @Test + public void testAddHourPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); + + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); + } + @Test public void testAddNamedPartition() { createTable("id bigint NOT NULL, category string, ts timestamp, data string"); diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 3611dd7960ac..cfcc3941c748 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -368,14 +368,18 @@ public static Term toIcebergTerm(Expression expr) { return org.apache.iceberg.expressions.Expressions.ref(colName); case "bucket": return org.apache.iceberg.expressions.Expressions.bucket(colName, findWidth(transform)); + case "year": case "years": return org.apache.iceberg.expressions.Expressions.year(colName); + case "month": case "months": return org.apache.iceberg.expressions.Expressions.month(colName); case "date": + case "day": case "days": return org.apache.iceberg.expressions.Expressions.day(colName); case "date_hour": + case "hour": case "hours": return org.apache.iceberg.expressions.Expressions.hour(colName); case "truncate": @@ -425,17 +429,21 @@ public static PartitionSpec toPartitionSpec(Schema schema, Transform[] partition case "bucket": builder.bucket(colName, findWidth(transform)); break; + case "year": case "years": builder.year(colName); break; + case "month": case "months": builder.month(colName); break; case "date": + case "day": case "days": builder.day(colName); break; case "date_hour": + case "hour": case "hours": builder.hour(colName); break; diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java index 01b8b99062a0..ecfd6759b900 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java @@ -63,6 +63,26 @@ public void testTransformIgnoreCase() { Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent)); } + @Test + public void testTransformSingularForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hour(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + + @Test + public void testTransformPluralForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hours(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + @Test public void testCreateTable() { Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent));