From 3b64393d0d28a5fd92bc2880214c6d9cd2baf14f Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 10 Dec 2024 15:55:08 +0800 Subject: [PATCH 01/23] fixup --- gluten-delta/pom.xml | 6 +----- gluten-hudi/pom.xml | 6 +----- gluten-iceberg/pom.xml | 6 +----- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/gluten-delta/pom.xml b/gluten-delta/pom.xml index f971ec607059..49adcf3fc17c 100755 --- a/gluten-delta/pom.xml +++ b/gluten-delta/pom.xml @@ -14,10 +14,6 @@ jar Gluten DeltaLake - - ${project.basedir}/src/main/resources - - org.apache.gluten @@ -94,7 +90,7 @@ target/scala-${scala.binary.version}/test-classes - ${resource.dir} + ${project.basedir}/src/main/resources diff --git a/gluten-hudi/pom.xml b/gluten-hudi/pom.xml index b65217ea5b98..ac937bdfe249 100755 --- a/gluten-hudi/pom.xml +++ b/gluten-hudi/pom.xml @@ -14,10 +14,6 @@ jar Gluten Hudi - - ${project.basedir}/src/main/resources - - org.apache.gluten @@ -109,7 +105,7 @@ target/scala-${scala.binary.version}/test-classes - ${resource.dir} + ${project.basedir}/src/main/resources diff --git a/gluten-iceberg/pom.xml b/gluten-iceberg/pom.xml index bad2ec20e8e8..6efbd4b8da9d 100644 --- a/gluten-iceberg/pom.xml +++ b/gluten-iceberg/pom.xml @@ -14,10 +14,6 @@ jar Gluten Iceberg - - ${project.basedir}/src/main/resources - - org.apache.gluten @@ -101,7 +97,7 @@ target/scala-${scala.binary.version}/test-classes - ${resource.dir} + ${project.basedir}/src/main/resources From f8890c916a441257b2147aaba550221a39b0e67f Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 10 Dec 2024 16:52:42 +0800 Subject: [PATCH 02/23] fixup --- backends-clickhouse/pom.xml | 46 ------------- .../org.apache.gluten.component.Component | 0 .../gluten/component/CHIcebergComponent.scala | 0 .../iceberg/ClickHouseIcebergSuite.scala | 0 backends-velox/pom.xml | 44 ------------- gluten-delta/pom.xml | 5 -- gluten-hudi/pom.xml | 5 -- gluten-iceberg/pom.xml | 5 -- pom.xml | 64 +++++++++++++++++++ 9 files changed, 64 insertions(+), 105 deletions(-) rename backends-clickhouse/{src/main-iceberg/resource => src-iceberg/main/resources}/META-INF/services/org.apache.gluten.component.Component (100%) rename backends-clickhouse/{src/main-iceberg => src-iceberg/main}/scala/org/apache/gluten/component/CHIcebergComponent.scala (100%) rename backends-clickhouse/{src/test-iceberg => src-iceberg/test}/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala (100%) diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml index a6042fd1cce5..1f96422aa13e 100644 --- a/backends-clickhouse/pom.xml +++ b/backends-clickhouse/pom.xml @@ -38,47 +38,6 @@ test - - - - ${project.basedir}/src/main-iceberg/resource - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-iceberg-sources - generate-sources - - add-source - - - - ${project.basedir}/src/main-iceberg/scala - ${project.basedir}/src/main-iceberg/java - - - - - add-iceberg-test-sources - generate-test-sources - - add-test-source - - - - ${project.basedir}/src/test-iceberg/scala - ${project.basedir}/src/test-iceberg/java - - - - - - - @@ -323,11 +282,6 @@ - - - ${project.basedir}/src/main/resources - - target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes diff --git a/backends-clickhouse/src/main-iceberg/resource/META-INF/services/org.apache.gluten.component.Component b/backends-clickhouse/src-iceberg/main/resources/META-INF/services/org.apache.gluten.component.Component similarity index 100% rename from backends-clickhouse/src/main-iceberg/resource/META-INF/services/org.apache.gluten.component.Component rename to backends-clickhouse/src-iceberg/main/resources/META-INF/services/org.apache.gluten.component.Component diff --git a/backends-clickhouse/src/main-iceberg/scala/org/apache/gluten/component/CHIcebergComponent.scala b/backends-clickhouse/src-iceberg/main/scala/org/apache/gluten/component/CHIcebergComponent.scala similarity index 100% rename from backends-clickhouse/src/main-iceberg/scala/org/apache/gluten/component/CHIcebergComponent.scala rename to backends-clickhouse/src-iceberg/main/scala/org/apache/gluten/component/CHIcebergComponent.scala diff --git a/backends-clickhouse/src/test-iceberg/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala b/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala similarity index 100% rename from backends-clickhouse/src/test-iceberg/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala rename to backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml index 2719cc655943..a5b92017b865 100755 --- a/backends-velox/pom.xml +++ b/backends-velox/pom.xml @@ -59,47 +59,6 @@ test - - - - ${project.basedir}/src/main-iceberg/resource - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-iceberg-sources - generate-sources - - add-source - - - - ${project.basedir}/src/main-iceberg/scala - ${project.basedir}/src/main-iceberg/java - - - - - add-iceberg-test-sources - generate-test-sources - - add-test-source - - - - ${project.basedir}/src/test-iceberg/scala - ${project.basedir}/src/test-iceberg/java - - - - - - - @@ -320,9 +279,6 @@ ${cpp.releases.dir} ${platform}/${arch} - - ${project.basedir}/src/main/resources - target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes diff --git a/gluten-delta/pom.xml b/gluten-delta/pom.xml index 49adcf3fc17c..d8cf7618d802 100755 --- a/gluten-delta/pom.xml +++ b/gluten-delta/pom.xml @@ -88,11 +88,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - ${project.basedir}/src/main/resources - - net.alchim31.maven diff --git a/gluten-hudi/pom.xml b/gluten-hudi/pom.xml index ac937bdfe249..7900182f853a 100755 --- a/gluten-hudi/pom.xml +++ b/gluten-hudi/pom.xml @@ -103,11 +103,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - ${project.basedir}/src/main/resources - - net.alchim31.maven diff --git a/gluten-iceberg/pom.xml b/gluten-iceberg/pom.xml index 6efbd4b8da9d..c8c811674a54 100644 --- a/gluten-iceberg/pom.xml +++ b/gluten-iceberg/pom.xml @@ -95,11 +95,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - ${project.basedir}/src/main/resources - - net.alchim31.maven diff --git a/pom.xml b/pom.xml index 9b27a6839712..03286f314013 100644 --- a/pom.xml +++ b/pom.xml @@ -449,6 +449,70 @@ gluten-iceberg + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-iceberg-sources + generate-sources + + add-source + + + + ${project.basedir}/src-iceberg/main/scala + ${project.basedir}/src-iceberg/main/java + + + + + add-iceberg-resources + generate-resources + + add-resource + + + + + ${project.basedir}/src-iceberg/main/resources + + + + + + add-iceberg-test-sources + generate-test-sources + + add-test-source + + + + ${project.basedir}/src-iceberg/test/scala + ${project.basedir}/src-iceberg/test/java + + + + + add-iceberg-test-resources + generate-test-resources + + add-test-resource + + + + + ${project.basedir}/src-iceberg/test/resources + + + + + + + + hudi From 7e717bb011afa162a2ce64559bb042fae1632592 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 10 Dec 2024 16:54:16 +0800 Subject: [PATCH 03/23] fixup --- .../META-INF/services/org.apache.gluten.component.Component | 0 .../scala/org/apache/gluten/component/VeloxIcebergComponent.scala | 0 .../scala/org/apache/gluten/execution/VeloxIcebergSuite.scala | 0 .../scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename backends-velox/{src/main-iceberg/resource => src-iceberg/main/resources}/META-INF/services/org.apache.gluten.component.Component (100%) rename backends-velox/{src/main-iceberg => src-iceberg/main}/scala/org/apache/gluten/component/VeloxIcebergComponent.scala (100%) rename backends-velox/{src/test-iceberg => src-iceberg/test}/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala (100%) rename backends-velox/{src/test-iceberg => src-iceberg/test}/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala (100%) diff --git a/backends-velox/src/main-iceberg/resource/META-INF/services/org.apache.gluten.component.Component b/backends-velox/src-iceberg/main/resources/META-INF/services/org.apache.gluten.component.Component similarity index 100% rename from backends-velox/src/main-iceberg/resource/META-INF/services/org.apache.gluten.component.Component rename to backends-velox/src-iceberg/main/resources/META-INF/services/org.apache.gluten.component.Component diff --git a/backends-velox/src/main-iceberg/scala/org/apache/gluten/component/VeloxIcebergComponent.scala b/backends-velox/src-iceberg/main/scala/org/apache/gluten/component/VeloxIcebergComponent.scala similarity index 100% rename from backends-velox/src/main-iceberg/scala/org/apache/gluten/component/VeloxIcebergComponent.scala rename to backends-velox/src-iceberg/main/scala/org/apache/gluten/component/VeloxIcebergComponent.scala diff --git a/backends-velox/src/test-iceberg/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala similarity index 100% rename from backends-velox/src/test-iceberg/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala rename to backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala diff --git a/backends-velox/src/test-iceberg/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala similarity index 100% rename from backends-velox/src/test-iceberg/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala rename to backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala From 1f6732f63baa2c5c6f54ef97de4cf389c758e817 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 10 Dec 2024 17:06:07 +0800 Subject: [PATCH 04/23] fixup --- .../io/delta/tables/ClickhouseTable.scala | 0 .../parser/GlutenCacheFilesSqlParser.scala | 0 .../parser/GlutenClickhouseSqlParser.scala | 0 .../ClickhouseOptimisticTransaction.scala | 0 .../apache/spark/sql/delta/DeltaAdapter.scala | 0 .../org/apache/spark/sql/delta/DeltaLog.scala | 0 .../org/apache/spark/sql/delta/Snapshot.scala | 0 .../sql/delta/catalog/ClickHouseTableV2.scala | 0 .../sql/delta/commands/DeleteCommand.scala | 0 .../sql/delta/commands/MergeIntoCommand.scala | 0 .../delta/commands/OptimizeTableCommand.scala | 0 .../OptimizeTableCommandOverwrites.scala | 0 .../sql/delta/commands/UpdateCommand.scala | 0 .../sql/delta/commands/VacuumCommand.scala | 0 .../MergeTreeDelayedCommitProtocol.scala | 0 .../CHOptimizeMetadataOnlyDeltaQuery.scala | 0 .../sql/execution/CHDeltaColumnarWrite.scala | 0 .../v2/clickhouse/ClickHouseDataSource.scala | 0 .../clickhouse/ClickHouseSparkCatalog.scala | 0 .../source/DeltaMergeTreeFileFormat.scala | 0 .../spark/gluten/delta/DeltaStatsUtils.scala | 0 .../io/delta/tables/ClickhouseTable.scala | 0 .../parser/GlutenCacheFilesSqlParser.scala | 0 .../parser/GlutenClickhouseSqlParser.scala | 0 .../ClickhouseOptimisticTransaction.scala | 0 .../apache/spark/sql/delta/DeltaAdapter.scala | 0 .../org/apache/spark/sql/delta/DeltaLog.scala | 0 .../org/apache/spark/sql/delta/Snapshot.scala | 0 .../sql/delta/catalog/ClickHouseTableV2.scala | 0 .../sql/delta/commands/DeleteCommand.scala | 0 .../sql/delta/commands/MergeIntoCommand.scala | 0 .../delta/commands/OptimizeTableCommand.scala | 0 .../OptimizeTableCommandOverwrites.scala | 0 .../sql/delta/commands/UpdateCommand.scala | 0 .../sql/delta/commands/VacuumCommand.scala | 0 .../MergeTreeDelayedCommitProtocol.scala | 0 .../CHOptimizeMetadataOnlyDeltaQuery.scala | 0 .../sql/delta/stats/PrepareDeltaScan.scala | 0 .../sql/execution/CHDeltaColumnarWrite.scala | 0 .../v2/clickhouse/ClickHouseDataSource.scala | 0 .../clickhouse/ClickHouseSparkCatalog.scala | 0 .../source/DeltaMergeTreeFileFormat.scala | 0 .../spark/gluten/delta/DeltaStatsUtils.scala | 0 .../io/delta/tables/ClickhouseTable.scala | 0 .../parser/GlutenCacheFilesSqlParser.scala | 0 .../parser/GlutenClickhouseSqlParser.scala | 0 .../ClickhouseOptimisticTransaction.scala | 0 .../apache/spark/sql/delta/DeltaAdapter.scala | 0 .../org/apache/spark/sql/delta/DeltaLog.scala | 0 .../org/apache/spark/sql/delta/Snapshot.scala | 0 .../sql/delta/catalog/ClickHouseTableV2.scala | 0 .../sql/delta/commands/DeleteCommand.scala | 0 .../delta/commands/OptimizeTableCommand.scala | 0 .../OptimizeTableCommandOverwrites.scala | 0 .../sql/delta/commands/UpdateCommand.scala | 0 .../sql/delta/commands/VacuumCommand.scala | 0 .../commands/merge/ClassicMergeExecutor.scala | 0 .../MergeTreeDelayedCommitProtocol.scala | 0 .../CHOptimizeMetadataOnlyDeltaQuery.scala | 0 .../execution/FileDeltaColumnarWrite.scala | 0 .../MergeTreeDeltaColumnarWrite.scala | 0 .../MergeTreeFileFormatDataWriter.scala | 0 .../MergeTreeFileFormatWriter.scala | 0 .../v2/clickhouse/ClickHouseDataSource.scala | 0 .../clickhouse/ClickHouseSparkCatalog.scala | 0 .../source/DeltaMergeTreeFileFormat.scala | 0 .../spark/gluten/delta/DeltaStatsUtils.scala | 0 pom.xml | 71 +++++++++++++++++++ 68 files changed, 71 insertions(+) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/io/delta/tables/ClickhouseTable.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/DeltaAdapter.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/DeltaLog.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/Snapshot.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/commands/DeleteCommand.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/commands/UpdateCommand.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/commands/VacuumCommand.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala (100%) rename backends-clickhouse/{src/test/delta-20 => src-delta-20/test/scala}/org/apache/spark/gluten/delta/DeltaStatsUtils.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/io/delta/tables/ClickhouseTable.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/DeltaAdapter.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/DeltaLog.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/Snapshot.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/commands/DeleteCommand.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/commands/UpdateCommand.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/commands/VacuumCommand.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala (100%) rename backends-clickhouse/{src/test/delta-23 => src-delta-23/test/scala}/org/apache/spark/gluten/delta/DeltaStatsUtils.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/io/delta/tables/ClickhouseTable.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/DeltaAdapter.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/DeltaLog.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/Snapshot.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/commands/DeleteCommand.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/commands/UpdateCommand.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/commands/VacuumCommand.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/FileDeltaColumnarWrite.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/MergeTreeDeltaColumnarWrite.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala (100%) rename backends-clickhouse/{src/test/delta-32 => src-delta-32/test/scala}/org/apache/spark/gluten/delta/DeltaStatsUtils.scala (100%) diff --git a/backends-clickhouse/src/main/delta-20/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src-delta-20/main/scala/io/delta/tables/ClickhouseTable.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/io/delta/tables/ClickhouseTable.scala rename to backends-clickhouse/src-delta-20/main/scala/io/delta/tables/ClickhouseTable.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaAdapter.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaAdapter.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/DeltaLog.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/DeltaLog.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/Snapshot.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/Snapshot.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/Snapshot.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/DeleteCommand.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/DeleteCommand.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/UpdateCommand.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/UpdateCommand.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala diff --git a/backends-clickhouse/src/test/delta-20/org/apache/spark/gluten/delta/DeltaStatsUtils.scala b/backends-clickhouse/src-delta-20/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala similarity index 100% rename from backends-clickhouse/src/test/delta-20/org/apache/spark/gluten/delta/DeltaStatsUtils.scala rename to backends-clickhouse/src-delta-20/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala diff --git a/backends-clickhouse/src/main/delta-23/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src-delta-23/main/scala/io/delta/tables/ClickhouseTable.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/io/delta/tables/ClickhouseTable.scala rename to backends-clickhouse/src-delta-23/main/scala/io/delta/tables/ClickhouseTable.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaAdapter.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaAdapter.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/DeltaLog.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaLog.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/DeltaLog.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/Snapshot.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/Snapshot.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/Snapshot.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/DeleteCommand.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/DeleteCommand.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/UpdateCommand.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/UpdateCommand.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala diff --git a/backends-clickhouse/src/test/delta-23/org/apache/spark/gluten/delta/DeltaStatsUtils.scala b/backends-clickhouse/src-delta-23/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala similarity index 100% rename from backends-clickhouse/src/test/delta-23/org/apache/spark/gluten/delta/DeltaStatsUtils.scala rename to backends-clickhouse/src-delta-23/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala diff --git a/backends-clickhouse/src/main/delta-32/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src-delta-32/main/scala/io/delta/tables/ClickhouseTable.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/io/delta/tables/ClickhouseTable.scala rename to backends-clickhouse/src-delta-32/main/scala/io/delta/tables/ClickhouseTable.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaAdapter.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaAdapter.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/DeltaLog.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaLog.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/DeltaLog.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/Snapshot.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/Snapshot.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/DeleteCommand.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/DeleteCommand.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/UpdateCommand.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/UpdateCommand.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/FileDeltaColumnarWrite.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/FileDeltaColumnarWrite.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/FileDeltaColumnarWrite.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/FileDeltaColumnarWrite.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/MergeTreeDeltaColumnarWrite.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/MergeTreeDeltaColumnarWrite.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/MergeTreeDeltaColumnarWrite.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/MergeTreeDeltaColumnarWrite.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala diff --git a/backends-clickhouse/src/test/delta-32/org/apache/spark/gluten/delta/DeltaStatsUtils.scala b/backends-clickhouse/src-delta-32/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala similarity index 100% rename from backends-clickhouse/src/test/delta-32/org/apache/spark/gluten/delta/DeltaStatsUtils.scala rename to backends-clickhouse/src-delta-32/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala diff --git a/pom.xml b/pom.xml index 03286f314013..e799e76beff1 100644 --- a/pom.xml +++ b/pom.xml @@ -440,6 +440,77 @@ gluten-delta + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-delta-sources + generate-sources + + add-source + + + + ${project.basedir}/src-delta/main/scala + ${project.basedir}/src-delta/main/java + ${project.basedir}/src-delta-${delta.binary.version}/main/scala + ${project.basedir}/src-delta-${delta.binary.version}/main/java + + + + + add-delta-resources + generate-resources + + add-resource + + + + + ${project.basedir}/src-delta/main/resources + ${project.basedir}/src-delta-${delta.binary.version}/main/resources + + + + + + add-delta-test-sources + generate-test-sources + + add-test-source + + + + ${project.basedir}/src-delta/test/scala + ${project.basedir}/src-delta/test/java + ${project.basedir}/src-delta-${delta.binary.version}/test/scala + ${project.basedir}/src-delta-${delta.binary.version}/test/java + + + + + add-delta-test-resources + generate-test-resources + + add-test-resource + + + + + ${project.basedir}/src-delta/test/resources + ${project.basedir}/src-delta-${delta.binary.version}/test/resources + + + + + + + + iceberg From c7cb3ef37ddc36c76163f7101a1158ac77cd67b8 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 10 Dec 2024 17:22:00 +0800 Subject: [PATCH 05/23] fixup --- backends-clickhouse/pom.xml | 44 +-- backends-velox/pom.xml | 37 --- .../gluten/execution/VeloxDeltaSuite.scala | 0 .../execution/VeloxTPCHDeltaSuite.scala | 0 gluten-delta/pom.xml | 30 +-- .../DeltaFilterExecTransformer.scala | 0 .../DeltaProjectExecTransformer.scala | 0 .../DeltaFilterExecTransformer.scala | 27 ++ .../DeltaProjectExecTransformer.scala | 27 ++ .../DeltaFilterExecTransformer.scala | 27 ++ .../DeltaProjectExecTransformer.scala | 27 ++ .../DeltaFilterExecTransformer.scala | 0 .../DeltaProjectExecTransformer.scala | 0 ...xecution.DataSourceScanTransformerRegister | 0 ...extension.columnar.RewriteTransformerRules | 0 .../execution/DeltaScanTransformer.scala | 0 .../DeltaScanTransformerProvider.scala | 0 .../DeltaRewriteTransformerRules.scala | 0 .../gluten/execution/VeloxDeltaSuite.scala | 253 ++++++++++++++++++ .../execution/VeloxTPCHDeltaSuite.scala | 57 ++++ .../rel/IcebergLocalFilesBuilder.java | 0 .../substrait/rel/IcebergLocalFilesNode.java | 0 .../execution/IcebergScanTransformer.scala | 0 .../gluten/execution/OffloadIcebergScan.scala | 0 .../source/GlutenIcebergSourceUtil.scala | 0 .../iceberg/spark/source/TypeUtil.scala | 0 .../gluten/execution/IcebergSuite.scala | 0 pom.xml | 2 +- 28 files changed, 430 insertions(+), 101 deletions(-) rename backends-velox/{src/test-delta => src-delta/test}/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala (100%) rename backends-velox/{src/test-delta => src-delta/test}/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala (100%) rename gluten-delta/{src => src-delta-20}/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala (100%) rename gluten-delta/{src => src-delta-20}/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala (100%) create mode 100644 gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala create mode 100644 gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala create mode 100644 gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala create mode 100644 gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala rename gluten-delta/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/gluten/execution/DeltaFilterExecTransformer.scala (100%) rename gluten-delta/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/gluten/execution/DeltaProjectExecTransformer.scala (100%) rename gluten-delta/{src => src-delta}/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister (100%) rename gluten-delta/{src => src-delta}/main/resources/META-INF/services/org.apache.gluten.extension.columnar.RewriteTransformerRules (100%) rename gluten-delta/{src => src-delta}/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala (100%) rename gluten-delta/{src => src-delta}/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala (100%) rename gluten-delta/{src => src-delta}/main/scala/org/apache/gluten/extension/DeltaRewriteTransformerRules.scala (100%) create mode 100644 gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala create mode 100644 gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala rename gluten-iceberg/{src => src-iceberg}/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesBuilder.java (100%) rename gluten-iceberg/{src => src-iceberg}/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java (100%) rename gluten-iceberg/{src => src-iceberg}/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala (100%) rename gluten-iceberg/{src => src-iceberg}/main/scala/org/apache/gluten/execution/OffloadIcebergScan.scala (100%) rename gluten-iceberg/{src => src-iceberg}/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala (100%) rename gluten-iceberg/{src => src-iceberg}/main/scala/org/apache/iceberg/spark/source/TypeUtil.scala (100%) rename gluten-iceberg/{src => src-iceberg}/test/scala/org/apache/gluten/execution/IcebergSuite.scala (100%) diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml index 1f96422aa13e..4b67d8c30d29 100644 --- a/backends-clickhouse/pom.xml +++ b/backends-clickhouse/pom.xml @@ -390,15 +390,15 @@ src/main/scala/**/*.scala src/test/scala/**/*.scala - src/main/delta-${delta.binary.version}/**/*.scala - src/test/delta-${delta.binary.version}/**/*.scala + src-delta-${delta.binary.version}/main/delta/**/*.scala + src-delta-${delta.binary.version}/test/delta/**/*.scala - src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/commands/*.scala - src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/commands/merge/*.scala - src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/stats/*.scala - src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/DeltaLog.scala - src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/Snapshot.scala + src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/commands/*.scala + src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/commands/merge/*.scala + src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/stats/*.scala + src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/DeltaLog.scala + src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/Snapshot.scala @@ -438,36 +438,6 @@ - - org.codehaus.mojo - build-helper-maven-plugin - - - add-sources - generate-sources - - add-source - - - - src/main/delta-${delta.binary.version} - - - - - add-test-sources - generate-test-sources - - add-test-source - - - - src/test/delta-${delta.binary.version} - - - - - org.antlr antlr4-maven-plugin diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml index a5b92017b865..7a0dbab90627 100755 --- a/backends-velox/pom.xml +++ b/backends-velox/pom.xml @@ -60,7 +60,6 @@ - delta @@ -82,42 +81,6 @@ provided - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-delta-sources - generate-sources - - add-source - - - - ${project.basedir}/src/main-delta/scala - ${project.basedir}/src/main-delta/java - - - - - add-delta-test-sources - generate-test-sources - - add-test-source - - - - ${project.basedir}/src/test-delta/scala - ${project.basedir}/src/test-delta/java - - - - - - - diff --git a/backends-velox/src/test-delta/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala b/backends-velox/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala similarity index 100% rename from backends-velox/src/test-delta/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala rename to backends-velox/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala diff --git a/backends-velox/src/test-delta/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala b/backends-velox/src-delta/test/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala similarity index 100% rename from backends-velox/src/test-delta/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala rename to backends-velox/src-delta/test/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala diff --git a/gluten-delta/pom.xml b/gluten-delta/pom.xml index d8cf7618d802..cf2fdd7518fa 100755 --- a/gluten-delta/pom.xml +++ b/gluten-delta/pom.xml @@ -114,10 +114,10 @@ ${project.basedir}/../.scalafmt.conf - src/main/scala/**/*.scala - src/test/scala/**/*.scala - src/main/delta-${delta.binary.version}/**/*.scala - src/test/delta-${delta.binary.version}/**/*.scala + src-delta/main/scala/**/*.scala + src-delta/test/scala/**/*.scala + src-delta-${delta.binary.version}/main/scala/**/*.scala + src-delta-${delta.binary.version}/test/scala/**/*.scala @@ -142,28 +142,6 @@ - - org.apache.maven.plugins - maven-resources-plugin - - - copy-resources - generate-sources - - copy-resources - - - src/main/scala/org/apache/gluten/execution - - - src/main/delta-${delta.binary.version}/org/apache/gluten/execution - - - true - - - - diff --git a/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala b/gluten-delta/src-delta-20/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala similarity index 100% rename from gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala rename to gluten-delta/src-delta-20/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala diff --git a/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala b/gluten-delta/src-delta-20/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala similarity index 100% rename from gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala rename to gluten-delta/src-delta-20/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala diff --git a/gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala b/gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala new file mode 100644 index 000000000000..ca4665c0d0cb --- /dev/null +++ b/gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.execution.SparkPlan + +case class DeltaFilterExecTransformer(condition: Expression, child: SparkPlan) + extends FilterExecTransformerBase(condition, child) { + + override protected def withNewChildInternal(newChild: SparkPlan): DeltaFilterExecTransformer = + copy(child = newChild) +} diff --git a/gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala b/gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala new file mode 100644 index 000000000000..9b720b19c5ba --- /dev/null +++ b/gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.spark.sql.catalyst.expressions.NamedExpression +import org.apache.spark.sql.execution.SparkPlan + +case class DeltaProjectExecTransformer(projectList: Seq[NamedExpression], child: SparkPlan) + extends ProjectExecTransformerBase(projectList, child) { + + override protected def withNewChildInternal(newChild: SparkPlan): DeltaProjectExecTransformer = + copy(child = newChild) +} diff --git a/gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala b/gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala new file mode 100644 index 000000000000..ca4665c0d0cb --- /dev/null +++ b/gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.execution.SparkPlan + +case class DeltaFilterExecTransformer(condition: Expression, child: SparkPlan) + extends FilterExecTransformerBase(condition, child) { + + override protected def withNewChildInternal(newChild: SparkPlan): DeltaFilterExecTransformer = + copy(child = newChild) +} diff --git a/gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala b/gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala new file mode 100644 index 000000000000..9b720b19c5ba --- /dev/null +++ b/gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.spark.sql.catalyst.expressions.NamedExpression +import org.apache.spark.sql.execution.SparkPlan + +case class DeltaProjectExecTransformer(projectList: Seq[NamedExpression], child: SparkPlan) + extends ProjectExecTransformerBase(projectList, child) { + + override protected def withNewChildInternal(newChild: SparkPlan): DeltaProjectExecTransformer = + copy(child = newChild) +} diff --git a/gluten-delta/src/main/delta-32/org/apache/gluten/execution/DeltaFilterExecTransformer.scala b/gluten-delta/src-delta-32/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala similarity index 100% rename from gluten-delta/src/main/delta-32/org/apache/gluten/execution/DeltaFilterExecTransformer.scala rename to gluten-delta/src-delta-32/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala diff --git a/gluten-delta/src/main/delta-32/org/apache/gluten/execution/DeltaProjectExecTransformer.scala b/gluten-delta/src-delta-32/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala similarity index 100% rename from gluten-delta/src/main/delta-32/org/apache/gluten/execution/DeltaProjectExecTransformer.scala rename to gluten-delta/src-delta-32/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala diff --git a/gluten-delta/src/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister b/gluten-delta/src-delta/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister similarity index 100% rename from gluten-delta/src/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister rename to gluten-delta/src-delta/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister diff --git a/gluten-delta/src/main/resources/META-INF/services/org.apache.gluten.extension.columnar.RewriteTransformerRules b/gluten-delta/src-delta/main/resources/META-INF/services/org.apache.gluten.extension.columnar.RewriteTransformerRules similarity index 100% rename from gluten-delta/src/main/resources/META-INF/services/org.apache.gluten.extension.columnar.RewriteTransformerRules rename to gluten-delta/src-delta/main/resources/META-INF/services/org.apache.gluten.extension.columnar.RewriteTransformerRules diff --git a/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala b/gluten-delta/src-delta/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala similarity index 100% rename from gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala rename to gluten-delta/src-delta/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala diff --git a/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala b/gluten-delta/src-delta/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala similarity index 100% rename from gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala rename to gluten-delta/src-delta/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala diff --git a/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaRewriteTransformerRules.scala b/gluten-delta/src-delta/main/scala/org/apache/gluten/extension/DeltaRewriteTransformerRules.scala similarity index 100% rename from gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaRewriteTransformerRules.scala rename to gluten-delta/src-delta/main/scala/org/apache/gluten/extension/DeltaRewriteTransformerRules.scala diff --git a/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala b/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala new file mode 100644 index 000000000000..caf86189a2d4 --- /dev/null +++ b/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.spark.SparkConf +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType} + +import scala.collection.JavaConverters._ + +abstract class DeltaSuite extends WholeStageTransformerSuite { + protected val rootPath: String = getClass.getResource("/").getPath + // FIXME: This folder doesn't exist in module gluten-delta so should be provided by + // backend modules that rely on this suite. + override protected val resourcePath: String = "/tpch-data-parquet" + override protected val fileFormat: String = "parquet" + + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") + .set("spark.sql.files.maxPartitionBytes", "1g") + .set("spark.sql.shuffle.partitions", "1") + .set("spark.memory.offHeap.size", "2g") + .set("spark.unsafe.exceptionOnMemoryLeak", "true") + .set("spark.sql.autoBroadcastJoinThreshold", "-1") + .set("spark.sql.sources.useV1SourceList", "avro") + .set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") + .set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") + } + + // IdMapping is supported in Delta 2.2 (related to Spark3.3.1) + testWithSpecifiedSparkVersion("column mapping mode = id", Some("3.3")) { + withTable("delta_cm1") { + spark.sql(s""" + |create table delta_cm1 (id int, name string) using delta + |tblproperties ("delta.columnMapping.mode"= "id") + |""".stripMargin) + spark.sql(s""" + |insert into delta_cm1 values (1, "v1"), (2, "v2") + |""".stripMargin) + val df1 = runQueryAndCompare("select * from delta_cm1") { _ => } + checkLengthAndPlan(df1, 2) + checkAnswer(df1, Row(1, "v1") :: Row(2, "v2") :: Nil) + + val df2 = runQueryAndCompare("select name from delta_cm1 where id = 2") { _ => } + checkLengthAndPlan(df2, 1) + checkAnswer(df2, Row("v2") :: Nil) + } + } + + // NameMapping is supported in Delta 2.0 (related to Spark3.2.0) + testWithSpecifiedSparkVersion("column mapping mode = name", Some("3.2")) { + withTable("delta_cm2") { + spark.sql(s""" + |create table delta_cm2 (id int, name string) using delta + |tblproperties ("delta.columnMapping.mode"= "name") + |""".stripMargin) + spark.sql(s""" + |insert into delta_cm2 values (1, "v1"), (2, "v2") + |""".stripMargin) + val df1 = runQueryAndCompare("select * from delta_cm2") { _ => } + checkLengthAndPlan(df1, 2) + checkAnswer(df1, Row(1, "v1") :: Row(2, "v2") :: Nil) + + val df2 = runQueryAndCompare("select name from delta_cm2 where id = 2") { _ => } + checkLengthAndPlan(df2, 1) + checkAnswer(df2, Row("v2") :: Nil) + } + } + + testWithSpecifiedSparkVersion("delta: time travel", Some("3.3")) { + withTable("delta_tm") { + spark.sql(s""" + |create table delta_tm (id int, name string) using delta + |""".stripMargin) + spark.sql(s""" + |insert into delta_tm values (1, "v1"), (2, "v2") + |""".stripMargin) + spark.sql(s""" + |insert into delta_tm values (3, "v3"), (4, "v4") + |""".stripMargin) + val df1 = runQueryAndCompare("select * from delta_tm VERSION AS OF 1") { _ => } + checkLengthAndPlan(df1, 2) + checkAnswer(df1, Row(1, "v1") :: Row(2, "v2") :: Nil) + val df2 = runQueryAndCompare("select * from delta_tm VERSION AS OF 2") { _ => } + checkLengthAndPlan(df2, 4) + checkAnswer(df2, Row(1, "v1") :: Row(2, "v2") :: Row(3, "v3") :: Row(4, "v4") :: Nil) + val df3 = runQueryAndCompare("select name from delta_tm VERSION AS OF 2 where id = 2") { + _ => + } + checkLengthAndPlan(df3, 1) + checkAnswer(df3, Row("v2") :: Nil) + } + } + + testWithSpecifiedSparkVersion("delta: partition filters", Some("3.2")) { + withTable("delta_pf") { + spark.sql(s""" + |create table delta_pf (id int, name string) using delta partitioned by (name) + |""".stripMargin) + spark.sql(s""" + |insert into delta_pf values (1, "v1"), (2, "v2"), (3, "v1"), (4, "v2") + |""".stripMargin) + val df1 = runQueryAndCompare("select * from delta_pf where name = 'v1'") { _ => } + val deltaScanTransformer = df1.queryExecution.executedPlan.collect { + case f: DeltaScanTransformer => f + }.head + // No data filters as only partition filters exist + assert(deltaScanTransformer.filterExprs().size == 0) + checkLengthAndPlan(df1, 2) + checkAnswer(df1, Row(1, "v1") :: Row(3, "v1") :: Nil) + } + } + + testWithSpecifiedSparkVersion("basic test with stats.skipping disabled", Some("3.2")) { + withTable("delta_test2") { + withSQLConf("spark.databricks.delta.stats.skipping" -> "false") { + spark.sql(s""" + |create table delta_test2 (id int, name string) using delta + |""".stripMargin) + spark.sql(s""" + |insert into delta_test2 values (1, "v1"), (2, "v2") + |""".stripMargin) + val df1 = runQueryAndCompare("select * from delta_test2") { _ => } + checkLengthAndPlan(df1, 2) + checkAnswer(df1, Row(1, "v1") :: Row(2, "v2") :: Nil) + + val df2 = runQueryAndCompare("select name from delta_test2 where id = 2") { _ => } + checkLengthAndPlan(df2, 1) + checkAnswer(df2, Row("v2") :: Nil) + } + } + } + + testWithSpecifiedSparkVersion("column mapping with complex type", Some("3.2")) { + withTable("t1") { + val simpleNestedSchema = new StructType() + .add("a", StringType, true) + .add("b", new StructType().add("c", StringType, true).add("d", IntegerType, true)) + .add("map", MapType(StringType, StringType), true) + .add("arr", ArrayType(IntegerType), true) + + val simpleNestedData = spark.createDataFrame( + Seq( + Row("str1", Row("str1.1", 1), Map("k1" -> "v1"), Array(1, 11)), + Row("str2", Row("str1.2", 2), Map("k2" -> "v2"), Array(2, 22))).asJava, + simpleNestedSchema) + + spark.sql( + """CREATE TABLE t1 + | (a STRING,b STRUCT,map MAP,arr ARRAY) + | USING DELTA + | PARTITIONED BY (`a`) + | TBLPROPERTIES ('delta.columnMapping.mode' = 'name')""".stripMargin) + + simpleNestedData.write.format("delta").mode("append").saveAsTable("t1") + + val df1 = runQueryAndCompare("select * from t1") { _ => } + checkAnswer( + df1, + Seq( + Row("str1", Row("str1.1", 1), Map("k1" -> "v1"), Array(1, 11)), + Row("str2", Row("str1.2", 2), Map("k2" -> "v2"), Array(2, 22)))) + spark.sql(s"Alter table t1 RENAME COLUMN b to b1") + spark.sql( + "insert into t1 " + + "values ('str3', struct('str1.3', 3), map('k3', 'v3'), array(3, 33))") + + val df2 = runQueryAndCompare("select b1 from t1") { _ => } + checkAnswer(df2, Seq(Row(Row("str1.1", 1)), Row(Row("str1.2", 2)), Row(Row("str1.3", 3)))) + + spark.sql(s"Alter table t1 RENAME COLUMN b1.c to c1") + val df3 = runQueryAndCompare("select * from t1") { _ => } + checkAnswer( + df3, + Seq( + Row("str1", Row("str1.1", 1), Map("k1" -> "v1"), Array(1, 11)), + Row("str2", Row("str1.2", 2), Map("k2" -> "v2"), Array(2, 22)), + Row("str3", Row("str1.3", 3), Map("k3" -> "v3"), Array(3, 33)) + ) + ) + } + } + + testWithSpecifiedSparkVersion("deletion vector", Some("3.4")) { + withTempPath { + p => + import testImplicits._ + val path = p.getCanonicalPath + val df1 = Seq(1, 2, 3, 4, 5).toDF("id") + val values2 = Seq(6, 7, 8, 9, 10) + val df2 = values2.toDF("id") + df1.union(df2).coalesce(1).write.format("delta").save(path) + spark.sql( + s"ALTER TABLE delta.`$path` SET TBLPROPERTIES ('delta.enableDeletionVectors' = true)") + checkAnswer(spark.read.format("delta").load(path), df1.union(df2)) + spark.sql(s"DELETE FROM delta.`$path` WHERE id IN (${values2.mkString(", ")})") + checkAnswer(spark.read.format("delta").load(path), df1) + } + } + + testWithSpecifiedSparkVersion("delta: push down input_file_name expression", Some("3.2")) { + withTable("source_table") { + withTable("target_table") { + spark.sql(s""" + |CREATE TABLE source_table(id INT, name STRING, age INT) USING delta; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE target_table(id INT, name STRING, age INT) USING delta; + | + |""".stripMargin) + + spark.sql(s""" + |INSERT INTO source_table VALUES(1, 'a', 10),(2, 'b', 20); + |""".stripMargin) + + spark.sql(s""" + |INSERT INTO target_table VALUES(1, 'c', 10),(3, 'c', 30); + |""".stripMargin) + + spark.sql(s""" + |MERGE INTO target_table AS target + |USING source_table AS source + |ON target.id = source.id + |WHEN MATCHED THEN + |UPDATE SET + | target.name = source.name, + | target.age = source.age + |WHEN NOT MATCHED THEN + |INSERT (id, name, age) VALUES (source.id, source.name, source.age); + |""".stripMargin) + + val df1 = runQueryAndCompare("SELECT * FROM target_table") { _ => } + checkAnswer(df1, Row(1, "a", 10) :: Row(2, "b", 20) :: Row(3, "c", 30) :: Nil) + } + } + } +} diff --git a/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala b/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala new file mode 100644 index 000000000000..238f5ca56f0a --- /dev/null +++ b/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.spark.SparkConf + +import java.io.File + +class VeloxTPCHDeltaSuite extends VeloxTPCHSuite { + protected val tpchBasePath: String = + getClass.getResource("/").getPath + "../../../src/test/resources" + + override protected val resourcePath: String = + new File(tpchBasePath, "tpch-data-parquet").getCanonicalPath + + override protected val queriesResults: String = + new File(tpchBasePath, "queries-output").getCanonicalPath + + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.executor.memory", "4g") + .set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") + .set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") + } + + override protected def createTPCHNotNullTables(): Unit = { + TPCHTables + .map(_.name) + .map { + table => + val tablePath = new File(resourcePath, table).getAbsolutePath + val tableDF = spark.read.format(fileFormat).load(tablePath) + tableDF.write.format("delta").mode("append").saveAsTable(table) + (table, tableDF) + } + .toMap + } + + override protected def afterAll(): Unit = { + TPCHTables.map(_.name).foreach(table => spark.sql(s"DROP TABLE IF EXISTS $table")) + super.afterAll() + } +} diff --git a/gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesBuilder.java b/gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesBuilder.java similarity index 100% rename from gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesBuilder.java rename to gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesBuilder.java diff --git a/gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java b/gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java similarity index 100% rename from gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java rename to gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java diff --git a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala b/gluten-iceberg/src-iceberg/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala similarity index 100% rename from gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala rename to gluten-iceberg/src-iceberg/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala diff --git a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/OffloadIcebergScan.scala b/gluten-iceberg/src-iceberg/main/scala/org/apache/gluten/execution/OffloadIcebergScan.scala similarity index 100% rename from gluten-iceberg/src/main/scala/org/apache/gluten/execution/OffloadIcebergScan.scala rename to gluten-iceberg/src-iceberg/main/scala/org/apache/gluten/execution/OffloadIcebergScan.scala diff --git a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala b/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala similarity index 100% rename from gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala rename to gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala diff --git a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/TypeUtil.scala b/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/TypeUtil.scala similarity index 100% rename from gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/TypeUtil.scala rename to gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/TypeUtil.scala diff --git a/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala b/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala similarity index 100% rename from gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala rename to gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala diff --git a/pom.xml b/pom.xml index e799e76beff1..57683e48ddad 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ 2.3.0 23 4.8 - 0.15.0 + 0.15.0 From 758b8d9d37890bcd2549f5a824bb3ef047530a1f Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 10 Dec 2024 17:24:34 +0800 Subject: [PATCH 06/23] fixup --- ...xecution.DataSourceScanTransformerRegister | 0 .../execution/HudiScanTransformer.scala | 0 .../HudiScanTransformerProvider.scala | 0 .../gluten/execution/VeloxHudiSuite.scala | 0 .../gluten/execution/VeloxTPCHHudiSuite.scala | 0 pom.xml | 65 ++++++++++++++++++- 6 files changed, 64 insertions(+), 1 deletion(-) rename gluten-hudi/{src => src-hudi}/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister (100%) rename gluten-hudi/{src => src-hudi}/main/scala/org/apache/gluten/execution/HudiScanTransformer.scala (100%) rename gluten-hudi/{src => src-hudi}/main/scala/org/apache/gluten/execution/HudiScanTransformerProvider.scala (100%) rename gluten-hudi/{src => src-hudi}/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala (100%) rename gluten-hudi/{src => src-hudi}/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala (100%) diff --git a/gluten-hudi/src/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister b/gluten-hudi/src-hudi/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister similarity index 100% rename from gluten-hudi/src/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister rename to gluten-hudi/src-hudi/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister diff --git a/gluten-hudi/src/main/scala/org/apache/gluten/execution/HudiScanTransformer.scala b/gluten-hudi/src-hudi/main/scala/org/apache/gluten/execution/HudiScanTransformer.scala similarity index 100% rename from gluten-hudi/src/main/scala/org/apache/gluten/execution/HudiScanTransformer.scala rename to gluten-hudi/src-hudi/main/scala/org/apache/gluten/execution/HudiScanTransformer.scala diff --git a/gluten-hudi/src/main/scala/org/apache/gluten/execution/HudiScanTransformerProvider.scala b/gluten-hudi/src-hudi/main/scala/org/apache/gluten/execution/HudiScanTransformerProvider.scala similarity index 100% rename from gluten-hudi/src/main/scala/org/apache/gluten/execution/HudiScanTransformerProvider.scala rename to gluten-hudi/src-hudi/main/scala/org/apache/gluten/execution/HudiScanTransformerProvider.scala diff --git a/gluten-hudi/src/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala b/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala similarity index 100% rename from gluten-hudi/src/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala rename to gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala diff --git a/gluten-hudi/src/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala b/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala similarity index 100% rename from gluten-hudi/src/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala rename to gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala diff --git a/pom.xml b/pom.xml index 57683e48ddad..a0f2a1932321 100644 --- a/pom.xml +++ b/pom.xml @@ -440,7 +440,6 @@ gluten-delta - @@ -593,6 +592,70 @@ gluten-hudi + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-hudi-sources + generate-sources + + add-source + + + + ${project.basedir}/src-hudi/main/scala + ${project.basedir}/src-hudi/main/java + + + + + add-hudi-resources + generate-resources + + add-resource + + + + + ${project.basedir}/src-hudi/main/resources + + + + + + add-hudi-test-sources + generate-test-sources + + add-test-source + + + + ${project.basedir}/src-hudi/test/scala + ${project.basedir}/src-hudi/test/java + + + + + add-hudi-test-resources + generate-test-resources + + add-test-resource + + + + + ${project.basedir}/src-hudi/test/resources + + + + + + + + backends-velox From d8f78a38e6e4d717ea2c3eb8e81ebeb82c46f0e5 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 11 Dec 2024 09:31:10 +0800 Subject: [PATCH 07/23] fixup --- backends-velox/pom.xml | 27 +++++++++++++++++++++------ gluten-substrait/pom.xml | 31 ++++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml index 7a0dbab90627..e7e73ac64350 100755 --- a/backends-velox/pom.xml +++ b/backends-velox/pom.xml @@ -237,12 +237,6 @@ - - - ${cpp.releases.dir} - ${platform}/${arch} - - target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes @@ -296,6 +290,27 @@ + + org.codehaus.mojo + build-helper-maven-plugin + + + add-cpp-libraries + generate-resources + + add-resource + + + + + ${cpp.releases.dir} + ${platform}/${arch} + + + + + + diff --git a/gluten-substrait/pom.xml b/gluten-substrait/pom.xml index e8eb6a57e6c4..2183f9b15982 100644 --- a/gluten-substrait/pom.xml +++ b/gluten-substrait/pom.xml @@ -181,13 +181,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - true - - ${project.build.directory}/extra-resources - - org.apache.maven.plugins @@ -217,6 +210,30 @@ + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-generated-resources + generate-resources + + add-resource + + + + + true + + ${project.build.directory}/extra-resources + + + + + + org.xolstice.maven.plugins From 683ca3446d90bfe52d497c7bc229a352137f8125 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 11 Dec 2024 11:40:10 +0800 Subject: [PATCH 08/23] fixup --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a0f2a1932321..8e4365b44cf2 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ - 3.2.0 + 3.6.0 4.8.0 3.12.1 3.2.2 From bc54bec704018a6db40cd1101ea836eb061d4470 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 11 Dec 2024 14:58:56 +0800 Subject: [PATCH 09/23] fixup --- backends-velox/pom.xml | 30 +++++++++--------------------- gluten-substrait/pom.xml | 26 +------------------------- pom.xml | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 46 deletions(-) diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml index e7e73ac64350..48a044a17f2f 100755 --- a/backends-velox/pom.xml +++ b/backends-velox/pom.xml @@ -237,6 +237,15 @@ + + + ${project.basedir}/src/main/resources + + + ${cpp.releases.dir} + ${platform}/${arch} + + target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes @@ -290,27 +299,6 @@ - - org.codehaus.mojo - build-helper-maven-plugin - - - add-cpp-libraries - generate-resources - - add-resource - - - - - ${cpp.releases.dir} - ${platform}/${arch} - - - - - - diff --git a/gluten-substrait/pom.xml b/gluten-substrait/pom.xml index 2183f9b15982..8a790aa15337 100644 --- a/gluten-substrait/pom.xml +++ b/gluten-substrait/pom.xml @@ -198,7 +198,7 @@ - + @@ -210,30 +210,6 @@ - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-generated-resources - generate-resources - - add-resource - - - - - true - - ${project.build.directory}/extra-resources - - - - - - org.xolstice.maven.plugins diff --git a/pom.xml b/pom.xml index 8e4365b44cf2..ca3289e6c19b 100644 --- a/pom.xml +++ b/pom.xml @@ -1075,6 +1075,16 @@ + + + ${project.basedir}/src/main/resources + + + + + ${project.basedir}/src/test/resources + + @@ -1320,6 +1330,30 @@ org.codehaus.mojo build-helper-maven-plugin ${build-helper-maven-plugin.version} + + + + + + + + + + add-generated-resources + generate-resources + + add-resource + + + + + + ${project.build.directory}/generated-resources + + + + + org.xolstice.maven.plugins From db3b0a53cf2bda8b0ed0494f37fff3be7623efda Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 11 Dec 2024 15:06:57 +0800 Subject: [PATCH 10/23] fixup --- gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index 4b28f23dd9e3..4b27cc75bd70 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -94,7 +94,7 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { val glutenBuildInfo = new mutable.LinkedHashMap[String, String]() val components = Component.sorted() - glutenBuildInfo.put("Components", components.map(_.buildInfo().name).mkString(",")) + glutenBuildInfo.put("Components", components.map(_.buildInfo().name).mkString(", ")) components.foreach { comp => val buildInfo = comp.buildInfo() From 8b89e4cc179c8a2f91500bdd4867816ce795c541 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 11 Dec 2024 15:28:00 +0800 Subject: [PATCH 11/23] fixup --- .github/workflows/util/setup_helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/util/setup_helper.sh b/.github/workflows/util/setup_helper.sh index 8b41d8326416..17d936a357f4 100644 --- a/.github/workflows/util/setup_helper.sh +++ b/.github/workflows/util/setup_helper.sh @@ -22,7 +22,7 @@ function install_maven { wget -nv https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven ) - echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + echo "PATH=/usr/lib/maven/bin:${PATH}" >> $GITHUB_ENV } for cmd in "$@" From b1f2f392134c03c09cdcc1b930e6134ae2e79245 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 11 Dec 2024 15:42:05 +0800 Subject: [PATCH 12/23] fixup --- pom.xml | 52 ++++++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/pom.xml b/pom.xml index ca3289e6c19b..9246838e767f 100644 --- a/pom.xml +++ b/pom.xml @@ -1330,30 +1330,6 @@ org.codehaus.mojo build-helper-maven-plugin ${build-helper-maven-plugin.version} - - - - - - - - - - add-generated-resources - generate-resources - - add-resource - - - - - - ${project.build.directory}/generated-resources - - - - - org.xolstice.maven.plugins @@ -1407,6 +1383,34 @@ versions-maven-plugin 2.15.0 + + org.codehaus.mojo + build-helper-maven-plugin + + + + + + + + + + add-generated-resources + generate-resources + + add-resource + + + + + + ${project.build.directory}/generated-resources + + + + + + From 4376cdcaa9fcddff99a0455d0d3d0299eafda2e4 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 11 Dec 2024 16:35:13 +0800 Subject: [PATCH 13/23] fixup --- .../scala/org/apache/gluten/execution/VeloxDeltaSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala b/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala index caf86189a2d4..bc28ce218166 100644 --- a/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala +++ b/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala @@ -42,7 +42,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite { .set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") } - // IdMapping is supported in Delta 2.2 (related to Spark3.3.1) + // IdMapping is supported in Delta 2.2 (related to Spark 3.3.1) testWithSpecifiedSparkVersion("column mapping mode = id", Some("3.3")) { withTable("delta_cm1") { spark.sql(s""" From d8d85456d8c4b83483373cb1efe3f5f96855b10e Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 11 Dec 2024 20:46:55 +0800 Subject: [PATCH 14/23] fixup --- pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pom.xml b/pom.xml index 9246838e767f..57a8b451c534 100644 --- a/pom.xml +++ b/pom.xml @@ -471,6 +471,8 @@ ${project.basedir}/src-delta/main/resources + + ${project.basedir}/src-delta-${delta.binary.version}/main/resources @@ -501,6 +503,8 @@ ${project.basedir}/src-delta/test/resources + + ${project.basedir}/src-delta-${delta.binary.version}/test/resources From f91d844b689ab741ee6af32d4cf6a07a61d5f4ad Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 11 Dec 2024 20:54:58 +0800 Subject: [PATCH 15/23] fixup --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index 57a8b451c534..a6a5efbc823a 100644 --- a/pom.xml +++ b/pom.xml @@ -469,6 +469,8 @@ + + ${project.basedir}/src-delta/main/resources From adf7b3182ecc695ab61f1cd8cfad446b91f3e5bc Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 11 Dec 2024 20:56:52 +0800 Subject: [PATCH 16/23] fixup --- .github/workflows/util/setup_helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/util/setup_helper.sh b/.github/workflows/util/setup_helper.sh index 17d936a357f4..8b41d8326416 100644 --- a/.github/workflows/util/setup_helper.sh +++ b/.github/workflows/util/setup_helper.sh @@ -22,7 +22,7 @@ function install_maven { wget -nv https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven ) - echo "PATH=/usr/lib/maven/bin:${PATH}" >> $GITHUB_ENV + echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV } for cmd in "$@" From 2fe7afb0978a8f744035689f5ecd516b38c0329c Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 11 Dec 2024 20:58:30 +0800 Subject: [PATCH 17/23] fixup --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a6a5efbc823a..5ea2c0eaa189 100644 --- a/pom.xml +++ b/pom.xml @@ -1396,7 +1396,7 @@ - + From 6b2d2038e298703b326742b65b89700e243eb549 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 11 Dec 2024 20:58:38 +0800 Subject: [PATCH 18/23] fixup --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5ea2c0eaa189..66a333409105 100644 --- a/pom.xml +++ b/pom.xml @@ -1396,7 +1396,7 @@ - + From ec1fe753e2124b8aac9c02d43e7c2bb413e7b448 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 11 Dec 2024 20:58:58 +0800 Subject: [PATCH 19/23] fixup --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 66a333409105..53dc7c1e6884 100644 --- a/pom.xml +++ b/pom.xml @@ -1395,7 +1395,7 @@ - + From dd846e5f680b7e0ba9291ee2f453e76136e3b9a2 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 12 Dec 2024 10:45:55 +0800 Subject: [PATCH 20/23] fixup --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index 53dc7c1e6884..dcc0a2fd8874 100644 --- a/pom.xml +++ b/pom.xml @@ -412,6 +412,7 @@ clickhouse + true @@ -436,6 +437,10 @@ delta false + + activateDelta + true + gluten-delta From 9926915d5e850bf41a1174d3ac52ed7f85d05f4b Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 12 Dec 2024 10:47:58 +0800 Subject: [PATCH 21/23] Revert "fixup" This reverts commit ae19979760a91e9190578ee63a5ea1a7ccedf82f. --- pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pom.xml b/pom.xml index dcc0a2fd8874..53dc7c1e6884 100644 --- a/pom.xml +++ b/pom.xml @@ -412,7 +412,6 @@ clickhouse - true @@ -437,10 +436,6 @@ delta false - - activateDelta - true - gluten-delta From 3cc55d55919c45e313d600d7f1bbd288828b0f40 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 12 Dec 2024 11:17:56 +0800 Subject: [PATCH 22/23] fixup --- backends-clickhouse/pom.xml | 21 +++++++++++++++++++++ package/pom.xml | 1 - pom.xml | 6 +++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml index 4b67d8c30d29..9a606c48e694 100644 --- a/backends-clickhouse/pom.xml +++ b/backends-clickhouse/pom.xml @@ -307,6 +307,27 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-delta-profile + + enforce + + + + + + delta + "-P delta" must be set when building Gluten with ClickHouse backend. + + + + + + org.apache.maven.plugins maven-resources-plugin diff --git a/package/pom.xml b/package/pom.xml index f4cc8d6f7015..e0620e5cf5e1 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -253,7 +253,6 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.3.0 org.codehaus.mojo diff --git a/pom.xml b/pom.xml index 53dc7c1e6884..3c59b4f19e11 100644 --- a/pom.xml +++ b/pom.xml @@ -1347,13 +1347,17 @@ antlr4-maven-plugin ${antlr4.version} + + org.apache.maven.plugins + maven-enforcer-plugin + 3.3.0 + org.apache.maven.plugins maven-enforcer-plugin - 1.0.1 enforce-versions From eb4c03f20c1bddf054e18a19a83563396b70c6f1 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 13 Dec 2024 10:14:36 +0800 Subject: [PATCH 23/23] fixup --- .../apache/gluten/execution/DeltaSuite.scala | 2 +- .../gluten/execution/VeloxDeltaSuite.scala | 253 ------------------ .../execution/VeloxTPCHDeltaSuite.scala | 57 ---- 3 files changed, 1 insertion(+), 311 deletions(-) rename gluten-delta/{src => src-delta}/test/scala/org/apache/gluten/execution/DeltaSuite.scala (99%) delete mode 100644 gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala delete mode 100644 gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala diff --git a/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala b/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/DeltaSuite.scala similarity index 99% rename from gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala rename to gluten-delta/src-delta/test/scala/org/apache/gluten/execution/DeltaSuite.scala index caf86189a2d4..df788bec99ad 100644 --- a/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala +++ b/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/DeltaSuite.scala @@ -18,7 +18,7 @@ package org.apache.gluten.execution import org.apache.spark.SparkConf import org.apache.spark.sql.Row -import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType} +import org.apache.spark.sql.types._ import scala.collection.JavaConverters._ diff --git a/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala b/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala deleted file mode 100644 index bc28ce218166..000000000000 --- a/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.execution - -import org.apache.spark.SparkConf -import org.apache.spark.sql.Row -import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType} - -import scala.collection.JavaConverters._ - -abstract class DeltaSuite extends WholeStageTransformerSuite { - protected val rootPath: String = getClass.getResource("/").getPath - // FIXME: This folder doesn't exist in module gluten-delta so should be provided by - // backend modules that rely on this suite. - override protected val resourcePath: String = "/tpch-data-parquet" - override protected val fileFormat: String = "parquet" - - override protected def sparkConf: SparkConf = { - super.sparkConf - .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") - .set("spark.sql.files.maxPartitionBytes", "1g") - .set("spark.sql.shuffle.partitions", "1") - .set("spark.memory.offHeap.size", "2g") - .set("spark.unsafe.exceptionOnMemoryLeak", "true") - .set("spark.sql.autoBroadcastJoinThreshold", "-1") - .set("spark.sql.sources.useV1SourceList", "avro") - .set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") - .set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") - } - - // IdMapping is supported in Delta 2.2 (related to Spark 3.3.1) - testWithSpecifiedSparkVersion("column mapping mode = id", Some("3.3")) { - withTable("delta_cm1") { - spark.sql(s""" - |create table delta_cm1 (id int, name string) using delta - |tblproperties ("delta.columnMapping.mode"= "id") - |""".stripMargin) - spark.sql(s""" - |insert into delta_cm1 values (1, "v1"), (2, "v2") - |""".stripMargin) - val df1 = runQueryAndCompare("select * from delta_cm1") { _ => } - checkLengthAndPlan(df1, 2) - checkAnswer(df1, Row(1, "v1") :: Row(2, "v2") :: Nil) - - val df2 = runQueryAndCompare("select name from delta_cm1 where id = 2") { _ => } - checkLengthAndPlan(df2, 1) - checkAnswer(df2, Row("v2") :: Nil) - } - } - - // NameMapping is supported in Delta 2.0 (related to Spark3.2.0) - testWithSpecifiedSparkVersion("column mapping mode = name", Some("3.2")) { - withTable("delta_cm2") { - spark.sql(s""" - |create table delta_cm2 (id int, name string) using delta - |tblproperties ("delta.columnMapping.mode"= "name") - |""".stripMargin) - spark.sql(s""" - |insert into delta_cm2 values (1, "v1"), (2, "v2") - |""".stripMargin) - val df1 = runQueryAndCompare("select * from delta_cm2") { _ => } - checkLengthAndPlan(df1, 2) - checkAnswer(df1, Row(1, "v1") :: Row(2, "v2") :: Nil) - - val df2 = runQueryAndCompare("select name from delta_cm2 where id = 2") { _ => } - checkLengthAndPlan(df2, 1) - checkAnswer(df2, Row("v2") :: Nil) - } - } - - testWithSpecifiedSparkVersion("delta: time travel", Some("3.3")) { - withTable("delta_tm") { - spark.sql(s""" - |create table delta_tm (id int, name string) using delta - |""".stripMargin) - spark.sql(s""" - |insert into delta_tm values (1, "v1"), (2, "v2") - |""".stripMargin) - spark.sql(s""" - |insert into delta_tm values (3, "v3"), (4, "v4") - |""".stripMargin) - val df1 = runQueryAndCompare("select * from delta_tm VERSION AS OF 1") { _ => } - checkLengthAndPlan(df1, 2) - checkAnswer(df1, Row(1, "v1") :: Row(2, "v2") :: Nil) - val df2 = runQueryAndCompare("select * from delta_tm VERSION AS OF 2") { _ => } - checkLengthAndPlan(df2, 4) - checkAnswer(df2, Row(1, "v1") :: Row(2, "v2") :: Row(3, "v3") :: Row(4, "v4") :: Nil) - val df3 = runQueryAndCompare("select name from delta_tm VERSION AS OF 2 where id = 2") { - _ => - } - checkLengthAndPlan(df3, 1) - checkAnswer(df3, Row("v2") :: Nil) - } - } - - testWithSpecifiedSparkVersion("delta: partition filters", Some("3.2")) { - withTable("delta_pf") { - spark.sql(s""" - |create table delta_pf (id int, name string) using delta partitioned by (name) - |""".stripMargin) - spark.sql(s""" - |insert into delta_pf values (1, "v1"), (2, "v2"), (3, "v1"), (4, "v2") - |""".stripMargin) - val df1 = runQueryAndCompare("select * from delta_pf where name = 'v1'") { _ => } - val deltaScanTransformer = df1.queryExecution.executedPlan.collect { - case f: DeltaScanTransformer => f - }.head - // No data filters as only partition filters exist - assert(deltaScanTransformer.filterExprs().size == 0) - checkLengthAndPlan(df1, 2) - checkAnswer(df1, Row(1, "v1") :: Row(3, "v1") :: Nil) - } - } - - testWithSpecifiedSparkVersion("basic test with stats.skipping disabled", Some("3.2")) { - withTable("delta_test2") { - withSQLConf("spark.databricks.delta.stats.skipping" -> "false") { - spark.sql(s""" - |create table delta_test2 (id int, name string) using delta - |""".stripMargin) - spark.sql(s""" - |insert into delta_test2 values (1, "v1"), (2, "v2") - |""".stripMargin) - val df1 = runQueryAndCompare("select * from delta_test2") { _ => } - checkLengthAndPlan(df1, 2) - checkAnswer(df1, Row(1, "v1") :: Row(2, "v2") :: Nil) - - val df2 = runQueryAndCompare("select name from delta_test2 where id = 2") { _ => } - checkLengthAndPlan(df2, 1) - checkAnswer(df2, Row("v2") :: Nil) - } - } - } - - testWithSpecifiedSparkVersion("column mapping with complex type", Some("3.2")) { - withTable("t1") { - val simpleNestedSchema = new StructType() - .add("a", StringType, true) - .add("b", new StructType().add("c", StringType, true).add("d", IntegerType, true)) - .add("map", MapType(StringType, StringType), true) - .add("arr", ArrayType(IntegerType), true) - - val simpleNestedData = spark.createDataFrame( - Seq( - Row("str1", Row("str1.1", 1), Map("k1" -> "v1"), Array(1, 11)), - Row("str2", Row("str1.2", 2), Map("k2" -> "v2"), Array(2, 22))).asJava, - simpleNestedSchema) - - spark.sql( - """CREATE TABLE t1 - | (a STRING,b STRUCT,map MAP,arr ARRAY) - | USING DELTA - | PARTITIONED BY (`a`) - | TBLPROPERTIES ('delta.columnMapping.mode' = 'name')""".stripMargin) - - simpleNestedData.write.format("delta").mode("append").saveAsTable("t1") - - val df1 = runQueryAndCompare("select * from t1") { _ => } - checkAnswer( - df1, - Seq( - Row("str1", Row("str1.1", 1), Map("k1" -> "v1"), Array(1, 11)), - Row("str2", Row("str1.2", 2), Map("k2" -> "v2"), Array(2, 22)))) - spark.sql(s"Alter table t1 RENAME COLUMN b to b1") - spark.sql( - "insert into t1 " + - "values ('str3', struct('str1.3', 3), map('k3', 'v3'), array(3, 33))") - - val df2 = runQueryAndCompare("select b1 from t1") { _ => } - checkAnswer(df2, Seq(Row(Row("str1.1", 1)), Row(Row("str1.2", 2)), Row(Row("str1.3", 3)))) - - spark.sql(s"Alter table t1 RENAME COLUMN b1.c to c1") - val df3 = runQueryAndCompare("select * from t1") { _ => } - checkAnswer( - df3, - Seq( - Row("str1", Row("str1.1", 1), Map("k1" -> "v1"), Array(1, 11)), - Row("str2", Row("str1.2", 2), Map("k2" -> "v2"), Array(2, 22)), - Row("str3", Row("str1.3", 3), Map("k3" -> "v3"), Array(3, 33)) - ) - ) - } - } - - testWithSpecifiedSparkVersion("deletion vector", Some("3.4")) { - withTempPath { - p => - import testImplicits._ - val path = p.getCanonicalPath - val df1 = Seq(1, 2, 3, 4, 5).toDF("id") - val values2 = Seq(6, 7, 8, 9, 10) - val df2 = values2.toDF("id") - df1.union(df2).coalesce(1).write.format("delta").save(path) - spark.sql( - s"ALTER TABLE delta.`$path` SET TBLPROPERTIES ('delta.enableDeletionVectors' = true)") - checkAnswer(spark.read.format("delta").load(path), df1.union(df2)) - spark.sql(s"DELETE FROM delta.`$path` WHERE id IN (${values2.mkString(", ")})") - checkAnswer(spark.read.format("delta").load(path), df1) - } - } - - testWithSpecifiedSparkVersion("delta: push down input_file_name expression", Some("3.2")) { - withTable("source_table") { - withTable("target_table") { - spark.sql(s""" - |CREATE TABLE source_table(id INT, name STRING, age INT) USING delta; - |""".stripMargin) - - spark.sql(s""" - |CREATE TABLE target_table(id INT, name STRING, age INT) USING delta; - | - |""".stripMargin) - - spark.sql(s""" - |INSERT INTO source_table VALUES(1, 'a', 10),(2, 'b', 20); - |""".stripMargin) - - spark.sql(s""" - |INSERT INTO target_table VALUES(1, 'c', 10),(3, 'c', 30); - |""".stripMargin) - - spark.sql(s""" - |MERGE INTO target_table AS target - |USING source_table AS source - |ON target.id = source.id - |WHEN MATCHED THEN - |UPDATE SET - | target.name = source.name, - | target.age = source.age - |WHEN NOT MATCHED THEN - |INSERT (id, name, age) VALUES (source.id, source.name, source.age); - |""".stripMargin) - - val df1 = runQueryAndCompare("SELECT * FROM target_table") { _ => } - checkAnswer(df1, Row(1, "a", 10) :: Row(2, "b", 20) :: Row(3, "c", 30) :: Nil) - } - } - } -} diff --git a/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala b/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala deleted file mode 100644 index 238f5ca56f0a..000000000000 --- a/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.execution - -import org.apache.spark.SparkConf - -import java.io.File - -class VeloxTPCHDeltaSuite extends VeloxTPCHSuite { - protected val tpchBasePath: String = - getClass.getResource("/").getPath + "../../../src/test/resources" - - override protected val resourcePath: String = - new File(tpchBasePath, "tpch-data-parquet").getCanonicalPath - - override protected val queriesResults: String = - new File(tpchBasePath, "queries-output").getCanonicalPath - - override protected def sparkConf: SparkConf = { - super.sparkConf - .set("spark.executor.memory", "4g") - .set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") - .set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") - } - - override protected def createTPCHNotNullTables(): Unit = { - TPCHTables - .map(_.name) - .map { - table => - val tablePath = new File(resourcePath, table).getAbsolutePath - val tableDF = spark.read.format(fileFormat).load(tablePath) - tableDF.write.format("delta").mode("append").saveAsTable(table) - (table, tableDF) - } - .toMap - } - - override protected def afterAll(): Unit = { - TPCHTables.map(_.name).foreach(table => spark.sql(s"DROP TABLE IF EXISTS $table")) - super.afterAll() - } -}