[VL] Add tests for Velox SMJ's coverage (apache#7195)

sharkdtu · Oct 14, 2024 · be23971 · be23971
1 parent bfb3467
commit be23971
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 8 deletions.
diff --git a/...ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/...ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.execution.datasources.parquet._
 import org.apache.spark.sql.execution.datasources.text.{GlutenTextV1Suite, GlutenTextV2Suite}
 import org.apache.spark.sql.execution.datasources.v2.{GlutenDataSourceV2StrategySuite, GlutenFileTableSuite, GlutenV2PredicateSuite}
 import org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
-import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite, GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite}
+import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite, GlutenExistenceJoinSuite, GlutenInnerJoinSuiteForceShjOn, GlutenOuterJoinSuiteForceShjOn}
 import org.apache.spark.sql.extension.{GlutenCustomerExtensionSuite, GlutenSessionExtensionSuite}
 import org.apache.spark.sql.gluten.GlutenFallbackSuite
 import org.apache.spark.sql.hive.execution.GlutenHiveSQLQueryCHSuite
@@ -1624,7 +1624,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
     .exclude("test composed unique condition (both non-equal) for left anti join using ShuffledHashJoin (whole-stage-codegen on)")
     .exclude("test composed unique condition (both non-equal) for left anti join using SortMergeJoin (whole-stage-codegen off)")
     .exclude("test composed unique condition (both non-equal) for left anti join using SortMergeJoin (whole-stage-codegen on)")
-  enableSuite[GlutenInnerJoinSuite]
+  enableSuite[GlutenInnerJoinSuiteForceShjOn]
     .exclude(
       "inner join, one match per row using ShuffledHashJoin (build=left) (whole-stage-codegen off)")
     .exclude(
@@ -1673,7 +1673,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
     .exclude("SPARK-15822 - test structs as keys using BroadcastNestedLoopJoin build left (whole-stage-codegen on)")
     .exclude("SPARK-15822 - test structs as keys using BroadcastNestedLoopJoin build right (whole-stage-codegen off)")
     .exclude("SPARK-15822 - test structs as keys using BroadcastNestedLoopJoin build right (whole-stage-codegen on)")
-  enableSuite[GlutenOuterJoinSuite]
+  enableSuite[GlutenOuterJoinSuiteForceShjOn]
     .exclude("basic left outer join using ShuffledHashJoin (whole-stage-codegen off)")
     .exclude("basic left outer join using ShuffledHashJoin (whole-stage-codegen on)")
     .exclude("basic left outer join using SortMergeJoin (whole-stage-codegen off)")

diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -35,7 +35,7 @@ import org.apache.spark.sql.execution.datasources.parquet._
 import org.apache.spark.sql.execution.datasources.text.{GlutenTextV1Suite, GlutenTextV2Suite}
 import org.apache.spark.sql.execution.datasources.v2.{GlutenDataSourceV2StrategySuite, GlutenFileTableSuite, GlutenV2PredicateSuite}
 import org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
-import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite, GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite}
+import org.apache.spark.sql.execution.joins._
 import org.apache.spark.sql.extension.{GlutenCollapseProjectExecTransformerSuite, GlutenSessionExtensionSuite, TestFileSourceScanExecTransformer}
 import org.apache.spark.sql.gluten.GlutenFallbackSuite
 import org.apache.spark.sql.hive.execution.GlutenHiveSQLQuerySuite
@@ -879,8 +879,15 @@ class VeloxTestSettings extends BackendTestSettings {
     .exclude("broadcast join where streamed side's output partitioning is HashPartitioning")
 
   enableSuite[GlutenExistenceJoinSuite]
-  enableSuite[GlutenInnerJoinSuite]
-  enableSuite[GlutenOuterJoinSuite]
+  enableSuite[GlutenInnerJoinSuiteForceShjOn]
+  enableSuite[GlutenInnerJoinSuiteForceShjOff]
+  enableSuite[GlutenOuterJoinSuiteForceShjOn]
+  enableSuite[GlutenOuterJoinSuiteForceShjOff]
+    // Caused by Velox SMJ result mismatches with Spark.
+    .exclude("basic right outer join using SortMergeJoin (whole-stage-codegen off)")
+    .exclude("basic right outer join using SortMergeJoin (whole-stage-codegen on)")
+    .exclude("right outer join with unique keys using SortMergeJoin (whole-stage-codegen off)")
+    .exclude("right outer join with unique keys using SortMergeJoin (whole-stage-codegen on)")
   enableSuite[FallbackStrategiesSuite]
   enableSuite[GlutenBroadcastExchangeSuite]
   enableSuite[GlutenLocalBroadcastExchangeSuite]

diff --git a/...ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala b/...ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala
@@ -16,6 +16,21 @@
  */
 package org.apache.spark.sql.execution.joins
 
+import org.apache.gluten.GlutenConfig
+
+import org.apache.spark.SparkConf
 import org.apache.spark.sql.GlutenSQLTestsBaseTrait
 
-class GlutenInnerJoinSuite extends InnerJoinSuite with GlutenSQLTestsBaseTrait {}
+class GlutenInnerJoinSuiteForceShjOn extends InnerJoinSuite with GlutenSQLTestsBaseTrait {
+  override def sparkConf: SparkConf = {
+    super.sparkConf
+      .set(GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key, "true")
+  }
+}
+
+class GlutenInnerJoinSuiteForceShjOff extends InnerJoinSuite with GlutenSQLTestsBaseTrait {
+  override def sparkConf: SparkConf = {
+    super.sparkConf
+      .set(GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key, "false")
+  }
+}
diff --git a/...ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala b/...ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala
@@ -16,6 +16,21 @@
  */
 package org.apache.spark.sql.execution.joins
 
+import org.apache.gluten.GlutenConfig
+
+import org.apache.spark.SparkConf
 import org.apache.spark.sql.GlutenSQLTestsBaseTrait
 
-class GlutenOuterJoinSuite extends OuterJoinSuite with GlutenSQLTestsBaseTrait {}
+class GlutenOuterJoinSuiteForceShjOn extends OuterJoinSuite with GlutenSQLTestsBaseTrait {
+  override def sparkConf: SparkConf = {
+    super.sparkConf
+      .set(GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key, "true")
+  }
+}
+
+class GlutenOuterJoinSuiteForceShjOff extends OuterJoinSuite with GlutenSQLTestsBaseTrait {
+  override def sparkConf: SparkConf = {
+    super.sparkConf
+      .set(GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key, "false")
+  }
+}