From a07e3b8f4f649ce77031eba76fe99c5348f673f0 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 27 May 2024 09:53:29 +0800 Subject: [PATCH] [VL] RAS: Add config option for setting user cost model, remove fallback strategies from RAS rules list (#5861) --- .github/workflows/velox_docker.yml | 4 +- .../v1-bhj-ras/spark32/1.txt | 105 +-- .../v1-bhj-ras/spark32/10.txt | 260 +++---- .../v1-bhj-ras/spark32/11.txt | 223 +++--- .../v1-bhj-ras/spark32/12.txt | 160 +++-- .../v1-bhj-ras/spark32/13.txt | 193 ++--- .../v1-bhj-ras/spark32/14.txt | 130 ++-- .../v1-bhj-ras/spark32/15.txt | 168 +++-- .../v1-bhj-ras/spark32/16.txt | 253 +++---- .../v1-bhj-ras/spark32/18.txt | 333 +++++---- .../v1-bhj-ras/spark32/19.txt | 126 ++-- .../v1-bhj-ras/spark32/20.txt | 434 +++++------ .../v1-bhj-ras/spark32/21.txt | 357 ++++----- .../v1-bhj-ras/spark32/22.txt | 143 ++-- .../v1-bhj-ras/spark32/3.txt | 205 +++--- .../v1-bhj-ras/spark32/4.txt | 168 +++-- .../v1-bhj-ras/spark32/5.txt | 388 +++++----- .../v1-bhj-ras/spark32/6.txt | 71 +- .../v1-bhj-ras/spark32/7.txt | 361 +++++----- .../v1-bhj-ras/spark32/8.txt | 502 +++++++------ .../v1-bhj-ras/spark32/9.txt | 380 +++++----- .../v1-bhj-ras/spark33/1.txt | 105 +-- .../v1-bhj-ras/spark33/10.txt | 260 +++---- .../v1-bhj-ras/spark33/11.txt | 390 +++++----- .../v1-bhj-ras/spark33/12.txt | 160 +++-- .../v1-bhj-ras/spark33/13.txt | 193 ++--- .../v1-bhj-ras/spark33/14.txt | 130 ++-- .../v1-bhj-ras/spark33/15.txt | 257 +++---- .../v1-bhj-ras/spark33/16.txt | 253 +++---- .../v1-bhj-ras/spark33/18.txt | 333 +++++---- .../v1-bhj-ras/spark33/19.txt | 126 ++-- .../v1-bhj-ras/spark33/20.txt | 424 +++++------ .../v1-bhj-ras/spark33/21.txt | 353 ++++----- .../v1-bhj-ras/spark33/22.txt | 216 +++--- .../v1-bhj-ras/spark33/3.txt | 205 +++--- .../v1-bhj-ras/spark33/4.txt | 168 +++-- .../v1-bhj-ras/spark33/5.txt | 388 +++++----- .../v1-bhj-ras/spark33/6.txt | 71 +- .../v1-bhj-ras/spark33/7.txt | 361 +++++----- .../v1-bhj-ras/spark33/8.txt | 502 +++++++------ .../v1-bhj-ras/spark33/9.txt | 380 +++++----- .../v1-bhj-ras/spark34/1.txt | 105 +-- .../v1-bhj-ras/spark34/10.txt | 260 +++---- .../v1-bhj-ras/spark34/11.txt | 390 +++++----- .../v1-bhj-ras/spark34/12.txt | 160 +++-- .../v1-bhj-ras/spark34/13.txt | 193 ++--- .../v1-bhj-ras/spark34/14.txt | 130 ++-- .../v1-bhj-ras/spark34/15.txt | 257 +++---- .../v1-bhj-ras/spark34/16.txt | 254 +++---- .../v1-bhj-ras/spark34/18.txt | 333 +++++---- .../v1-bhj-ras/spark34/19.txt | 126 ++-- .../v1-bhj-ras/spark34/20.txt | 375 +++++----- .../v1-bhj-ras/spark34/21.txt | 353 ++++----- .../v1-bhj-ras/spark34/22.txt | 216 +++--- .../v1-bhj-ras/spark34/3.txt | 205 +++--- .../v1-bhj-ras/spark34/4.txt | 168 +++-- .../v1-bhj-ras/spark34/5.txt | 388 +++++----- .../v1-bhj-ras/spark34/6.txt | 71 +- .../v1-bhj-ras/spark34/7.txt | 361 +++++----- .../v1-bhj-ras/spark34/8.txt | 502 +++++++------ .../v1-bhj-ras/spark34/9.txt | 380 +++++----- .../tpch-approved-plan/v1-ras/spark32/1.txt | 105 +-- .../tpch-approved-plan/v1-ras/spark32/10.txt | 334 +++++---- .../tpch-approved-plan/v1-ras/spark32/11.txt | 275 +++---- .../tpch-approved-plan/v1-ras/spark32/12.txt | 186 ++--- .../tpch-approved-plan/v1-ras/spark32/13.txt | 171 ++--- .../tpch-approved-plan/v1-ras/spark32/14.txt | 134 ++-- .../tpch-approved-plan/v1-ras/spark32/15.txt | 172 ++--- .../tpch-approved-plan/v1-ras/spark32/16.txt | 244 ++++--- .../tpch-approved-plan/v1-ras/spark32/17.txt | 221 +++--- .../tpch-approved-plan/v1-ras/spark32/18.txt | 375 +++++----- .../tpch-approved-plan/v1-ras/spark32/19.txt | 130 ++-- .../tpch-approved-plan/v1-ras/spark32/20.txt | 485 +++++++------ .../tpch-approved-plan/v1-ras/spark32/21.txt | 457 ++++++------ .../tpch-approved-plan/v1-ras/spark32/22.txt | 173 ++--- .../tpch-approved-plan/v1-ras/spark32/3.txt | 227 +++--- .../tpch-approved-plan/v1-ras/spark32/4.txt | 190 ++--- .../tpch-approved-plan/v1-ras/spark32/5.txt | 514 ++++++------- .../tpch-approved-plan/v1-ras/spark32/6.txt | 71 +- .../tpch-approved-plan/v1-ras/spark32/7.txt | 491 +++++++------ .../tpch-approved-plan/v1-ras/spark32/8.txt | 680 +++++++++--------- .../tpch-approved-plan/v1-ras/spark32/9.txt | 510 ++++++------- .../tpch-approved-plan/v1-ras/spark33/1.txt | 105 +-- .../tpch-approved-plan/v1-ras/spark33/10.txt | 334 +++++---- .../tpch-approved-plan/v1-ras/spark33/11.txt | 468 ++++++------ .../tpch-approved-plan/v1-ras/spark33/12.txt | 186 ++--- .../tpch-approved-plan/v1-ras/spark33/13.txt | 171 ++--- .../tpch-approved-plan/v1-ras/spark33/14.txt | 134 ++-- .../tpch-approved-plan/v1-ras/spark33/15.txt | 261 +++---- .../tpch-approved-plan/v1-ras/spark33/16.txt | 244 ++++--- .../tpch-approved-plan/v1-ras/spark33/17.txt | 221 +++--- .../tpch-approved-plan/v1-ras/spark33/18.txt | 375 +++++----- .../tpch-approved-plan/v1-ras/spark33/19.txt | 130 ++-- .../tpch-approved-plan/v1-ras/spark33/20.txt | 475 ++++++------ .../tpch-approved-plan/v1-ras/spark33/21.txt | 453 ++++++------ .../tpch-approved-plan/v1-ras/spark33/22.txt | 246 ++++--- .../tpch-approved-plan/v1-ras/spark33/3.txt | 227 +++--- .../tpch-approved-plan/v1-ras/spark33/4.txt | 190 ++--- .../tpch-approved-plan/v1-ras/spark33/5.txt | 514 ++++++------- .../tpch-approved-plan/v1-ras/spark33/6.txt | 71 +- .../tpch-approved-plan/v1-ras/spark33/7.txt | 491 +++++++------ .../tpch-approved-plan/v1-ras/spark33/8.txt | 680 +++++++++--------- .../tpch-approved-plan/v1-ras/spark33/9.txt | 510 ++++++------- .../tpch-approved-plan/v1-ras/spark34/1.txt | 105 +-- .../tpch-approved-plan/v1-ras/spark34/10.txt | 334 +++++---- .../tpch-approved-plan/v1-ras/spark34/11.txt | 468 ++++++------ .../tpch-approved-plan/v1-ras/spark34/12.txt | 186 ++--- .../tpch-approved-plan/v1-ras/spark34/13.txt | 171 ++--- .../tpch-approved-plan/v1-ras/spark34/14.txt | 134 ++-- .../tpch-approved-plan/v1-ras/spark34/15.txt | 261 +++---- .../tpch-approved-plan/v1-ras/spark34/16.txt | 244 ++++--- .../tpch-approved-plan/v1-ras/spark34/17.txt | 221 +++--- .../tpch-approved-plan/v1-ras/spark34/18.txt | 375 +++++----- .../tpch-approved-plan/v1-ras/spark34/19.txt | 130 ++-- .../tpch-approved-plan/v1-ras/spark34/20.txt | 475 ++++++------ .../tpch-approved-plan/v1-ras/spark34/21.txt | 453 ++++++------ .../tpch-approved-plan/v1-ras/spark34/22.txt | 246 ++++--- .../tpch-approved-plan/v1-ras/spark34/3.txt | 227 +++--- .../tpch-approved-plan/v1-ras/spark34/4.txt | 190 ++--- .../tpch-approved-plan/v1-ras/spark34/5.txt | 514 ++++++------- .../tpch-approved-plan/v1-ras/spark34/6.txt | 71 +- .../tpch-approved-plan/v1-ras/spark34/7.txt | 491 +++++++------ .../tpch-approved-plan/v1-ras/spark34/8.txt | 680 +++++++++--------- .../tpch-approved-plan/v1-ras/spark34/9.txt | 510 ++++++------- .../gluten/execution/VeloxTPCHSuite.scala | 4 +- .../apache/gluten/planner/VeloxRasSuite.scala | 67 +- docs/Configuration.md | 2 +- .../enumerated/EnumeratedApplier.scala | 49 +- .../enumerated/EnumeratedTransform.scala | 9 +- .../columnar/enumerated/RemoveFilter.scala | 37 +- .../columnar/heuristic/HeuristicApplier.scala | 4 +- .../columnar/util/AdaptiveContext.scala | 9 +- .../gluten/planner/GlutenOptimization.scala | 45 +- .../gluten/planner/cost/GlutenCostModel.scala | 32 +- .../spark/sql/utils/ReflectionUtil.scala | 28 + .../org/apache/gluten/GlutenConfig.scala | 36 +- 136 files changed, 18469 insertions(+), 16436 deletions(-) create mode 100644 gluten-core/src/main/scala/org/apache/spark/sql/utils/ReflectionUtil.scala diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 9b00535b474d..5c39fec35a78 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -189,10 +189,10 @@ jobs: cd $GITHUB_WORKSPACE/tools/gluten-it GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - --extra-conf=spark.gluten.sql.ras.enabled=true \ + --extra-conf=spark.gluten.ras.enabled=true \ && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - --extra-conf=spark.gluten.sql.ras.enabled=true + --extra-conf=spark.gluten.ras.enabled=true run-tpc-test-ubuntu-oom: needs: build-native-lib-centos-7 diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt index 699c17ea4562..8d67aad16c3d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (27) +AdaptiveSparkPlan (28) +- == Final Plan == - VeloxColumnarToRowExec (18) - +- ^ SortExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (19) + +- ^ SortExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (26) - +- Exchange (25) - +- HashAggregate (24) - +- Exchange (23) - +- HashAggregate (22) - +- Project (21) - +- Filter (20) - +- Scan parquet (19) + Sort (27) + +- Exchange (26) + +- HashAggregate (25) + +- Exchange (24) + +- HashAggregate (23) + +- Project (22) + +- Filter (21) + +- Scan parquet (20) (1) Scan parquet @@ -34,116 +35,120 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(4) ProjectExecTransformer +(5) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(6) ColumnarExchange +(7) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(12) ColumnarExchange +(13) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(14) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) InputAdapter +(15) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(15) InputIteratorTransformer +(16) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) SortExecTransformer +(17) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(17) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(18) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(19) Scan parquet +(20) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(20) Filter +(21) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(21) Project +(22) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(22) HashAggregate +(23) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(23) Exchange +(24) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(24) HashAggregate +(25) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(25) Exchange +(26) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Sort +(27) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(27) AdaptiveSparkPlan +(28) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt index 5b7e141a4beb..c0153da0cda9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt @@ -1,64 +1,68 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ ShuffleQueryStage (32) - +- ColumnarExchange (31) - +- ^ ProjectExecTransformer (29) - +- ^ FlushableHashAggregateExecTransformer (28) - +- ^ ProjectExecTransformer (27) - +- ^ BroadcastHashJoinExecTransformer Inner (26) - :- ^ ProjectExecTransformer (19) - : +- ^ BroadcastHashJoinExecTransformer Inner (18) - : :- ^ ProjectExecTransformer (10) - : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : :- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ Scan parquet (2) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ BroadcastQueryStage (15) - : +- ColumnarBroadcastExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ Scan parquet (11) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ BroadcastQueryStage (23) - +- ColumnarBroadcastExchange (22) - +- ^ Scan parquet (20) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- ^ InputAdapter (9) + : : +- ^ BroadcastQueryStage (8) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- ^ InputAdapter (19) + : +- ^ BroadcastQueryStage (18) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ BroadcastQueryStage (27) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- BroadcastHashJoin Inner BuildRight (57) - :- Project (53) - : +- BroadcastHashJoin Inner BuildRight (52) - : :- Project (47) - : : +- BroadcastHashJoin Inner BuildRight (46) - : : :- Filter (41) - : : : +- Scan parquet (40) - : : +- BroadcastExchange (45) - : : +- Project (44) - : : +- Filter (43) - : : +- Scan parquet (42) - : +- BroadcastExchange (51) - : +- Project (50) - : +- Filter (49) - : +- Scan parquet (48) - +- BroadcastExchange (56) - +- Filter (55) - +- Scan parquet (54) + TakeOrderedAndProject (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- BroadcastHashJoin Inner BuildRight (61) + :- Project (57) + : +- BroadcastHashJoin Inner BuildRight (56) + : :- Project (51) + : : +- BroadcastHashJoin Inner BuildRight (50) + : : :- Filter (45) + : : : +- Scan parquet (44) + : : +- BroadcastExchange (49) + : : +- Project (48) + : : +- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (55) + : +- Project (54) + : +- Filter (53) + : +- Scan parquet (52) + +- BroadcastExchange (60) + +- Filter (59) + +- Scan parquet (58) (1) Scan parquet @@ -68,280 +72,296 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(3) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(3) ProjectExecTransformer +(4) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(5) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(4) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(12) ProjectExecTransformer +(14) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(15) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(13) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(15) BroadcastQueryStage +(18) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(19) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(18) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) Scan parquet +(23) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(21) WholeStageCodegenTransformer (X) +(24) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(25) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(22) ColumnarBroadcastExchange +(26) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(27) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(24) InputAdapter +(28) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(25) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(26) BroadcastHashJoinExecTransformer +(30) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(27) ProjectExecTransformer +(31) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(28) FlushableHashAggregateExecTransformer +(32) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(29) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(30) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(31) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(33) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(34) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(40) Scan parquet +(44) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(42) Scan parquet +(46) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(43) Filter +(47) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(44) Project +(48) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(45) BroadcastExchange +(49) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(46) BroadcastHashJoin +(50) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(47) Project +(51) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(48) Scan parquet +(52) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(49) Filter +(53) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(50) Project +(54) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(51) BroadcastExchange +(55) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(52) BroadcastHashJoin +(56) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(53) Project +(57) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(54) Scan parquet +(58) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(55) Filter +(59) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(56) BroadcastExchange +(60) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) BroadcastHashJoin +(61) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(58) Project +(62) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(59) HashAggregate +(63) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(60) Exchange +(64) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(65) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(62) TakeOrderedAndProject +(66) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(63) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt index 32cb5ccf87b3..219d4b7c14de 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ FilterExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ BroadcastHashJoinExecTransformer Inner (17) - :- ^ ProjectExecTransformer (9) - : +- ^ BroadcastHashJoinExecTransformer Inner (8) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ Scan parquet (2) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ BroadcastQueryStage (14) - +- ColumnarBroadcastExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ^ InputAdapter (34) + +- ^ ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ^ InputAdapter (27) + +- ^ ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- ^ InputAdapter (8) + : +- ^ BroadcastQueryStage (7) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- ^ InputAdapter (18) + +- ^ BroadcastQueryStage (17) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (54) - +- Exchange (53) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildRight (41) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (57) + +- Exchange (56) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Project (51) + +- BroadcastHashJoin Inner BuildRight (50) + :- Project (45) + : +- BroadcastHashJoin Inner BuildRight (44) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (49) + +- Project (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -60,240 +63,252 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(3) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(3) WholeStageCodegenTransformer (X) +(4) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(5) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(7) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(9) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(10) Scan parquet +(12) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(11) ProjectExecTransformer +(13) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(14) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(12) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [1]: [n_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [1]: [n_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(19) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(20) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(21) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(22) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(27) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(29) ColumnarExchange +(32) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(31) InputAdapter +(34) InputAdapter Input [2]: [ps_partkey#X, value#X] -(32) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(33) SortExecTransformer +(36) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(35) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(36) Scan parquet +(39) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(38) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(40) BroadcastExchange +(43) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(41) BroadcastHashJoin +(44) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(42) Project +(45) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(45) Project +(48) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(46) BroadcastExchange +(49) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(50) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) Project +(51) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(49) HashAggregate +(52) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(50) Exchange +(53) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(52) Filter +(55) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(53) Exchange +(56) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Sort +(57) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(55) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt index a9cea181da0f..985cbf4c3c59 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (40) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ InputIteratorTransformer (6) - : +- ^ InputAdapter (5) - : +- ^ BroadcastQueryStage (4) - : +- ColumnarBroadcastExchange (3) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (8) - +- ^ Scan parquet (7) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ^ InputAdapter (18) + +- ^ ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (39) - +- Exchange (38) - +- HashAggregate (37) - +- Exchange (36) - +- HashAggregate (35) - +- Project (34) - +- BroadcastHashJoin Inner BuildLeft (33) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- BroadcastHashJoin Inner BuildLeft (35) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -46,174 +48,182 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderpriority#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(10) ProjectExecTransformer Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(12) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(13) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(14) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(16) InputAdapter +(18) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(17) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(24) SortExecTransformer +(26) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(27) Scan parquet +(29) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(28) Filter +(30) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(29) BroadcastExchange +(31) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(32) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(31) Filter +(33) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(32) Project +(34) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(33) BroadcastHashJoin +(35) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(34) Project +(36) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(35) HashAggregate +(37) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(36) Exchange +(38) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) HashAggregate +(39) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(38) Exchange +(40) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Sort +(41) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(40) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt index 1963624ed312..4a3a239381ca 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (51) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ ProjectExecTransformer (3) - +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (50) - +- Exchange (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Exchange (44) - +- HashAggregate (43) - +- Project (42) - +- BroadcastHashJoin LeftOuter BuildRight (41) - :- Scan parquet (36) - +- BroadcastExchange (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- BroadcastHashJoin LeftOuter BuildRight (42) + :- Scan parquet (37) + +- BroadcastExchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -62,220 +63,224 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(3) ProjectExecTransformer +(3) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] + +(4) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(7) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(11) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, count#X] Input [2]: [c_custkey#X, count#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(14) ColumnarExchange +(15) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [2]: [c_custkey#X, count#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(19) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(29) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(36) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(38) Filter +(39) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(39) Project +(40) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(40) BroadcastExchange +(41) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(43) HashAggregate +(44) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(44) Exchange +(45) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) HashAggregate +(46) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(46) HashAggregate +(47) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(47) Exchange +(48) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(49) Exchange +(50) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Sort +(51) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(51) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt index 1d64bb80886d..ee1fe0a3316c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (32) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ ProjectExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (22) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (31) - +- Exchange (30) - +- HashAggregate (29) - +- Project (28) - +- BroadcastHashJoin Inner BuildRight (27) - :- Project (23) - : +- Filter (22) - : +- Scan parquet (21) - +- BroadcastExchange (26) - +- Filter (25) - +- Scan parquet (24) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -39,144 +41,152 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) Scan parquet +(4) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(5) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(6) WholeStageCodegenTransformer (X) Input [2]: [p_partkey#X, p_type#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(20) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(21) Scan parquet +(23) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(22) Filter +(24) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(23) Project +(25) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(26) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(26) BroadcastExchange +(28) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(27) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(28) Project +(30) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(29) HashAggregate +(31) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(30) Exchange +(32) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(31) HashAggregate +(33) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(32) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt index d8563db5b941..076e82bd53e3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt @@ -1,44 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ BroadcastHashJoinExecTransformer Inner (18) - :- ^ InputIteratorTransformer (6) - : +- ^ InputAdapter (5) - : +- ^ BroadcastQueryStage (4) - : +- ColumnarBroadcastExchange (3) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (17) - +- ^ RegularHashAggregateExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FlushableHashAggregateExecTransformer (9) - +- ^ ProjectExecTransformer (8) - +- ^ Scan parquet (7) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner (20) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- BroadcastHashJoin Inner BuildLeft (38) - :- BroadcastExchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- Project (41) + +- BroadcastHashJoin Inner BuildLeft (40) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Filter (39) + +- HashAggregate (38) + +- Exchange (37) + +- HashAggregate (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -48,182 +50,190 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(3) WholeStageCodegenTransformer (X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(10) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(9) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(11) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(12) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(16) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(17) FilterExecTransformer +(19) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(18) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(25) SortExecTransformer +(27) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(30) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) BroadcastExchange +(32) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(31) Scan parquet +(33) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(35) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(37) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(37) Filter +(39) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) BroadcastHashJoin +(40) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(39) Project +(41) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(42) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt index d8664006f507..603ea771b167 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt @@ -1,62 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (62) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (41) - +- ^ SortExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36) - +- ColumnarExchange (35) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ ProjectExecTransformer (27) - +- ^ FlushableHashAggregateExecTransformer (26) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ BroadcastHashJoinExecTransformer Inner (16) - :- ^ BroadcastHashJoinExecTransformer LeftAnti (9) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ BroadcastQueryStage (6) - : +- ColumnarBroadcastExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ Scan parquet (2) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ BroadcastQueryStage (13) - +- ColumnarBroadcastExchange (12) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (61) - +- Exchange (60) - +- HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- BroadcastHashJoin Inner BuildRight (52) - :- BroadcastHashJoin LeftAnti BuildRight (48) - : :- Filter (43) - : : +- Scan parquet (42) - : +- BroadcastExchange (47) - : +- Project (46) - : +- Filter (45) - : +- Scan parquet (44) - +- BroadcastExchange (51) - +- Filter (50) - +- Scan parquet (49) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- BroadcastHashJoin Inner BuildRight (46) + :- BroadcastHashJoin LeftAnti BuildRight (42) + : :- Filter (37) + : : +- Scan parquet (36) + : +- BroadcastExchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- BroadcastExchange (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -66,278 +61,252 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) Scan parquet -Output [2]: [s_suppkey#X, s_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_comment)] -ReadSchema: struct - -(3) ProjectExecTransformer -Output [1]: [s_suppkey#X] -Input [2]: [s_suppkey#X, s_comment#X] - -(4) WholeStageCodegenTransformer (X) -Input [1]: [s_suppkey#X] -Arguments: false - -(5) ColumnarBroadcastExchange -Input [1]: [s_suppkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] - -(6) BroadcastQueryStage -Output [1]: [s_suppkey#X] -Arguments: X - -(7) InputAdapter -Input [1]: [s_suppkey#X] - -(8) InputIteratorTransformer -Input [1]: [s_suppkey#X] - -(9) BroadcastHashJoinExecTransformer -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None +(2) NoopFilter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X, ps_suppkey#X] -(10) Scan parquet +(3) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(11) WholeStageCodegenTransformer (X) +(4) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(5) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(12) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(7) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(14) InputAdapter +(8) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(15) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(17) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(18) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) ProjectExecTransformer +(13) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(21) ColumnarExchange +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(23) InputAdapter +(17) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(24) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(29) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(31) InputAdapter +(25) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(32) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(34) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) ColumnarExchange +(29) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(30) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(37) InputAdapter +(31) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(38) InputIteratorTransformer +(32) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) SortExecTransformer +(33) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(40) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(41) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) Scan parquet +(36) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(43) Filter +(37) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(44) Scan parquet +(38) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(45) Filter +(39) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(46) Project +(40) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(47) BroadcastExchange +(41) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(48) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(49) Scan parquet +(43) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(50) Filter +(44) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(51) BroadcastExchange +(45) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(52) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(53) Project +(47) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(54) HashAggregate +(48) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(55) Exchange +(49) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(50) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(58) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(60) Exchange +(54) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(55) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(62) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt index 468f95bff9fd..c294e8e6dac1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt @@ -1,83 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (83) +AdaptiveSparkPlan (86) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- TakeOrderedAndProjectExecTransformer (49) - +- ^ RegularHashAggregateExecTransformer (47) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ BroadcastHashJoinExecTransformer Inner (38) - :- ^ ProjectExecTransformer (26) - : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : :- ^ InputIteratorTransformer (6) - : : +- ^ InputAdapter (5) - : : +- ^ BroadcastQueryStage (4) - : : +- ColumnarBroadcastExchange (3) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (24) - : :- ^ Scan parquet (7) - : +- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ BroadcastQueryStage (21) - : +- ColumnarBroadcastExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ FilterExecTransformer (17) - : +- ^ RegularHashAggregateExecTransformer (16) - : +- ^ InputIteratorTransformer (15) - : +- ^ InputAdapter (14) - : +- ^ ShuffleQueryStage (13) - : +- ColumnarExchange (12) - : +- ^ ProjectExecTransformer (10) - : +- ^ FlushableHashAggregateExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ BroadcastQueryStage (35) - +- ColumnarBroadcastExchange (34) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (32) - :- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ BroadcastQueryStage (29) - +- ReusedExchange (28) + VeloxColumnarToRowExec (53) + +- TakeOrderedAndProjectExecTransformer (52) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ ShuffleQueryStage (15) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + :- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ BroadcastQueryStage (32) + +- ReusedExchange (31) +- == Initial Plan == - TakeOrderedAndProject (82) - +- HashAggregate (81) - +- Exchange (80) - +- HashAggregate (79) - +- Project (78) - +- BroadcastHashJoin Inner BuildRight (77) - :- Project (65) - : +- BroadcastHashJoin Inner BuildLeft (64) - : :- BroadcastExchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- BroadcastHashJoin LeftSemi BuildRight (63) - : :- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastExchange (62) - : +- Project (61) - : +- Filter (60) - : +- HashAggregate (59) - : +- Exchange (58) - : +- HashAggregate (57) - : +- Scan parquet (56) - +- BroadcastExchange (76) - +- BroadcastHashJoin LeftSemi BuildRight (75) - :- Filter (67) - : +- Scan parquet (66) - +- BroadcastExchange (74) - +- Project (73) - +- Filter (72) - +- HashAggregate (71) - +- Exchange (70) - +- HashAggregate (69) - +- Scan parquet (68) + TakeOrderedAndProject (85) + +- HashAggregate (84) + +- Exchange (83) + +- HashAggregate (82) + +- Project (81) + +- BroadcastHashJoin Inner BuildRight (80) + :- Project (68) + : +- BroadcastHashJoin Inner BuildLeft (67) + : :- BroadcastExchange (56) + : : +- Filter (55) + : : +- Scan parquet (54) + : +- BroadcastHashJoin LeftSemi BuildRight (66) + : :- Filter (58) + : : +- Scan parquet (57) + : +- BroadcastExchange (65) + : +- Project (64) + : +- Filter (63) + : +- HashAggregate (62) + : +- Exchange (61) + : +- HashAggregate (60) + : +- Scan parquet (59) + +- BroadcastExchange (79) + +- BroadcastHashJoin LeftSemi BuildRight (78) + :- Filter (70) + : +- Scan parquet (69) + +- BroadcastExchange (77) + +- Project (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Scan parquet (71) (1) Scan parquet @@ -87,375 +90,387 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X, c_name#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_name#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(8) Scan parquet +(9) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(10) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(9) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(11) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(12) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(16) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(17) FilterExecTransformer +(19) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(20) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastQueryStage +(23) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [1]: [l_orderkey#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [1]: [l_orderkey#X] -(24) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(25) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(26) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(27) Scan parquet +(29) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(28) ReusedExchange [Reuses operator id: 20] +(30) NoopFilter +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X, l_quantity#X] + +(31) ReusedExchange [Reuses operator id: 22] Output [1]: [l_orderkey#X] -(29) BroadcastQueryStage +(32) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [1]: [l_orderkey#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [1]: [l_orderkey#X] -(32) BroadcastHashJoinExecTransformer +(35) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(34) ColumnarBroadcastExchange +(37) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(38) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(36) InputAdapter +(39) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(37) InputIteratorTransformer +(40) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(38) BroadcastHashJoinExecTransformer +(41) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(39) ProjectExecTransformer +(42) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(40) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(41) ProjectExecTransformer +(44) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(42) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(43) ColumnarExchange +(46) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(47) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(45) InputAdapter +(48) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(46) InputIteratorTransformer +(49) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(47) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(48) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(49) TakeOrderedAndProjectExecTransformer +(52) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(50) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(51) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(53) BroadcastExchange +(56) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(54) Scan parquet +(57) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(55) Filter +(58) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(56) Scan parquet +(59) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(57) HashAggregate +(60) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(58) Exchange +(61) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(62) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(60) Filter +(63) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(61) Project +(64) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(62) BroadcastExchange +(65) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(64) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(65) Project +(68) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(66) Scan parquet +(69) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(67) Filter +(70) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(68) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(69) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(70) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(72) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(73) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(74) BroadcastExchange +(77) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(75) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) BroadcastExchange +(79) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(81) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(79) HashAggregate +(82) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(80) Exchange +(83) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(84) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(82) TakeOrderedAndProject +(85) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(83) AdaptiveSparkPlan +(86) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt index 1be7fcaa15c3..267ae6add0ac 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt @@ -1,34 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (31) +AdaptiveSparkPlan (33) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (30) - +- Exchange (29) - +- HashAggregate (28) - +- Project (27) - +- BroadcastHashJoin Inner BuildRight (26) - :- Project (22) - : +- Filter (21) - : +- Scan parquet (20) - +- BroadcastExchange (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (32) + +- Exchange (31) + +- HashAggregate (30) + +- Project (29) + +- BroadcastHashJoin Inner BuildRight (28) + :- Project (24) + : +- Filter (23) + : +- Scan parquet (22) + +- BroadcastExchange (27) + +- Filter (26) + +- Scan parquet (25) (1) Scan parquet @@ -38,140 +40,148 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(3) ProjectExecTransformer Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(3) Scan parquet +(4) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(5) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(6) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [1]: [revenue#X] -(20) Scan parquet +(22) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(21) Filter +(23) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(22) Project +(24) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(23) Scan parquet +(25) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(24) Filter +(26) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(25) BroadcastExchange +(27) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(26) BroadcastHashJoin +(28) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(27) Project +(29) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(28) HashAggregate +(30) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(29) Exchange +(31) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(30) HashAggregate +(32) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(31) AdaptiveSparkPlan +(33) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt index d19495f88198..1b4522028bb4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt @@ -1,103 +1,105 @@ == Physical Plan == -AdaptiveSparkPlan (105) +AdaptiveSparkPlan (107) +- == Final Plan == - VeloxColumnarToRowExec (68) - +- ^ SortExecTransformer (66) - +- ^ InputIteratorTransformer (65) - +- ^ InputAdapter (64) - +- ^ ShuffleQueryStage (63) - +- ColumnarExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ BroadcastHashJoinExecTransformer Inner (59) - :- ^ ProjectExecTransformer (51) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (50) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ AQEShuffleRead (6) - : : +- ^ ShuffleQueryStage (5) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ BroadcastQueryStage (47) - : +- ColumnarBroadcastExchange (46) - : +- AQEShuffleRead (45) - : +- ShuffleQueryStage (44) - : +- ColumnarExchange (43) - : +- ^ ProjectExecTransformer (41) - : +- ^ BroadcastHashJoinExecTransformer Inner (40) - : :- ^ InputIteratorTransformer (22) - : : +- ^ InputAdapter (21) - : : +- ^ BroadcastQueryStage (20) - : : +- ColumnarBroadcastExchange (19) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (17) - : : :- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ BroadcastQueryStage (14) - : : +- ColumnarBroadcastExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ Scan parquet (10) - : +- ^ FilterExecTransformer (39) - : +- ^ ProjectExecTransformer (38) - : +- ^ RegularHashAggregateExecTransformer (37) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ ShuffleQueryStage (34) - : +- ColumnarExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FlushableHashAggregateExecTransformer (30) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (29) - : :- ^ ProjectExecTransformer (24) - : : +- ^ Scan parquet (23) - : +- ^ InputIteratorTransformer (28) - : +- ^ InputAdapter (27) - : +- ^ BroadcastQueryStage (26) - : +- ReusedExchange (25) - +- ^ InputIteratorTransformer (58) - +- ^ InputAdapter (57) - +- ^ BroadcastQueryStage (56) - +- ColumnarBroadcastExchange (55) - +- ^ ProjectExecTransformer (53) - +- ^ Scan parquet (52) + VeloxColumnarToRowExec (70) + +- ^ SortExecTransformer (68) + +- ^ InputIteratorTransformer (67) + +- ^ InputAdapter (66) + +- ^ ShuffleQueryStage (65) + +- ColumnarExchange (64) + +- ^ ProjectExecTransformer (62) + +- ^ BroadcastHashJoinExecTransformer Inner (61) + :- ^ ProjectExecTransformer (52) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (51) + : :- ^ InputIteratorTransformer (9) + : : +- ^ InputAdapter (8) + : : +- ^ AQEShuffleRead (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (50) + : +- ^ InputAdapter (49) + : +- ^ BroadcastQueryStage (48) + : +- ColumnarBroadcastExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer Inner (44) + : :- ^ InputIteratorTransformer (25) + : : +- ^ InputAdapter (24) + : : +- ^ BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (20) + : : :- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (19) + : : +- ^ InputAdapter (18) + : : +- ^ BroadcastQueryStage (17) + : : +- ColumnarBroadcastExchange (16) + : : +- ^ ProjectExecTransformer (14) + : : +- ^ NoopFilter (13) + : : +- ^ Scan parquet (12) + : +- ^ FilterExecTransformer (43) + : +- ^ ProjectExecTransformer (42) + : +- ^ RegularHashAggregateExecTransformer (41) + : +- ^ InputIteratorTransformer (40) + : +- ^ InputAdapter (39) + : +- ^ ShuffleQueryStage (38) + : +- ColumnarExchange (37) + : +- ^ ProjectExecTransformer (35) + : +- ^ FlushableHashAggregateExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (33) + : :- ^ ProjectExecTransformer (28) + : : +- ^ NoopFilter (27) + : : +- ^ Scan parquet (26) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ BroadcastQueryStage (30) + : +- ReusedExchange (29) + +- ^ InputIteratorTransformer (60) + +- ^ InputAdapter (59) + +- ^ BroadcastQueryStage (58) + +- ColumnarBroadcastExchange (57) + +- ^ ProjectExecTransformer (55) + +- ^ NoopFilter (54) + +- ^ Scan parquet (53) +- == Initial Plan == - Sort (104) - +- Exchange (103) - +- Project (102) - +- BroadcastHashJoin Inner BuildRight (101) - :- Project (96) - : +- ShuffledHashJoin LeftSemi BuildRight (95) - : :- Exchange (71) - : : +- Filter (70) - : : +- Scan parquet (69) - : +- Exchange (94) - : +- Project (93) - : +- BroadcastHashJoin Inner BuildLeft (92) - : :- BroadcastExchange (79) - : : +- BroadcastHashJoin LeftSemi BuildRight (78) - : : :- Filter (73) - : : : +- Scan parquet (72) - : : +- BroadcastExchange (77) - : : +- Project (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Filter (91) - : +- HashAggregate (90) - : +- Exchange (89) - : +- HashAggregate (88) - : +- BroadcastHashJoin LeftSemi BuildRight (87) - : :- Project (82) - : : +- Filter (81) - : : +- Scan parquet (80) - : +- BroadcastExchange (86) - : +- Project (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (100) - +- Project (99) - +- Filter (98) - +- Scan parquet (97) + Sort (106) + +- Exchange (105) + +- Project (104) + +- BroadcastHashJoin Inner BuildRight (103) + :- Project (98) + : +- ShuffledHashJoin LeftSemi BuildRight (97) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (96) + : +- Project (95) + : +- BroadcastHashJoin Inner BuildLeft (94) + : :- BroadcastExchange (81) + : : +- BroadcastHashJoin LeftSemi BuildRight (80) + : : :- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (79) + : : +- Project (78) + : : +- Filter (77) + : : +- Scan parquet (76) + : +- Filter (93) + : +- HashAggregate (92) + : +- Exchange (91) + : +- HashAggregate (90) + : +- BroadcastHashJoin LeftSemi BuildRight (89) + : :- Project (84) + : : +- Filter (83) + : : +- Scan parquet (82) + : +- BroadcastExchange (88) + : +- Project (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (102) + +- Project (101) + +- Filter (100) + +- Scan parquet (99) (1) Scan parquet @@ -107,452 +109,460 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(6) AQEShuffleRead +(7) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) Scan parquet +(11) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(12) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(11) ProjectExecTransformer +(13) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(14) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(12) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(19) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(20) BroadcastQueryStage +(23) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(21) InputAdapter +(24) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(22) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(23) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(24) ProjectExecTransformer +(27) NoopFilter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(28) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(25) ReusedExchange [Reuses operator id: 13] +(29) ReusedExchange [Reuses operator id: 16] Output [1]: [p_partkey#X] -(26) BroadcastQueryStage +(30) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(27) InputAdapter +(31) InputAdapter Input [1]: [p_partkey#X] -(28) InputIteratorTransformer +(32) InputIteratorTransformer Input [1]: [p_partkey#X] -(29) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) FlushableHashAggregateExecTransformer +(34) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(31) ProjectExecTransformer +(35) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(32) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(33) ColumnarExchange +(37) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(38) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(35) InputAdapter +(39) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) InputIteratorTransformer +(40) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(37) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(38) ProjectExecTransformer +(42) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(39) FilterExecTransformer +(43) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(40) BroadcastHashJoinExecTransformer +(44) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(41) ProjectExecTransformer -Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(42) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, ps_suppkey#X] -Arguments: false - -(43) ColumnarExchange -Input [2]: [hash_partition_key#X, ps_suppkey#X] -Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] - -(44) ShuffleQueryStage +(45) ProjectExecTransformer Output [1]: [ps_suppkey#X] -Arguments: X +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(45) AQEShuffleRead +(46) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] -Arguments: local +Arguments: false -(46) ColumnarBroadcastExchange +(47) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastQueryStage +(48) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(48) InputAdapter +(49) InputAdapter Input [1]: [ps_suppkey#X] -(49) InputIteratorTransformer +(50) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(50) BroadcastHashJoinExecTransformer +(51) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(51) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(52) Scan parquet +(53) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(53) ProjectExecTransformer +(54) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(55) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(54) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(55) ColumnarBroadcastExchange +(57) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastQueryStage +(58) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(57) InputAdapter +(59) InputAdapter Input [1]: [n_nationkey#X] -(58) InputIteratorTransformer +(60) InputIteratorTransformer Input [1]: [n_nationkey#X] -(59) BroadcastHashJoinExecTransformer +(61) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(60) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(61) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(62) ColumnarExchange +(64) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(63) ShuffleQueryStage +(65) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(64) InputAdapter +(66) InputAdapter Input [2]: [s_name#X, s_address#X] -(65) InputIteratorTransformer +(67) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(66) SortExecTransformer +(68) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(67) WholeStageCodegenTransformer (X) +(69) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(68) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(69) Scan parquet +(71) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(71) Exchange +(73) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(72) Scan parquet +(74) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(73) Filter +(75) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(74) Scan parquet +(76) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(75) Filter +(77) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(76) Project +(78) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(80) Scan parquet +(82) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(81) Filter +(83) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(82) Project +(84) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(85) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(86) BroadcastExchange +(88) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(87) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(88) HashAggregate +(90) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(89) Exchange +(91) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(92) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(91) Filter +(93) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(92) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(93) Project +(95) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(94) Exchange +(96) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(96) Project +(98) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(97) Scan parquet +(99) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(98) Filter +(100) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(99) Project +(101) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(100) BroadcastExchange +(102) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(101) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(102) Project +(104) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(103) Exchange +(105) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Sort +(106) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(105) AdaptiveSparkPlan +(107) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt index faf7f5799f34..9c09032689eb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt @@ -1,86 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (54) - +- TakeOrderedAndProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48) - +- ColumnarExchange (47) - +- ^ ProjectExecTransformer (45) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ BroadcastHashJoinExecTransformer Inner (42) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (25) - : : +- ^ BroadcastHashJoinExecTransformer Inner (24) - : : :- ^ InputIteratorTransformer (6) - : : : +- ^ InputAdapter (5) - : : : +- ^ BroadcastQueryStage (4) - : : : +- ColumnarBroadcastExchange (3) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (23) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (15) - : : : :- ^ ProjectExecTransformer (8) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (14) - : : : +- ^ InputAdapter (13) - : : : +- ^ BroadcastQueryStage (12) - : : : +- ColumnarBroadcastExchange (11) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (22) - : : +- ^ InputAdapter (21) - : : +- ^ BroadcastQueryStage (20) - : : +- ColumnarBroadcastExchange (19) - : : +- ^ ProjectExecTransformer (17) - : : +- ^ Scan parquet (16) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30) - : +- ColumnarBroadcastExchange (29) - : +- ^ ProjectExecTransformer (27) - : +- ^ Scan parquet (26) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ BroadcastQueryStage (39) - +- ColumnarBroadcastExchange (38) - +- ^ ProjectExecTransformer (36) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (59) + +- TakeOrderedAndProjectExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ BroadcastQueryStage (5) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- ^ InputAdapter (24) + : : +- ^ BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- BroadcastHashJoin Inner BuildRight (81) - :- Project (76) - : +- BroadcastHashJoin Inner BuildRight (75) - : :- Project (70) - : : +- BroadcastHashJoin Inner BuildLeft (69) - : : :- BroadcastExchange (57) - : : : +- Filter (56) - : : : +- Scan parquet (55) - : : +- BroadcastHashJoin LeftAnti BuildRight (68) - : : :- BroadcastHashJoin LeftSemi BuildRight (63) - : : : :- Project (60) - : : : : +- Filter (59) - : : : : +- Scan parquet (58) - : : : +- BroadcastExchange (62) - : : : +- Scan parquet (61) - : : +- BroadcastExchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- BroadcastExchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- BroadcastExchange (80) - +- Project (79) - +- Filter (78) - +- Scan parquet (77) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -90,386 +95,406 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(10) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(9) Scan parquet +(11) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(11) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(12) BroadcastQueryStage +(14) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(15) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(16) Scan parquet +(18) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(17) ProjectExecTransformer +(19) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(20) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(18) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(19) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(20) BroadcastQueryStage +(23) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(21) InputAdapter +(24) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(22) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(23) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(24) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(25) ProjectExecTransformer +(28) ProjectExecTransformer Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(26) Scan parquet +(29) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(27) ProjectExecTransformer +(30) NoopFilter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: [o_orderkey#X, o_orderstatus#X] + +(31) ProjectExecTransformer Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(28) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [1]: [o_orderkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(33) ColumnarBroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(30) BroadcastQueryStage +(34) BroadcastQueryStage Output [1]: [o_orderkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [1]: [o_orderkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [1]: [o_orderkey#X] -(33) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(35) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(36) ProjectExecTransformer +(40) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(41) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(37) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(38) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) BroadcastQueryStage +(44) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(40) InputAdapter +(45) InputAdapter Input [1]: [n_nationkey#X] -(41) InputIteratorTransformer +(46) InputIteratorTransformer Input [1]: [n_nationkey#X] -(42) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(43) ProjectExecTransformer +(48) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(44) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(45) ProjectExecTransformer +(50) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(46) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(47) ColumnarExchange +(52) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(49) InputAdapter +(54) InputAdapter Input [2]: [s_name#X, count#X] -(50) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(51) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(53) TakeOrderedAndProjectExecTransformer +(58) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(54) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(55) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(56) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(57) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(58) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(59) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(60) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(61) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(62) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(64) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(65) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(66) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(67) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(68) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(69) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(70) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(71) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(72) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(73) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(74) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(75) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(76) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(77) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(78) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(79) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(80) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(81) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(83) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(84) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(86) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(87) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt index e33dc6b7fb9b..3dc92be2d96e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt @@ -1,39 +1,40 @@ == Physical Plan == -AdaptiveSparkPlan (37) +AdaptiveSparkPlan (38) +- == Final Plan == - VeloxColumnarToRowExec (25) - +- ^ SortExecTransformer (23) - +- ^ InputIteratorTransformer (22) - +- ^ InputAdapter (21) - +- ^ ShuffleQueryStage (20) - +- ColumnarExchange (19) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FlushableHashAggregateExecTransformer (10) - +- ^ ProjectExecTransformer (9) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (8) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (7) - +- ^ InputAdapter (6) - +- ^ BroadcastQueryStage (5) - +- ColumnarBroadcastExchange (4) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (26) + +- ^ SortExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (36) - +- Exchange (35) - +- HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin LeftAnti BuildRight (30) - :- Filter (27) - : +- Scan parquet (26) - +- BroadcastExchange (29) - +- Scan parquet (28) + Sort (37) + +- Exchange (36) + +- HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin LeftAnti BuildRight (31) + :- Filter (28) + : +- Scan parquet (27) + +- BroadcastExchange (30) + +- Scan parquet (29) (1) Scan parquet @@ -43,160 +44,164 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X, c_phone#X, c_acctbal#X] + +(3) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [o_custkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [o_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [o_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [o_custkey#X] -(8) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(9) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(10) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(14) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(19) ColumnarExchange +(20) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(20) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(21) InputAdapter +(22) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(22) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) SortExecTransformer +(24) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(24) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(25) VeloxColumnarToRowExec +(26) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(26) Scan parquet +(27) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(27) Filter +(28) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(28) Scan parquet +(29) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(29) BroadcastExchange +(30) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(31) Project +(32) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(32) HashAggregate +(33) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(35) Exchange +(36) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Sort +(37) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(37) AdaptiveSparkPlan +(38) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt index e6304542a9ce..f962dbdd765f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt @@ -1,52 +1,55 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (31) - +- TakeOrderedAndProjectExecTransformer (30) - +- ^ ProjectExecTransformer (28) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ BroadcastHashJoinExecTransformer Inner (18) - :- ^ ProjectExecTransformer (10) - : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ BroadcastQueryStage (15) - +- ColumnarBroadcastExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ Scan parquet (11) + VeloxColumnarToRowExec (34) + +- TakeOrderedAndProjectExecTransformer (33) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ BroadcastQueryStage (18) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- Project (45) - +- BroadcastHashJoin Inner BuildRight (44) - :- Project (39) - : +- BroadcastHashJoin Inner BuildLeft (38) - : :- BroadcastExchange (35) - : : +- Project (34) - : : +- Filter (33) - : : +- Scan parquet (32) - : +- Filter (37) - : +- Scan parquet (36) - +- BroadcastExchange (43) - +- Project (42) - +- Filter (41) - +- Scan parquet (40) + TakeOrderedAndProject (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- Project (42) + : +- BroadcastHashJoin Inner BuildLeft (41) + : :- BroadcastExchange (38) + : : +- Project (37) + : : +- Filter (36) + : : +- Scan parquet (35) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -56,222 +59,234 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: [c_custkey#X, c_mktsegment#X] + +(3) ProjectExecTransformer Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [c_custkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(12) ProjectExecTransformer +(14) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(15) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(13) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(15) BroadcastQueryStage +(18) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(19) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(18) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(21) ProjectExecTransformer +(24) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, o_orderdate#X, o_shippriority#X, 42) AS hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(22) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(26) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(28) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(29) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(28) ProjectExecTransformer +(31) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(30) TakeOrderedAndProjectExecTransformer +(33) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(31) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(32) Scan parquet +(35) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(33) Filter +(36) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(34) Project +(37) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(35) BroadcastExchange +(38) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(36) Scan parquet +(39) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(38) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(39) Project +(42) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(43) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(41) Filter +(44) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(42) Project +(45) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(43) BroadcastExchange +(46) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(44) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(45) Project +(48) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(46) HashAggregate +(49) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(47) Exchange +(50) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(51) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(49) TakeOrderedAndProject +(52) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt index 5ea636f9ac11..e50973eb5abf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt @@ -1,44 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (10) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- ^ InputAdapter (10) + +- ^ BroadcastQueryStage (9) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin LeftSemi BuildRight (35) - :- Project (30) - : +- Filter (29) - : +- Scan parquet (28) - +- BroadcastExchange (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin LeftSemi BuildRight (37) + :- Project (32) + : +- Filter (31) + : +- Scan parquet (30) + +- BroadcastExchange (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -48,182 +50,190 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(3) Scan parquet +(4) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(4) ProjectExecTransformer +(5) NoopFilter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(6) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(5) WholeStageCodegenTransformer (X) +(7) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(8) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(7) BroadcastQueryStage +(9) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(8) InputAdapter +(10) InputAdapter Input [1]: [l_orderkey#X] -(9) InputIteratorTransformer +(11) InputIteratorTransformer Input [1]: [l_orderkey#X] -(10) BroadcastHashJoinExecTransformer +(12) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(11) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(12) FlushableHashAggregateExecTransformer +(14) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(13) ProjectExecTransformer +(15) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(14) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(15) ColumnarExchange +(17) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(18) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(17) InputAdapter +(19) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(18) InputIteratorTransformer +(20) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(25) SortExecTransformer +(27) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(28) Scan parquet +(30) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(29) Filter +(31) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(30) Project +(32) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(31) Scan parquet +(33) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(32) Filter +(34) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(33) Project +(35) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(34) BroadcastExchange +(36) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(38) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(37) HashAggregate +(39) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(38) Exchange +(40) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(40) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt index 377110d0e7a1..274c176ff8c8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt @@ -1,92 +1,98 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- ^ SortExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54) - +- ColumnarExchange (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48) - +- ColumnarExchange (47) - +- ^ ProjectExecTransformer (45) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ BroadcastHashJoinExecTransformer Inner (42) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (26) - : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : :- ^ ProjectExecTransformer (18) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : :- ^ InputIteratorTransformer (6) - : : : : : +- ^ InputAdapter (5) - : : : : : +- ^ BroadcastQueryStage (4) - : : : : : +- ColumnarBroadcastExchange (3) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (8) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ BroadcastQueryStage (22) - : : +- ColumnarBroadcastExchange (21) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30) - : +- ColumnarBroadcastExchange (29) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ BroadcastQueryStage (39) - +- ColumnarBroadcastExchange (38) - +- ^ ProjectExecTransformer (36) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ^ InputAdapter (61) + +- ^ ShuffleQueryStage (60) + +- ColumnarExchange (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ^ InputAdapter (55) + +- ^ ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- ^ InputAdapter (18) + : : : +- ^ BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- ^ InputAdapter (36) + : +- ^ BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ BroadcastQueryStage (45) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (93) - +- Exchange (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (62) - : : : : : +- Filter (61) - : : : : : +- Scan parquet (60) - : : : : +- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (86) - +- Project (85) - +- Filter (84) - +- Scan parquet (83) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Project (71) + : : : : +- Filter (70) + : : : : +- Scan parquet (69) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (92) + +- Project (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -96,414 +102,438 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(10) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(15) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) Scan parquet +(22) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(24) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(27) Scan parquet +(31) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] + +(33) WholeStageCodegenTransformer (X) Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(36) ProjectExecTransformer +(41) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(42) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(37) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(38) ColumnarBroadcastExchange +(44) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) BroadcastQueryStage +(45) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(40) InputAdapter +(46) InputAdapter Input [1]: [r_regionkey#X] -(41) InputIteratorTransformer +(47) InputIteratorTransformer Input [1]: [r_regionkey#X] -(42) BroadcastHashJoinExecTransformer +(48) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(43) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(44) FlushableHashAggregateExecTransformer +(50) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(45) ProjectExecTransformer +(51) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(46) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(47) ColumnarExchange +(53) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(49) InputAdapter +(55) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(50) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(51) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(52) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(53) ColumnarExchange +(59) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(55) InputAdapter +(61) InputAdapter Input [2]: [n_name#X, revenue#X] -(56) InputIteratorTransformer +(62) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(57) SortExecTransformer +(63) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(60) Scan parquet +(66) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(61) Filter +(67) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(62) BroadcastExchange +(68) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(69) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(64) Filter +(70) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(65) Project +(71) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(66) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(67) Project +(73) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(68) Scan parquet +(74) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(69) Filter +(75) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(70) BroadcastExchange +(76) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(72) Project +(78) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(79) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(80) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(81) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(77) Project +(83) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(84) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(80) BroadcastExchange +(86) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(88) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(83) Scan parquet +(89) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(84) Filter +(90) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(85) Project +(91) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(86) BroadcastExchange +(92) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(87) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(88) Project +(94) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(89) HashAggregate +(95) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(90) Exchange +(96) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(97) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(92) Exchange +(98) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(99) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(94) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt index cc87fe6d2bbf..8d1a71e9751e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (18) +AdaptiveSparkPlan (19) +- == Final Plan == - VeloxColumnarToRowExec (11) - +- ^ RegularHashAggregateExecTransformer (9) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ ShuffleQueryStage (6) - +- ColumnarExchange (5) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (17) - +- Exchange (16) - +- HashAggregate (15) - +- Project (14) - +- Filter (13) - +- Scan parquet (12) + HashAggregate (18) + +- Exchange (17) + +- HashAggregate (16) + +- Project (15) + +- Filter (14) + +- Scan parquet (13) (1) Scan parquet @@ -26,82 +27,86 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(5) ColumnarExchange +(6) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [sum#X, isEmpty#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(9) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(11) VeloxColumnarToRowExec +(12) VeloxColumnarToRowExec Input [1]: [revenue#X] -(12) Scan parquet +(13) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(13) Filter +(14) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(14) Project +(15) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) HashAggregate +(16) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(16) Exchange +(17) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(17) HashAggregate +(18) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(18) AdaptiveSparkPlan +(19) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt index 8716498ef067..3d5b6496fe27 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt @@ -1,87 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (88) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ SortExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ^ InputAdapter (51) - +- ^ ShuffleQueryStage (50) - +- ColumnarExchange (49) - +- ^ RegularHashAggregateExecTransformer (47) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ BroadcastHashJoinExecTransformer Inner (38) - :- ^ ProjectExecTransformer (33) - : +- ^ BroadcastHashJoinExecTransformer Inner (32) - : :- ^ ProjectExecTransformer (25) - : : +- ^ BroadcastHashJoinExecTransformer Inner (24) - : : :- ^ ProjectExecTransformer (17) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (16) - : : : :- ^ ProjectExecTransformer (9) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (8) - : : : : :- ^ InputIteratorTransformer (6) - : : : : : +- ^ InputAdapter (5) - : : : : : +- ^ BroadcastQueryStage (4) - : : : : : +- ColumnarBroadcastExchange (3) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (15) - : : : +- ^ InputAdapter (14) - : : : +- ^ BroadcastQueryStage (13) - : : : +- ColumnarBroadcastExchange (12) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ BroadcastQueryStage (21) - : : +- ColumnarBroadcastExchange (20) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ BroadcastQueryStage (29) - : +- ColumnarBroadcastExchange (28) - : +- ^ Scan parquet (26) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ BroadcastQueryStage (35) - +- ReusedExchange (34) + VeloxColumnarToRowExec (60) + +- ^ SortExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ^ InputAdapter (56) + +- ^ ShuffleQueryStage (55) + +- ColumnarExchange (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ^ InputAdapter (50) + +- ^ ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- ^ InputAdapter (17) + : : : +- ^ BroadcastQueryStage (16) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- ^ InputAdapter (26) + : : +- ^ BroadcastQueryStage (25) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- ^ InputAdapter (41) + +- ^ BroadcastQueryStage (40) + +- ReusedExchange (39) +- == Initial Plan == - Sort (87) - +- Exchange (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- BroadcastHashJoin Inner BuildRight (81) - :- Project (77) - : +- BroadcastHashJoin Inner BuildRight (76) - : :- Project (72) - : : +- BroadcastHashJoin Inner BuildRight (71) - : : :- Project (67) - : : : +- BroadcastHashJoin Inner BuildRight (66) - : : : :- Project (62) - : : : : +- BroadcastHashJoin Inner BuildLeft (61) - : : : : :- BroadcastExchange (58) - : : : : : +- Filter (57) - : : : : : +- Scan parquet (56) - : : : : +- Filter (60) - : : : : +- Scan parquet (59) - : : : +- BroadcastExchange (65) - : : : +- Filter (64) - : : : +- Scan parquet (63) - : : +- BroadcastExchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (75) - : +- Filter (74) - : +- Scan parquet (73) - +- BroadcastExchange (80) - +- Filter (79) - +- Scan parquet (78) + Sort (92) + +- Exchange (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (77) + : : +- BroadcastHashJoin Inner BuildRight (76) + : : :- Project (72) + : : : +- BroadcastHashJoin Inner BuildRight (71) + : : : :- Project (67) + : : : : +- BroadcastHashJoin Inner BuildLeft (66) + : : : : :- BroadcastExchange (63) + : : : : : +- Filter (62) + : : : : : +- Scan parquet (61) + : : : : +- Filter (65) + : : : : +- Scan parquet (64) + : : : +- BroadcastExchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- BroadcastExchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (80) + : +- Filter (79) + : +- Scan parquet (78) + +- BroadcastExchange (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -91,386 +96,406 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(8) BroadcastHashJoinExecTransformer +(9) NoopFilter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(10) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(9) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) Scan parquet +(12) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(11) WholeStageCodegenTransformer (X) +(13) NoopFilter +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X, o_custkey#X] + +(14) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarBroadcastExchange +(15) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(16) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) BroadcastHashJoinExecTransformer +(19) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(17) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(18) Scan parquet +(21) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(19) WholeStageCodegenTransformer (X) +(22) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(23) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(20) ColumnarBroadcastExchange +(24) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(21) BroadcastQueryStage +(25) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(22) InputAdapter +(26) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(23) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(24) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(25) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(26) Scan parquet +(30) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(27) WholeStageCodegenTransformer (X) +(31) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(32) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(28) ColumnarBroadcastExchange +(33) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastQueryStage +(34) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(30) InputAdapter +(35) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(31) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(32) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(38) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(34) ReusedExchange [Reuses operator id: 28] +(39) ReusedExchange [Reuses operator id: 33] Output [2]: [n_nationkey#X, n_name#X] -(35) BroadcastQueryStage +(40) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(36) InputAdapter +(41) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(37) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(38) BroadcastHashJoinExecTransformer +(43) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(39) ProjectExecTransformer +(44) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(40) FlushableHashAggregateExecTransformer +(45) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(43) ColumnarExchange +(48) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(49) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(45) InputAdapter +(50) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(46) InputIteratorTransformer +(51) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(47) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(48) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(49) ColumnarExchange +(54) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(55) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(51) InputAdapter +(56) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(52) InputIteratorTransformer +(57) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(53) SortExecTransformer +(58) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(54) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(55) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(56) Scan parquet +(61) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(57) Filter +(62) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(58) BroadcastExchange +(63) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(59) Scan parquet +(64) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(60) Filter +(65) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(61) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(62) Project +(67) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(63) Scan parquet +(68) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(64) Filter +(69) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(65) BroadcastExchange +(70) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(66) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(67) Project +(72) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(68) Scan parquet +(73) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(70) BroadcastExchange +(75) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(72) Project +(77) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(73) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(74) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(75) BroadcastExchange +(80) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(77) Project +(82) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(78) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(79) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(80) BroadcastExchange +(85) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(82) Project +(87) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(88) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(84) Exchange +(89) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(90) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(86) Exchange +(91) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Sort +(92) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(88) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt index 94e003e55d93..9489452b5272 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt @@ -1,117 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (121) +AdaptiveSparkPlan (129) +- == Final Plan == - VeloxColumnarToRowExec (76) - +- ^ SortExecTransformer (74) - +- ^ InputIteratorTransformer (73) - +- ^ InputAdapter (72) - +- ^ ShuffleQueryStage (71) - +- ColumnarExchange (70) - +- ^ ProjectExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (67) - +- ^ InputIteratorTransformer (66) - +- ^ InputAdapter (65) - +- ^ ShuffleQueryStage (64) - +- ColumnarExchange (63) - +- ^ ProjectExecTransformer (61) - +- ^ FlushableHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ BroadcastHashJoinExecTransformer Inner (58) - :- ^ ProjectExecTransformer (50) - : +- ^ BroadcastHashJoinExecTransformer Inner (49) - : :- ^ ProjectExecTransformer (42) - : : +- ^ BroadcastHashJoinExecTransformer Inner (41) - : : :- ^ ProjectExecTransformer (34) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : : : :- ^ ProjectExecTransformer (26) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : : : :- ^ ProjectExecTransformer (18) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : : : :- ^ ProjectExecTransformer (10) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : : : :- ^ InputIteratorTransformer (7) - : : : : : : : +- ^ InputAdapter (6) - : : : : : : : +- ^ BroadcastQueryStage (5) - : : : : : : : +- ColumnarBroadcastExchange (4) - : : : : : : : +- ^ ProjectExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ Scan parquet (8) - : : : : : +- ^ InputIteratorTransformer (16) - : : : : : +- ^ InputAdapter (15) - : : : : : +- ^ BroadcastQueryStage (14) - : : : : : +- ColumnarBroadcastExchange (13) - : : : : : +- ^ Scan parquet (11) - : : : : +- ^ InputIteratorTransformer (24) - : : : : +- ^ InputAdapter (23) - : : : : +- ^ BroadcastQueryStage (22) - : : : : +- ColumnarBroadcastExchange (21) - : : : : +- ^ Scan parquet (19) - : : : +- ^ InputIteratorTransformer (32) - : : : +- ^ InputAdapter (31) - : : : +- ^ BroadcastQueryStage (30) - : : : +- ColumnarBroadcastExchange (29) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (40) - : : +- ^ InputAdapter (39) - : : +- ^ BroadcastQueryStage (38) - : : +- ColumnarBroadcastExchange (37) - : : +- ^ Scan parquet (35) - : +- ^ InputIteratorTransformer (48) - : +- ^ InputAdapter (47) - : +- ^ BroadcastQueryStage (46) - : +- ColumnarBroadcastExchange (45) - : +- ^ Scan parquet (43) - +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ BroadcastQueryStage (55) - +- ColumnarBroadcastExchange (54) - +- ^ ProjectExecTransformer (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (84) + +- ^ SortExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ^ InputAdapter (80) + +- ^ ShuffleQueryStage (79) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ^ InputAdapter (73) + +- ^ ShuffleQueryStage (72) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ^ InputAdapter (7) + : : : : : : : +- ^ BroadcastQueryStage (6) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- ^ InputAdapter (18) + : : : : : +- ^ BroadcastQueryStage (17) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- ^ InputAdapter (27) + : : : : +- ^ BroadcastQueryStage (26) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- ^ InputAdapter (36) + : : : +- ^ BroadcastQueryStage (35) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ BroadcastQueryStage (44) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- ^ InputAdapter (54) + : +- ^ BroadcastQueryStage (53) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- ^ InputAdapter (64) + +- ^ BroadcastQueryStage (63) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (120) - +- Exchange (119) - +- HashAggregate (118) - +- Exchange (117) - +- HashAggregate (116) - +- Project (115) - +- BroadcastHashJoin Inner BuildRight (114) - :- Project (109) - : +- BroadcastHashJoin Inner BuildRight (108) - : :- Project (104) - : : +- BroadcastHashJoin Inner BuildRight (103) - : : :- Project (99) - : : : +- BroadcastHashJoin Inner BuildRight (98) - : : : :- Project (94) - : : : : +- BroadcastHashJoin Inner BuildRight (93) - : : : : :- Project (89) - : : : : : +- BroadcastHashJoin Inner BuildRight (88) - : : : : : :- Project (84) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (83) - : : : : : : :- BroadcastExchange (80) - : : : : : : : +- Project (79) - : : : : : : : +- Filter (78) - : : : : : : : +- Scan parquet (77) - : : : : : : +- Filter (82) - : : : : : : +- Scan parquet (81) - : : : : : +- BroadcastExchange (87) - : : : : : +- Filter (86) - : : : : : +- Scan parquet (85) - : : : : +- BroadcastExchange (92) - : : : : +- Filter (91) - : : : : +- Scan parquet (90) - : : : +- BroadcastExchange (97) - : : : +- Filter (96) - : : : +- Scan parquet (95) - : : +- BroadcastExchange (102) - : : +- Filter (101) - : : +- Scan parquet (100) - : +- BroadcastExchange (107) - : +- Filter (106) - : +- Scan parquet (105) - +- BroadcastExchange (113) - +- Project (112) - +- Filter (111) - +- Scan parquet (110) + Sort (128) + +- Exchange (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- BroadcastHashJoin Inner BuildRight (122) + :- Project (117) + : +- BroadcastHashJoin Inner BuildRight (116) + : :- Project (112) + : : +- BroadcastHashJoin Inner BuildRight (111) + : : :- Project (107) + : : : +- BroadcastHashJoin Inner BuildRight (106) + : : : :- Project (102) + : : : : +- BroadcastHashJoin Inner BuildRight (101) + : : : : :- Project (97) + : : : : : +- BroadcastHashJoin Inner BuildRight (96) + : : : : : :- Project (92) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) + : : : : : : :- BroadcastExchange (88) + : : : : : : : +- Project (87) + : : : : : : : +- Filter (86) + : : : : : : : +- Scan parquet (85) + : : : : : : +- Filter (90) + : : : : : : +- Scan parquet (89) + : : : : : +- BroadcastExchange (95) + : : : : : +- Filter (94) + : : : : : +- Scan parquet (93) + : : : : +- BroadcastExchange (100) + : : : : +- Filter (99) + : : : : +- Scan parquet (98) + : : : +- BroadcastExchange (105) + : : : +- Filter (104) + : : : +- Scan parquet (103) + : : +- BroadcastExchange (110) + : : +- Filter (109) + : : +- Scan parquet (108) + : +- BroadcastExchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- BroadcastExchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -121,534 +129,566 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(3) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) Scan parquet +(13) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(15) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(19) Scan parquet +(22) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(24) WholeStageCodegenTransformer (X) Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(27) Scan parquet +(31) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(33) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(36) WholeStageCodegenTransformer (X) +(41) NoopFilter +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_regionkey#X] + +(42) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: false -(37) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(44) BroadcastQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(39) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(40) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(41) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(42) ProjectExecTransformer +(48) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(43) Scan parquet +(49) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(44) WholeStageCodegenTransformer (X) +(50) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(51) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(45) ColumnarBroadcastExchange +(52) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastQueryStage +(53) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(47) InputAdapter +(54) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(48) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastHashJoinExecTransformer +(56) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(50) ProjectExecTransformer +(57) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(51) Scan parquet +(58) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(52) ProjectExecTransformer +(59) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(60) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(53) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(54) ColumnarBroadcastExchange +(62) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(55) BroadcastQueryStage +(63) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(56) InputAdapter +(64) InputAdapter Input [1]: [r_regionkey#X] -(57) InputIteratorTransformer +(65) InputIteratorTransformer Input [1]: [r_regionkey#X] -(58) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(59) ProjectExecTransformer +(67) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(60) FlushableHashAggregateExecTransformer +(68) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(61) ProjectExecTransformer +(69) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(62) WholeStageCodegenTransformer (X) +(70) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(63) ColumnarExchange +(71) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(64) ShuffleQueryStage +(72) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(65) InputAdapter +(73) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(66) InputIteratorTransformer +(74) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(67) RegularHashAggregateExecTransformer +(75) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(68) ProjectExecTransformer +(76) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(69) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(70) ColumnarExchange +(78) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(71) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(72) InputAdapter +(80) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(73) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(74) SortExecTransformer +(82) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(75) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(76) VeloxColumnarToRowExec +(84) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(77) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(78) Filter +(86) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(79) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(80) BroadcastExchange +(88) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(81) Scan parquet +(89) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(82) Filter +(90) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(83) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(84) Project +(92) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(85) Scan parquet +(93) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(94) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(95) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(96) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(89) Project +(97) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(98) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(91) Filter +(99) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(92) BroadcastExchange +(100) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(93) BroadcastHashJoin +(101) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(94) Project +(102) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(95) Scan parquet +(103) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(104) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) BroadcastExchange +(105) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(98) BroadcastHashJoin +(106) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(99) Project +(107) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(100) Scan parquet +(108) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(101) Filter +(109) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(102) BroadcastExchange +(110) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(103) BroadcastHashJoin +(111) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) Project +(112) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(105) Scan parquet +(113) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(106) Filter +(114) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(107) BroadcastExchange +(115) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(108) BroadcastHashJoin +(116) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(109) Project +(117) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(110) Scan parquet +(118) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(111) Filter +(119) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(112) Project +(120) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(113) BroadcastExchange +(121) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(114) BroadcastHashJoin +(122) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(115) Project +(123) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(116) HashAggregate +(124) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(117) Exchange +(125) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) HashAggregate +(126) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(119) Exchange +(127) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Sort +(128) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(121) AdaptiveSparkPlan +(129) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt index d337c7f9e468..29e6d2f72f57 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt @@ -1,90 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (98) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ SortExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (26) - : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : :- ^ ProjectExecTransformer (18) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ BroadcastQueryStage (22) - : : +- ColumnarBroadcastExchange (21) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30) - : +- ColumnarBroadcastExchange (29) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38) - +- ColumnarBroadcastExchange (37) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (64) + +- ^ SortExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ^ InputAdapter (60) + +- ^ ShuffleQueryStage (59) + +- ColumnarExchange (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ BroadcastQueryStage (6) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- ^ InputAdapter (18) + : : : +- ^ BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- ^ InputAdapter (36) + : +- ^ BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (91) - +- Exchange (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (76) - : : +- BroadcastHashJoin Inner BuildRight (75) - : : :- Project (71) - : : : +- BroadcastHashJoin Inner BuildRight (70) - : : : :- Project (66) - : : : : +- BroadcastHashJoin Inner BuildLeft (65) - : : : : :- BroadcastExchange (62) - : : : : : +- Project (61) - : : : : : +- Filter (60) - : : : : : +- Scan parquet (59) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- BroadcastExchange (74) - : : +- Filter (73) - : : +- Scan parquet (72) - : +- BroadcastExchange (79) - : +- Filter (78) - : +- Scan parquet (77) - +- BroadcastExchange (84) - +- Filter (83) - +- Scan parquet (82) + Sort (97) + +- Exchange (96) + +- HashAggregate (95) + +- Exchange (94) + +- HashAggregate (93) + +- Project (92) + +- BroadcastHashJoin Inner BuildRight (91) + :- Project (87) + : +- BroadcastHashJoin Inner BuildRight (86) + : :- Project (82) + : : +- BroadcastHashJoin Inner BuildRight (81) + : : :- Project (77) + : : : +- BroadcastHashJoin Inner BuildRight (76) + : : : :- Project (72) + : : : : +- BroadcastHashJoin Inner BuildLeft (71) + : : : : :- BroadcastExchange (68) + : : : : : +- Project (67) + : : : : : +- Filter (66) + : : : : : +- Scan parquet (65) + : : : : +- Filter (70) + : : : : +- Scan parquet (69) + : : : +- BroadcastExchange (75) + : : : +- Filter (74) + : : : +- Scan parquet (73) + : : +- BroadcastExchange (80) + : : +- Filter (79) + : : +- Scan parquet (78) + : +- BroadcastExchange (85) + : +- Filter (84) + : +- Scan parquet (83) + +- BroadcastExchange (90) + +- Filter (89) + +- Scan parquet (88) (1) Scan parquet @@ -94,406 +100,430 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(3) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) Scan parquet +(13) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(15) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(19) Scan parquet +(22) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(24) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(27) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_orderdate#X] + +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(36) WholeStageCodegenTransformer (X) +(41) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(42) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(37) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(44) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(39) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(40) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(41) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(42) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(43) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(50) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(52) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(53) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(54) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(55) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(56) SortExecTransformer +(62) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(59) Scan parquet +(65) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(60) Filter +(66) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(61) Project +(67) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(62) BroadcastExchange +(68) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) Scan parquet +(69) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(64) Filter +(70) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(65) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(66) Project +(72) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(67) Scan parquet +(73) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(68) Filter +(74) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(69) BroadcastExchange +(75) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(70) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(71) Project +(77) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(72) Scan parquet +(78) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(73) Filter +(79) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(74) BroadcastExchange +(80) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(75) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(76) Project +(82) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(77) Scan parquet +(83) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(78) Filter +(84) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(79) BroadcastExchange +(85) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(81) Project +(87) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(82) Scan parquet +(88) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(83) Filter +(89) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(84) BroadcastExchange +(90) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(85) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) Project +(92) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(87) HashAggregate +(93) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(88) Exchange +(94) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(95) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(90) Exchange +(96) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Sort +(97) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(92) AdaptiveSparkPlan +(98) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt index c5d58658acd5..74f79bd3ee64 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (27) +AdaptiveSparkPlan (28) +- == Final Plan == - VeloxColumnarToRowExec (18) - +- ^ SortExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (19) + +- ^ SortExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (26) - +- Exchange (25) - +- HashAggregate (24) - +- Exchange (23) - +- HashAggregate (22) - +- Project (21) - +- Filter (20) - +- Scan parquet (19) + Sort (27) + +- Exchange (26) + +- HashAggregate (25) + +- Exchange (24) + +- HashAggregate (23) + +- Project (22) + +- Filter (21) + +- Scan parquet (20) (1) Scan parquet @@ -34,116 +35,120 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(4) ProjectExecTransformer +(5) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(6) ColumnarExchange +(7) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(12) ColumnarExchange +(13) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(14) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) InputAdapter +(15) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(15) InputIteratorTransformer +(16) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) SortExecTransformer +(17) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(17) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(18) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(19) Scan parquet +(20) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(20) Filter +(21) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(21) Project +(22) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(22) HashAggregate +(23) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(23) Exchange +(24) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(24) HashAggregate +(25) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(25) Exchange +(26) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Sort +(27) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(27) AdaptiveSparkPlan +(28) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt index c599bb00a7bb..276588749734 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt @@ -1,64 +1,68 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- ^ ProjectExecTransformer (29) - +- ^ FlushableHashAggregateExecTransformer (28) - +- ^ ProjectExecTransformer (27) - +- ^ BroadcastHashJoinExecTransformer Inner (26) - :- ^ ProjectExecTransformer (19) - : +- ^ BroadcastHashJoinExecTransformer Inner (18) - : :- ^ ProjectExecTransformer (10) - : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : :- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ Scan parquet (2) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ BroadcastQueryStage (15), Statistics(X) - : +- ColumnarBroadcastExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ Scan parquet (11) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ BroadcastQueryStage (23), Statistics(X) - +- ColumnarBroadcastExchange (22) - +- ^ Scan parquet (20) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- ^ InputAdapter (9) + : : +- ^ BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- ^ InputAdapter (19) + : +- ^ BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- BroadcastHashJoin Inner BuildRight (57) - :- Project (53) - : +- BroadcastHashJoin Inner BuildRight (52) - : :- Project (47) - : : +- BroadcastHashJoin Inner BuildRight (46) - : : :- Filter (41) - : : : +- Scan parquet (40) - : : +- BroadcastExchange (45) - : : +- Project (44) - : : +- Filter (43) - : : +- Scan parquet (42) - : +- BroadcastExchange (51) - : +- Project (50) - : +- Filter (49) - : +- Scan parquet (48) - +- BroadcastExchange (56) - +- Filter (55) - +- Scan parquet (54) + TakeOrderedAndProject (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- BroadcastHashJoin Inner BuildRight (61) + :- Project (57) + : +- BroadcastHashJoin Inner BuildRight (56) + : :- Project (51) + : : +- BroadcastHashJoin Inner BuildRight (50) + : : :- Filter (45) + : : : +- Scan parquet (44) + : : +- BroadcastExchange (49) + : : +- Project (48) + : : +- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (55) + : +- Project (54) + : +- Filter (53) + : +- Scan parquet (52) + +- BroadcastExchange (60) + +- Filter (59) + +- Scan parquet (58) (1) Scan parquet @@ -68,280 +72,296 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(3) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(3) ProjectExecTransformer +(4) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(5) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(4) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(12) ProjectExecTransformer +(14) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(15) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(13) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(15) BroadcastQueryStage +(18) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(19) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(18) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) Scan parquet +(23) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(21) WholeStageCodegenTransformer (X) +(24) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(25) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(22) ColumnarBroadcastExchange +(26) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(27) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(24) InputAdapter +(28) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(25) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(26) BroadcastHashJoinExecTransformer +(30) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(27) ProjectExecTransformer +(31) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(28) FlushableHashAggregateExecTransformer +(32) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(29) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(30) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(31) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(33) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(34) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(40) Scan parquet +(44) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(42) Scan parquet +(46) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(43) Filter +(47) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(44) Project +(48) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(45) BroadcastExchange +(49) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(46) BroadcastHashJoin +(50) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(47) Project +(51) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(48) Scan parquet +(52) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(49) Filter +(53) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(50) Project +(54) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(51) BroadcastExchange +(55) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(52) BroadcastHashJoin +(56) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(53) Project +(57) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(54) Scan parquet +(58) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(55) Filter +(59) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(56) BroadcastExchange +(60) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) BroadcastHashJoin +(61) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(58) Project +(62) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(59) HashAggregate +(63) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(60) Exchange +(64) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(65) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(62) TakeOrderedAndProject +(66) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(63) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt index 459c52738acb..54f535dfbfb6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ FilterExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ BroadcastHashJoinExecTransformer Inner (17) - :- ^ ProjectExecTransformer (9) - : +- ^ BroadcastHashJoinExecTransformer Inner (8) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ Scan parquet (2) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ BroadcastQueryStage (14), Statistics(X) - +- ColumnarBroadcastExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ^ InputAdapter (34) + +- ^ ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ^ InputAdapter (27) + +- ^ ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- ^ InputAdapter (8) + : +- ^ BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- ^ InputAdapter (18) + +- ^ BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (54) - +- Exchange (53) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildRight (41) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (57) + +- Exchange (56) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Project (51) + +- BroadcastHashJoin Inner BuildRight (50) + :- Project (45) + : +- BroadcastHashJoin Inner BuildRight (44) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (49) + +- Project (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -60,464 +63,481 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(3) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(3) WholeStageCodegenTransformer (X) +(4) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(5) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(7) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(9) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(10) Scan parquet +(12) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(11) ProjectExecTransformer +(13) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(14) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(12) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [1]: [n_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [1]: [n_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(19) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(20) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(21) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(22) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(27) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(29) ColumnarExchange +(32) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(31) InputAdapter +(34) InputAdapter Input [2]: [ps_partkey#X, value#X] -(32) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(33) SortExecTransformer +(36) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(35) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(36) Scan parquet +(39) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(38) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(40) BroadcastExchange +(43) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(41) BroadcastHashJoin +(44) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(42) Project +(45) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(45) Project +(48) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(46) BroadcastExchange +(49) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(50) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) Project +(51) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(49) HashAggregate +(52) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(50) Exchange +(53) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(52) Filter +(55) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(53) Exchange +(56) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Sort +(57) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(55) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 27 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (95) +Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ^ InputAdapter (73) - +- ^ ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ FlushableHashAggregateExecTransformer (69) - +- ^ ProjectExecTransformer (68) - +- ^ BroadcastHashJoinExecTransformer Inner (67) - :- ^ ProjectExecTransformer (62) - : +- ^ BroadcastHashJoinExecTransformer Inner (61) - : :- ^ Scan parquet (56) - : +- ^ InputIteratorTransformer (60) - : +- ^ InputAdapter (59) - : +- ^ BroadcastQueryStage (58), Statistics(X) - : +- ReusedExchange (57) - +- ^ InputIteratorTransformer (66) - +- ^ InputAdapter (65) - +- ^ BroadcastQueryStage (64), Statistics(X) - +- ReusedExchange (63) + VeloxColumnarToRowExec (82) + +- ^ ProjectExecTransformer (80) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ^ InputAdapter (77) + +- ^ ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ BroadcastHashJoinExecTransformer Inner (71) + :- ^ ProjectExecTransformer (66) + : +- ^ BroadcastHashJoinExecTransformer Inner (65) + : :- ^ NoopFilter (60) + : : +- ^ Scan parquet (59) + : +- ^ InputIteratorTransformer (64) + : +- ^ InputAdapter (63) + : +- ^ BroadcastQueryStage (62), Statistics(X) + : +- ReusedExchange (61) + +- ^ InputIteratorTransformer (70) + +- ^ InputAdapter (69) + +- ^ BroadcastQueryStage (68), Statistics(X) + +- ReusedExchange (67) +- == Initial Plan == - HashAggregate (94) - +- Exchange (93) - +- HashAggregate (92) - +- Project (91) - +- BroadcastHashJoin Inner BuildRight (90) - :- Project (85) - : +- BroadcastHashJoin Inner BuildRight (84) - : :- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (83) - : +- Filter (82) - : +- Scan parquet (81) - +- BroadcastExchange (89) - +- Project (88) - +- Filter (87) - +- Scan parquet (86) - - -(56) Scan parquet + HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Filter (84) + : : +- Scan parquet (83) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) + + +(59) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(57) ReusedExchange [Reuses operator id: 4] +(60) NoopFilter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(61) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(58) BroadcastQueryStage +(62) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(59) InputAdapter +(63) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(60) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(65) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(62) ProjectExecTransformer +(66) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(63) ReusedExchange [Reuses operator id: 13] +(67) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(64) BroadcastQueryStage +(68) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(65) InputAdapter +(69) InputAdapter Input [1]: [n_nationkey#X] -(66) InputIteratorTransformer +(70) InputIteratorTransformer Input [1]: [n_nationkey#X] -(67) BroadcastHashJoinExecTransformer +(71) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(68) ProjectExecTransformer +(72) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(69) FlushableHashAggregateExecTransformer +(73) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(70) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(75) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(77) InputAdapter Input [2]: [sum#X, isEmpty#X] -(74) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(79) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(76) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(77) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(78) VeloxColumnarToRowExec +(82) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(79) Scan parquet +(83) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(80) Filter +(84) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(81) Scan parquet +(85) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(82) Filter +(86) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(83) BroadcastExchange +(87) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(84) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(85) Project +(89) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(86) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(87) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(88) Project +(92) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(89) BroadcastExchange +(93) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(90) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(91) Project +(95) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(92) HashAggregate +(96) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(93) Exchange +(97) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(98) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(95) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt index d0dca8b40c03..1f1fcd3fb577 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (40) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ InputIteratorTransformer (6) - : +- ^ InputAdapter (5) - : +- ^ BroadcastQueryStage (4), Statistics(X) - : +- ColumnarBroadcastExchange (3) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (8) - +- ^ Scan parquet (7) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ^ InputAdapter (18) + +- ^ ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (39) - +- Exchange (38) - +- HashAggregate (37) - +- Exchange (36) - +- HashAggregate (35) - +- Project (34) - +- BroadcastHashJoin Inner BuildLeft (33) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- BroadcastHashJoin Inner BuildLeft (35) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -46,174 +48,182 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderpriority#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(10) ProjectExecTransformer Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(12) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(13) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(14) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(16) InputAdapter +(18) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(17) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(24) SortExecTransformer +(26) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(27) Scan parquet +(29) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(28) Filter +(30) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(29) BroadcastExchange +(31) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(32) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(31) Filter +(33) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(32) Project +(34) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(33) BroadcastHashJoin +(35) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(34) Project +(36) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(35) HashAggregate +(37) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(36) Exchange +(38) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) HashAggregate +(39) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(38) Exchange +(40) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Sort +(41) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(40) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt index a9269637304d..e67819b82860 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (51) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ ProjectExecTransformer (3) - +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (50) - +- Exchange (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Exchange (44) - +- HashAggregate (43) - +- Project (42) - +- BroadcastHashJoin LeftOuter BuildRight (41) - :- Scan parquet (36) - +- BroadcastExchange (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- BroadcastHashJoin LeftOuter BuildRight (42) + :- Scan parquet (37) + +- BroadcastExchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -62,220 +63,224 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(3) ProjectExecTransformer +(3) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] + +(4) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(7) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(11) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, count#X] Input [2]: [c_custkey#X, count#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(14) ColumnarExchange +(15) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [2]: [c_custkey#X, count#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(19) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(29) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(36) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(38) Filter +(39) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(39) Project +(40) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(40) BroadcastExchange +(41) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(43) HashAggregate +(44) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(44) Exchange +(45) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) HashAggregate +(46) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(46) HashAggregate +(47) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(47) Exchange +(48) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(49) Exchange +(50) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Sort +(51) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(51) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt index 2b4a4b697d1d..eb04b6c18271 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (32) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ ProjectExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (22) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (31) - +- Exchange (30) - +- HashAggregate (29) - +- Project (28) - +- BroadcastHashJoin Inner BuildRight (27) - :- Project (23) - : +- Filter (22) - : +- Scan parquet (21) - +- BroadcastExchange (26) - +- Filter (25) - +- Scan parquet (24) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -39,144 +41,152 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) Scan parquet +(4) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(5) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(6) WholeStageCodegenTransformer (X) Input [2]: [p_partkey#X, p_type#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(20) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(21) Scan parquet +(23) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(22) Filter +(24) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(23) Project +(25) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(26) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(26) BroadcastExchange +(28) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(27) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(28) Project +(30) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(29) HashAggregate +(31) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(30) Exchange +(32) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(31) HashAggregate +(33) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(32) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt index 3bb88d987670..5c6304211102 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (39) +AdaptiveSparkPlan (41) +- == Final Plan == - VeloxColumnarToRowExec (24) - +- AQEShuffleRead (23) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ BroadcastHashJoinExecTransformer Inner (18) - :- ^ InputIteratorTransformer (6) - : +- ^ InputAdapter (5) - : +- ^ BroadcastQueryStage (4), Statistics(X) - : +- ColumnarBroadcastExchange (3) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (17) - +- ^ RegularHashAggregateExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FlushableHashAggregateExecTransformer (9) - +- ^ ProjectExecTransformer (8) - +- ^ Scan parquet (7) + VeloxColumnarToRowExec (26) + +- AQEShuffleRead (25) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner (20) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (38) - +- Exchange (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Filter (34) - +- HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- Filter (29) - +- Scan parquet (28) + Sort (40) + +- Exchange (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (29) + : +- Filter (28) + : +- Scan parquet (27) + +- Filter (36) + +- HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -46,324 +48,337 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(3) WholeStageCodegenTransformer (X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(10) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(9) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(11) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(12) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(16) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(17) FilterExecTransformer +(19) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(18) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(23) AQEShuffleRead +(25) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(24) VeloxColumnarToRowExec +(26) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(25) Scan parquet +(27) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(26) Filter +(28) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(27) BroadcastExchange +(29) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(30) Project +(32) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(31) HashAggregate +(33) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(32) Exchange +(34) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(34) Filter +(36) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(36) Project +(38) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(37) Exchange +(39) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Sort +(40) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(39) AdaptiveSparkPlan +(41) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 17 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (63) +Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (66) +- == Final Plan == - VeloxColumnarToRowExec (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ ProjectExecTransformer (50) - +- ^ RegularHashAggregateExecTransformer (49) - +- ^ InputIteratorTransformer (48) - +- ^ InputAdapter (47) - +- ^ ShuffleQueryStage (46), Statistics(X) - +- ColumnarExchange (45) - +- ^ ProjectExecTransformer (43) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (57) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ^ InputAdapter (50) + +- ^ ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ NoopFilter (43) + +- ^ Scan parquet (42) +- == Initial Plan == - HashAggregate (62) - +- HashAggregate (61) - +- HashAggregate (60) - +- Exchange (59) - +- HashAggregate (58) - +- Project (57) - +- Filter (56) - +- Scan parquet (55) + HashAggregate (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- Filter (59) + +- Scan parquet (58) -(40) Scan parquet +(42) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(41) ProjectExecTransformer +(43) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(44) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(42) FlushableHashAggregateExecTransformer +(45) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(43) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(44) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(45) ColumnarExchange +(48) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(46) ShuffleQueryStage +(49) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(47) InputAdapter +(50) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) InputIteratorTransformer +(51) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(49) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(50) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(51) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(52) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(53) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(54) VeloxColumnarToRowExec +(57) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(55) Scan parquet +(58) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(56) Filter +(59) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(57) Project +(60) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(58) HashAggregate +(61) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(59) Exchange +(62) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) HashAggregate +(63) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(61) HashAggregate +(64) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(63) AdaptiveSparkPlan +(66) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt index 2d1099920a41..9a4005e3da85 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt @@ -1,62 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (62) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (41) - +- ^ SortExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ ProjectExecTransformer (27) - +- ^ FlushableHashAggregateExecTransformer (26) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ BroadcastHashJoinExecTransformer Inner (16) - :- ^ BroadcastHashJoinExecTransformer LeftAnti (9) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ BroadcastQueryStage (6), Statistics(X) - : +- ColumnarBroadcastExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ Scan parquet (2) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ BroadcastQueryStage (13), Statistics(X) - +- ColumnarBroadcastExchange (12) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (61) - +- Exchange (60) - +- HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- BroadcastHashJoin Inner BuildRight (52) - :- BroadcastHashJoin LeftAnti BuildRight (48) - : :- Filter (43) - : : +- Scan parquet (42) - : +- BroadcastExchange (47) - : +- Project (46) - : +- Filter (45) - : +- Scan parquet (44) - +- BroadcastExchange (51) - +- Filter (50) - +- Scan parquet (49) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- BroadcastHashJoin Inner BuildRight (46) + :- BroadcastHashJoin LeftAnti BuildRight (42) + : :- Filter (37) + : : +- Scan parquet (36) + : +- BroadcastExchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- BroadcastExchange (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -66,278 +61,252 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) Scan parquet -Output [2]: [s_suppkey#X, s_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_comment)] -ReadSchema: struct - -(3) ProjectExecTransformer -Output [1]: [s_suppkey#X] -Input [2]: [s_suppkey#X, s_comment#X] - -(4) WholeStageCodegenTransformer (X) -Input [1]: [s_suppkey#X] -Arguments: false - -(5) ColumnarBroadcastExchange -Input [1]: [s_suppkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] - -(6) BroadcastQueryStage -Output [1]: [s_suppkey#X] -Arguments: X - -(7) InputAdapter -Input [1]: [s_suppkey#X] - -(8) InputIteratorTransformer -Input [1]: [s_suppkey#X] - -(9) BroadcastHashJoinExecTransformer -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None +(2) NoopFilter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X, ps_suppkey#X] -(10) Scan parquet +(3) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(11) WholeStageCodegenTransformer (X) +(4) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(5) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(12) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(7) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(14) InputAdapter +(8) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(15) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(17) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(18) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) ProjectExecTransformer +(13) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(21) ColumnarExchange +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(23) InputAdapter +(17) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(24) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(29) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(31) InputAdapter +(25) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(32) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(34) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) ColumnarExchange +(29) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(30) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(37) InputAdapter +(31) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(38) InputIteratorTransformer +(32) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) SortExecTransformer +(33) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(40) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(41) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) Scan parquet +(36) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(43) Filter +(37) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(44) Scan parquet +(38) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(45) Filter +(39) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(46) Project +(40) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(47) BroadcastExchange +(41) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(48) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(49) Scan parquet +(43) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(50) Filter +(44) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(51) BroadcastExchange +(45) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(52) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(53) Project +(47) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(54) HashAggregate +(48) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(55) Exchange +(49) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(50) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(58) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(60) Exchange +(54) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(55) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(62) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt index 48f5a91500b2..5a941687c525 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt @@ -1,83 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (83) +AdaptiveSparkPlan (86) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- TakeOrderedAndProjectExecTransformer (49) - +- ^ RegularHashAggregateExecTransformer (47) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ BroadcastHashJoinExecTransformer Inner (38) - :- ^ ProjectExecTransformer (26) - : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : :- ^ InputIteratorTransformer (6) - : : +- ^ InputAdapter (5) - : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : +- ColumnarBroadcastExchange (3) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (24) - : :- ^ Scan parquet (7) - : +- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ BroadcastQueryStage (21), Statistics(X) - : +- ColumnarBroadcastExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ FilterExecTransformer (17) - : +- ^ RegularHashAggregateExecTransformer (16) - : +- ^ InputIteratorTransformer (15) - : +- ^ InputAdapter (14) - : +- ^ ShuffleQueryStage (13), Statistics(X) - : +- ColumnarExchange (12) - : +- ^ ProjectExecTransformer (10) - : +- ^ FlushableHashAggregateExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ BroadcastQueryStage (35), Statistics(X) - +- ColumnarBroadcastExchange (34) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (32) - :- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ BroadcastQueryStage (29), Statistics(X) - +- ReusedExchange (28) + VeloxColumnarToRowExec (53) + +- TakeOrderedAndProjectExecTransformer (52) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23), Statistics(X) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ ShuffleQueryStage (15), Statistics(X) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + :- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ BroadcastQueryStage (32), Statistics(X) + +- ReusedExchange (31) +- == Initial Plan == - TakeOrderedAndProject (82) - +- HashAggregate (81) - +- Exchange (80) - +- HashAggregate (79) - +- Project (78) - +- BroadcastHashJoin Inner BuildRight (77) - :- Project (65) - : +- BroadcastHashJoin Inner BuildLeft (64) - : :- BroadcastExchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- BroadcastHashJoin LeftSemi BuildRight (63) - : :- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastExchange (62) - : +- Project (61) - : +- Filter (60) - : +- HashAggregate (59) - : +- Exchange (58) - : +- HashAggregate (57) - : +- Scan parquet (56) - +- BroadcastExchange (76) - +- BroadcastHashJoin LeftSemi BuildRight (75) - :- Filter (67) - : +- Scan parquet (66) - +- BroadcastExchange (74) - +- Project (73) - +- Filter (72) - +- HashAggregate (71) - +- Exchange (70) - +- HashAggregate (69) - +- Scan parquet (68) + TakeOrderedAndProject (85) + +- HashAggregate (84) + +- Exchange (83) + +- HashAggregate (82) + +- Project (81) + +- BroadcastHashJoin Inner BuildRight (80) + :- Project (68) + : +- BroadcastHashJoin Inner BuildLeft (67) + : :- BroadcastExchange (56) + : : +- Filter (55) + : : +- Scan parquet (54) + : +- BroadcastHashJoin LeftSemi BuildRight (66) + : :- Filter (58) + : : +- Scan parquet (57) + : +- BroadcastExchange (65) + : +- Project (64) + : +- Filter (63) + : +- HashAggregate (62) + : +- Exchange (61) + : +- HashAggregate (60) + : +- Scan parquet (59) + +- BroadcastExchange (79) + +- BroadcastHashJoin LeftSemi BuildRight (78) + :- Filter (70) + : +- Scan parquet (69) + +- BroadcastExchange (77) + +- Project (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Scan parquet (71) (1) Scan parquet @@ -87,375 +90,387 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X, c_name#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_name#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(8) Scan parquet +(9) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(10) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(9) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(11) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(12) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(16) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(17) FilterExecTransformer +(19) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(20) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastQueryStage +(23) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [1]: [l_orderkey#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [1]: [l_orderkey#X] -(24) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(25) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(26) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(27) Scan parquet +(29) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(28) ReusedExchange [Reuses operator id: 20] +(30) NoopFilter +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X, l_quantity#X] + +(31) ReusedExchange [Reuses operator id: 22] Output [1]: [l_orderkey#X] -(29) BroadcastQueryStage +(32) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [1]: [l_orderkey#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [1]: [l_orderkey#X] -(32) BroadcastHashJoinExecTransformer +(35) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(34) ColumnarBroadcastExchange +(37) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(38) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(36) InputAdapter +(39) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(37) InputIteratorTransformer +(40) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(38) BroadcastHashJoinExecTransformer +(41) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(39) ProjectExecTransformer +(42) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(40) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(41) ProjectExecTransformer +(44) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(42) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(43) ColumnarExchange +(46) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(47) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(45) InputAdapter +(48) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(46) InputIteratorTransformer +(49) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(47) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(48) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(49) TakeOrderedAndProjectExecTransformer +(52) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(50) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(51) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(53) BroadcastExchange +(56) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(54) Scan parquet +(57) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(55) Filter +(58) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(56) Scan parquet +(59) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(57) HashAggregate +(60) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(58) Exchange +(61) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(62) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(60) Filter +(63) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(61) Project +(64) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(62) BroadcastExchange +(65) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(64) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(65) Project +(68) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(66) Scan parquet +(69) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(67) Filter +(70) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(68) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(69) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(70) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(72) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(73) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(74) BroadcastExchange +(77) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(75) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) BroadcastExchange +(79) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(81) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(79) HashAggregate +(82) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(80) Exchange +(83) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(84) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(82) TakeOrderedAndProject +(85) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(83) AdaptiveSparkPlan +(86) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt index 603998ea792a..244f1c6ffd89 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt @@ -1,34 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (31) +AdaptiveSparkPlan (33) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (30) - +- Exchange (29) - +- HashAggregate (28) - +- Project (27) - +- BroadcastHashJoin Inner BuildRight (26) - :- Project (22) - : +- Filter (21) - : +- Scan parquet (20) - +- BroadcastExchange (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (32) + +- Exchange (31) + +- HashAggregate (30) + +- Project (29) + +- BroadcastHashJoin Inner BuildRight (28) + :- Project (24) + : +- Filter (23) + : +- Scan parquet (22) + +- BroadcastExchange (27) + +- Filter (26) + +- Scan parquet (25) (1) Scan parquet @@ -38,140 +40,148 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(3) ProjectExecTransformer Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(3) Scan parquet +(4) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(5) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(6) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [1]: [revenue#X] -(20) Scan parquet +(22) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(21) Filter +(23) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(22) Project +(24) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(23) Scan parquet +(25) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(24) Filter +(26) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(25) BroadcastExchange +(27) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(26) BroadcastHashJoin +(28) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(27) Project +(29) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(28) HashAggregate +(30) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(29) Exchange +(31) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(30) HashAggregate +(32) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(31) AdaptiveSparkPlan +(33) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt index c0c669bbce45..fd6deaabc6f5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt @@ -1,101 +1,103 @@ == Physical Plan == -AdaptiveSparkPlan (102) +AdaptiveSparkPlan (104) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- AQEShuffleRead (64) - +- ShuffleQueryStage (63), Statistics(X) - +- ColumnarExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ BroadcastHashJoinExecTransformer Inner (59) - :- ^ ProjectExecTransformer (51) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (50) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ AQEShuffleRead (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ BroadcastQueryStage (47), Statistics(X) - : +- ColumnarBroadcastExchange (46) - : +- AQEShuffleRead (45) - : +- ShuffleQueryStage (44), Statistics(X) - : +- ColumnarExchange (43) - : +- ^ ProjectExecTransformer (41) - : +- ^ BroadcastHashJoinExecTransformer Inner (40) - : :- ^ InputIteratorTransformer (22) - : : +- ^ InputAdapter (21) - : : +- ^ BroadcastQueryStage (20), Statistics(X) - : : +- ColumnarBroadcastExchange (19) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (17) - : : :- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : +- ColumnarBroadcastExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ Scan parquet (10) - : +- ^ FilterExecTransformer (39) - : +- ^ ProjectExecTransformer (38) - : +- ^ RegularHashAggregateExecTransformer (37) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ ShuffleQueryStage (34), Statistics(X) - : +- ColumnarExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FlushableHashAggregateExecTransformer (30) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (29) - : :- ^ ProjectExecTransformer (24) - : : +- ^ Scan parquet (23) - : +- ^ InputIteratorTransformer (28) - : +- ^ InputAdapter (27) - : +- ^ BroadcastQueryStage (26), Statistics(X) - : +- ReusedExchange (25) - +- ^ InputIteratorTransformer (58) - +- ^ InputAdapter (57) - +- ^ BroadcastQueryStage (56), Statistics(X) - +- ColumnarBroadcastExchange (55) - +- ^ ProjectExecTransformer (53) - +- ^ Scan parquet (52) + VeloxColumnarToRowExec (67) + +- AQEShuffleRead (66) + +- ShuffleQueryStage (65), Statistics(X) + +- ColumnarExchange (64) + +- ^ ProjectExecTransformer (62) + +- ^ BroadcastHashJoinExecTransformer Inner (61) + :- ^ ProjectExecTransformer (52) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (51) + : :- ^ InputIteratorTransformer (9) + : : +- ^ InputAdapter (8) + : : +- ^ AQEShuffleRead (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (50) + : +- ^ InputAdapter (49) + : +- ^ BroadcastQueryStage (48), Statistics(X) + : +- ColumnarBroadcastExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer Inner (44) + : :- ^ InputIteratorTransformer (25) + : : +- ^ InputAdapter (24) + : : +- ^ BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (20) + : : :- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (19) + : : +- ^ InputAdapter (18) + : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : +- ColumnarBroadcastExchange (16) + : : +- ^ ProjectExecTransformer (14) + : : +- ^ NoopFilter (13) + : : +- ^ Scan parquet (12) + : +- ^ FilterExecTransformer (43) + : +- ^ ProjectExecTransformer (42) + : +- ^ RegularHashAggregateExecTransformer (41) + : +- ^ InputIteratorTransformer (40) + : +- ^ InputAdapter (39) + : +- ^ ShuffleQueryStage (38), Statistics(X) + : +- ColumnarExchange (37) + : +- ^ ProjectExecTransformer (35) + : +- ^ FlushableHashAggregateExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (33) + : :- ^ ProjectExecTransformer (28) + : : +- ^ NoopFilter (27) + : : +- ^ Scan parquet (26) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ BroadcastQueryStage (30), Statistics(X) + : +- ReusedExchange (29) + +- ^ InputIteratorTransformer (60) + +- ^ InputAdapter (59) + +- ^ BroadcastQueryStage (58), Statistics(X) + +- ColumnarBroadcastExchange (57) + +- ^ ProjectExecTransformer (55) + +- ^ NoopFilter (54) + +- ^ Scan parquet (53) +- == Initial Plan == - Sort (101) - +- Exchange (100) - +- Project (99) - +- BroadcastHashJoin Inner BuildRight (98) - :- Project (93) - : +- ShuffledHashJoin LeftSemi BuildRight (92) - : :- Exchange (68) - : : +- Filter (67) - : : +- Scan parquet (66) - : +- Exchange (91) - : +- Project (90) - : +- BroadcastHashJoin Inner BuildLeft (89) - : :- BroadcastExchange (76) - : : +- BroadcastHashJoin LeftSemi BuildRight (75) - : : :- Filter (70) - : : : +- Scan parquet (69) - : : +- BroadcastExchange (74) - : : +- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Filter (88) - : +- HashAggregate (87) - : +- Exchange (86) - : +- HashAggregate (85) - : +- BroadcastHashJoin LeftSemi BuildRight (84) - : :- Project (79) - : : +- Filter (78) - : : +- Scan parquet (77) - : +- BroadcastExchange (83) - : +- Project (82) - : +- Filter (81) - : +- Scan parquet (80) - +- BroadcastExchange (97) - +- Project (96) - +- Filter (95) - +- Scan parquet (94) + Sort (103) + +- Exchange (102) + +- Project (101) + +- BroadcastHashJoin Inner BuildRight (100) + :- Project (95) + : +- ShuffledHashJoin LeftSemi BuildRight (94) + : :- Exchange (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- Exchange (93) + : +- Project (92) + : +- BroadcastHashJoin Inner BuildLeft (91) + : :- BroadcastExchange (78) + : : +- BroadcastHashJoin LeftSemi BuildRight (77) + : : :- Filter (72) + : : : +- Scan parquet (71) + : : +- BroadcastExchange (76) + : : +- Project (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Filter (90) + : +- HashAggregate (89) + : +- Exchange (88) + : +- HashAggregate (87) + : +- BroadcastHashJoin LeftSemi BuildRight (86) + : :- Project (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (85) + : +- Project (84) + : +- Filter (83) + : +- Scan parquet (82) + +- BroadcastExchange (99) + +- Project (98) + +- Filter (97) + +- Scan parquet (96) (1) Scan parquet @@ -105,442 +107,450 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(6) AQEShuffleRead +(7) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) Scan parquet +(11) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(12) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(11) ProjectExecTransformer +(13) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(14) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(12) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(19) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(20) BroadcastQueryStage +(23) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(21) InputAdapter +(24) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(22) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(23) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(24) ProjectExecTransformer +(27) NoopFilter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(28) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(25) ReusedExchange [Reuses operator id: 13] +(29) ReusedExchange [Reuses operator id: 16] Output [1]: [p_partkey#X] -(26) BroadcastQueryStage +(30) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(27) InputAdapter +(31) InputAdapter Input [1]: [p_partkey#X] -(28) InputIteratorTransformer +(32) InputIteratorTransformer Input [1]: [p_partkey#X] -(29) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) FlushableHashAggregateExecTransformer +(34) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(31) ProjectExecTransformer +(35) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(32) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(33) ColumnarExchange +(37) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(38) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(35) InputAdapter +(39) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) InputIteratorTransformer +(40) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(37) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(38) ProjectExecTransformer +(42) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(39) FilterExecTransformer +(43) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(40) BroadcastHashJoinExecTransformer +(44) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(41) ProjectExecTransformer -Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(42) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, ps_suppkey#X] -Arguments: false - -(43) ColumnarExchange -Input [2]: [hash_partition_key#X, ps_suppkey#X] -Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] - -(44) ShuffleQueryStage +(45) ProjectExecTransformer Output [1]: [ps_suppkey#X] -Arguments: X +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(45) AQEShuffleRead +(46) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] -Arguments: local +Arguments: false -(46) ColumnarBroadcastExchange +(47) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastQueryStage +(48) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(48) InputAdapter +(49) InputAdapter Input [1]: [ps_suppkey#X] -(49) InputIteratorTransformer +(50) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(50) BroadcastHashJoinExecTransformer +(51) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(51) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(52) Scan parquet +(53) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(53) ProjectExecTransformer +(54) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(55) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(54) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(55) ColumnarBroadcastExchange +(57) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastQueryStage +(58) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(57) InputAdapter +(59) InputAdapter Input [1]: [n_nationkey#X] -(58) InputIteratorTransformer +(60) InputIteratorTransformer Input [1]: [n_nationkey#X] -(59) BroadcastHashJoinExecTransformer +(61) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(60) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(61) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(62) ColumnarExchange +(64) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(63) ShuffleQueryStage +(65) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(64) AQEShuffleRead +(66) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(65) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(66) Scan parquet +(68) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(67) Filter +(69) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(68) Exchange +(70) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(71) Scan parquet +(73) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(72) Filter +(74) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(73) Project +(75) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(74) BroadcastExchange +(76) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(75) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(76) BroadcastExchange +(78) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(77) Scan parquet +(79) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(78) Filter +(80) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(79) Project +(81) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(80) Scan parquet +(82) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(81) Filter +(83) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(82) Project +(84) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(83) BroadcastExchange +(85) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(84) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(85) HashAggregate +(87) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(86) Exchange +(88) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) HashAggregate +(89) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(88) Filter +(90) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(89) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(90) Project +(92) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(91) Exchange +(93) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(94) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(93) Project +(95) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(94) Scan parquet +(96) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(95) Filter +(97) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(96) Project +(98) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(97) BroadcastExchange +(99) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(98) BroadcastHashJoin +(100) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(99) Project +(101) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(100) Exchange +(102) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Sort +(103) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(102) AdaptiveSparkPlan +(104) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt index f8a97759d567..ac047fdbcabf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt @@ -1,85 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (91) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48), Statistics(X) - +- ColumnarExchange (47) - +- ^ ProjectExecTransformer (45) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ BroadcastHashJoinExecTransformer Inner (42) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (25) - : : +- ^ BroadcastHashJoinExecTransformer Inner (24) - : : :- ^ InputIteratorTransformer (6) - : : : +- ^ InputAdapter (5) - : : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : : +- ColumnarBroadcastExchange (3) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (23) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (15) - : : : :- ^ ProjectExecTransformer (8) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (14) - : : : +- ^ InputAdapter (13) - : : : +- ^ BroadcastQueryStage (12), Statistics(X) - : : : +- ColumnarBroadcastExchange (11) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (22) - : : +- ^ InputAdapter (21) - : : +- ^ BroadcastQueryStage (20), Statistics(X) - : : +- ColumnarBroadcastExchange (19) - : : +- ^ ProjectExecTransformer (17) - : : +- ^ Scan parquet (16) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ColumnarBroadcastExchange (29) - : +- ^ ProjectExecTransformer (27) - : +- ^ Scan parquet (26) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ BroadcastQueryStage (39), Statistics(X) - +- ColumnarBroadcastExchange (38) - +- ^ ProjectExecTransformer (36) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- ^ InputAdapter (24) + : : +- ^ BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (75) - : +- BroadcastHashJoin Inner BuildRight (74) - : :- Project (69) - : : +- BroadcastHashJoin Inner BuildLeft (68) - : : :- BroadcastExchange (56) - : : : +- Filter (55) - : : : +- Scan parquet (54) - : : +- BroadcastHashJoin LeftAnti BuildRight (67) - : : :- BroadcastHashJoin LeftSemi BuildRight (62) - : : : :- Project (59) - : : : : +- Filter (58) - : : : : +- Scan parquet (57) - : : : +- BroadcastExchange (61) - : : : +- Scan parquet (60) - : : +- BroadcastExchange (66) - : : +- Project (65) - : : +- Filter (64) - : : +- Scan parquet (63) - : +- BroadcastExchange (73) - : +- Project (72) - : +- Filter (71) - : +- Scan parquet (70) - +- BroadcastExchange (79) - +- Project (78) - +- Filter (77) - +- Scan parquet (76) + TakeOrderedAndProject (90) + +- HashAggregate (89) + +- Exchange (88) + +- HashAggregate (87) + +- Project (86) + +- BroadcastHashJoin Inner BuildRight (85) + :- Project (80) + : +- BroadcastHashJoin Inner BuildRight (79) + : :- Project (74) + : : +- BroadcastHashJoin Inner BuildLeft (73) + : : :- BroadcastExchange (61) + : : : +- Filter (60) + : : : +- Scan parquet (59) + : : +- BroadcastHashJoin LeftAnti BuildRight (72) + : : :- BroadcastHashJoin LeftSemi BuildRight (67) + : : : :- Project (64) + : : : : +- Filter (63) + : : : : +- Scan parquet (62) + : : : +- BroadcastExchange (66) + : : : +- Scan parquet (65) + : : +- BroadcastExchange (71) + : : +- Project (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- BroadcastExchange (78) + : +- Project (77) + : +- Filter (76) + : +- Scan parquet (75) + +- BroadcastExchange (84) + +- Project (83) + +- Filter (82) + +- Scan parquet (81) (1) Scan parquet @@ -89,382 +94,402 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(10) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(9) Scan parquet +(11) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(11) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(12) BroadcastQueryStage +(14) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(15) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(16) Scan parquet +(18) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(17) ProjectExecTransformer +(19) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(20) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(18) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(19) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(20) BroadcastQueryStage +(23) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(21) InputAdapter +(24) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(22) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(23) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(24) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(25) ProjectExecTransformer +(28) ProjectExecTransformer Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(26) Scan parquet +(29) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(27) ProjectExecTransformer +(30) NoopFilter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: [o_orderkey#X, o_orderstatus#X] + +(31) ProjectExecTransformer Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(28) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [1]: [o_orderkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(33) ColumnarBroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(30) BroadcastQueryStage +(34) BroadcastQueryStage Output [1]: [o_orderkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [1]: [o_orderkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [1]: [o_orderkey#X] -(33) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(35) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(36) ProjectExecTransformer +(40) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(41) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(37) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(38) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) BroadcastQueryStage +(44) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(40) InputAdapter +(45) InputAdapter Input [1]: [n_nationkey#X] -(41) InputIteratorTransformer +(46) InputIteratorTransformer Input [1]: [n_nationkey#X] -(42) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(43) ProjectExecTransformer +(48) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(44) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(45) ProjectExecTransformer +(50) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(46) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(47) ColumnarExchange +(52) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(49) InputAdapter +(54) InputAdapter Input [2]: [s_name#X, count#X] -(50) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(51) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(53) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(54) Scan parquet +(59) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) BroadcastExchange +(61) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(62) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(58) Filter +(63) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(59) Project +(64) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(60) Scan parquet +(65) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(61) BroadcastExchange +(66) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(62) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(63) Scan parquet +(68) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(64) Filter +(69) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(65) Project +(70) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(66) BroadcastExchange +(71) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(69) Project +(74) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(70) Scan parquet +(75) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(76) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(72) Project +(77) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(73) BroadcastExchange +(78) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(74) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(75) Project +(80) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(76) Scan parquet +(81) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(77) Filter +(82) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(78) Project +(83) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(79) BroadcastExchange +(84) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(85) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(81) Project +(86) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(82) HashAggregate +(87) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(83) Exchange +(88) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(89) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(85) TakeOrderedAndProject +(90) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(86) AdaptiveSparkPlan +(91) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt index 64084ebda520..3306ea6b2fd9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt @@ -1,39 +1,40 @@ == Physical Plan == -AdaptiveSparkPlan (37) +AdaptiveSparkPlan (38) +- == Final Plan == - VeloxColumnarToRowExec (25) - +- ^ SortExecTransformer (23) - +- ^ InputIteratorTransformer (22) - +- ^ InputAdapter (21) - +- ^ ShuffleQueryStage (20), Statistics(X) - +- ColumnarExchange (19) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FlushableHashAggregateExecTransformer (10) - +- ^ ProjectExecTransformer (9) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (8) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (7) - +- ^ InputAdapter (6) - +- ^ BroadcastQueryStage (5), Statistics(X) - +- ColumnarBroadcastExchange (4) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (26) + +- ^ SortExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (36) - +- Exchange (35) - +- HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin LeftAnti BuildRight (30) - :- Filter (27) - : +- Scan parquet (26) - +- BroadcastExchange (29) - +- Scan parquet (28) + Sort (37) + +- Exchange (36) + +- HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin LeftAnti BuildRight (31) + :- Filter (28) + : +- Scan parquet (27) + +- BroadcastExchange (30) + +- Scan parquet (29) (1) Scan parquet @@ -43,270 +44,279 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X, c_phone#X, c_acctbal#X] + +(3) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [o_custkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [o_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [o_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [o_custkey#X] -(8) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(9) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(10) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(14) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(19) ColumnarExchange +(20) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(20) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(21) InputAdapter +(22) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(22) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) SortExecTransformer +(24) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(24) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(25) VeloxColumnarToRowExec +(26) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(26) Scan parquet +(27) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(27) Filter +(28) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(28) Scan parquet +(29) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(29) BroadcastExchange +(30) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(31) Project +(32) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(32) HashAggregate +(33) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(35) Exchange +(36) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Sort +(37) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(37) AdaptiveSparkPlan +(38) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (48) - +- ^ RegularHashAggregateExecTransformer (46) - +- ^ InputIteratorTransformer (45) - +- ^ InputAdapter (44) - +- ^ ShuffleQueryStage (43), Statistics(X) - +- ColumnarExchange (42) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ Scan parquet (38) + VeloxColumnarToRowExec (50) + +- ^ RegularHashAggregateExecTransformer (48) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FlushableHashAggregateExecTransformer (42) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- Filter (50) - +- Scan parquet (49) + HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- Filter (52) + +- Scan parquet (51) -(38) Scan parquet +(39) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(39) ProjectExecTransformer +(40) NoopFilter +Input [2]: [c_phone#X, c_acctbal#X] +Arguments: [c_phone#X, c_acctbal#X] + +(41) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(40) FlushableHashAggregateExecTransformer +(42) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(41) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(42) ColumnarExchange +(44) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(43) ShuffleQueryStage +(45) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(44) InputAdapter +(46) InputAdapter Input [2]: [sum#X, count#X] -(45) InputIteratorTransformer +(47) InputIteratorTransformer Input [2]: [sum#X, count#X] -(46) RegularHashAggregateExecTransformer +(48) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(47) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(48) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(49) Scan parquet +(51) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(50) Filter +(52) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(51) Project +(53) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(52) HashAggregate +(54) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(53) Exchange +(55) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(55) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt index 8b97940da5b5..0bcfb2c8228d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt @@ -1,52 +1,55 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (31) - +- TakeOrderedAndProjectExecTransformer (30) - +- ^ ProjectExecTransformer (28) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ BroadcastHashJoinExecTransformer Inner (18) - :- ^ ProjectExecTransformer (10) - : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ BroadcastQueryStage (15), Statistics(X) - +- ColumnarBroadcastExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ Scan parquet (11) + VeloxColumnarToRowExec (34) + +- TakeOrderedAndProjectExecTransformer (33) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- Project (45) - +- BroadcastHashJoin Inner BuildRight (44) - :- Project (39) - : +- BroadcastHashJoin Inner BuildLeft (38) - : :- BroadcastExchange (35) - : : +- Project (34) - : : +- Filter (33) - : : +- Scan parquet (32) - : +- Filter (37) - : +- Scan parquet (36) - +- BroadcastExchange (43) - +- Project (42) - +- Filter (41) - +- Scan parquet (40) + TakeOrderedAndProject (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- Project (42) + : +- BroadcastHashJoin Inner BuildLeft (41) + : :- BroadcastExchange (38) + : : +- Project (37) + : : +- Filter (36) + : : +- Scan parquet (35) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -56,222 +59,234 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: [c_custkey#X, c_mktsegment#X] + +(3) ProjectExecTransformer Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [c_custkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(12) ProjectExecTransformer +(14) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(15) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(13) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(15) BroadcastQueryStage +(18) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(19) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(18) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(21) ProjectExecTransformer +(24) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, o_orderdate#X, o_shippriority#X, 42) AS hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(22) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(26) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(28) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(29) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(28) ProjectExecTransformer +(31) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(30) TakeOrderedAndProjectExecTransformer +(33) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(31) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(32) Scan parquet +(35) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(33) Filter +(36) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(34) Project +(37) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(35) BroadcastExchange +(38) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(36) Scan parquet +(39) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(38) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(39) Project +(42) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(43) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(41) Filter +(44) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(42) Project +(45) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(43) BroadcastExchange +(46) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(44) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(45) Project +(48) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(46) HashAggregate +(49) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(47) Exchange +(50) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(51) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(49) TakeOrderedAndProject +(52) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt index 31cba5c73b0a..b547b4051bcf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt @@ -1,44 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (10) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- ^ InputAdapter (10) + +- ^ BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin LeftSemi BuildRight (35) - :- Project (30) - : +- Filter (29) - : +- Scan parquet (28) - +- BroadcastExchange (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin LeftSemi BuildRight (37) + :- Project (32) + : +- Filter (31) + : +- Scan parquet (30) + +- BroadcastExchange (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -48,182 +50,190 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(3) Scan parquet +(4) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(4) ProjectExecTransformer +(5) NoopFilter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(6) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(5) WholeStageCodegenTransformer (X) +(7) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(8) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(7) BroadcastQueryStage +(9) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(8) InputAdapter +(10) InputAdapter Input [1]: [l_orderkey#X] -(9) InputIteratorTransformer +(11) InputIteratorTransformer Input [1]: [l_orderkey#X] -(10) BroadcastHashJoinExecTransformer +(12) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(11) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(12) FlushableHashAggregateExecTransformer +(14) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(13) ProjectExecTransformer +(15) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(14) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(15) ColumnarExchange +(17) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(18) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(17) InputAdapter +(19) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(18) InputIteratorTransformer +(20) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(25) SortExecTransformer +(27) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(28) Scan parquet +(30) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(29) Filter +(31) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(30) Project +(32) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(31) Scan parquet +(33) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(32) Filter +(34) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(33) Project +(35) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(34) BroadcastExchange +(36) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(38) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(37) HashAggregate +(39) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(38) Exchange +(40) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(40) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt index dfdfede045a2..a87ef7c618c0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt @@ -1,92 +1,98 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- ^ SortExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48), Statistics(X) - +- ColumnarExchange (47) - +- ^ ProjectExecTransformer (45) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ BroadcastHashJoinExecTransformer Inner (42) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (26) - : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : :- ^ ProjectExecTransformer (18) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : :- ^ InputIteratorTransformer (6) - : : : : : +- ^ InputAdapter (5) - : : : : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (3) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (8) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ BroadcastQueryStage (22), Statistics(X) - : : +- ColumnarBroadcastExchange (21) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ColumnarBroadcastExchange (29) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ BroadcastQueryStage (39), Statistics(X) - +- ColumnarBroadcastExchange (38) - +- ^ ProjectExecTransformer (36) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ^ InputAdapter (61) + +- ^ ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ^ InputAdapter (55) + +- ^ ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- ^ InputAdapter (18) + : : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- ^ InputAdapter (36) + : +- ^ BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (93) - +- Exchange (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (62) - : : : : : +- Filter (61) - : : : : : +- Scan parquet (60) - : : : : +- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (86) - +- Project (85) - +- Filter (84) - +- Scan parquet (83) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Project (71) + : : : : +- Filter (70) + : : : : +- Scan parquet (69) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (92) + +- Project (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -96,414 +102,438 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(10) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(15) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) Scan parquet +(22) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(24) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(27) Scan parquet +(31) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] + +(33) WholeStageCodegenTransformer (X) Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(36) ProjectExecTransformer +(41) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(42) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(37) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(38) ColumnarBroadcastExchange +(44) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) BroadcastQueryStage +(45) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(40) InputAdapter +(46) InputAdapter Input [1]: [r_regionkey#X] -(41) InputIteratorTransformer +(47) InputIteratorTransformer Input [1]: [r_regionkey#X] -(42) BroadcastHashJoinExecTransformer +(48) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(43) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(44) FlushableHashAggregateExecTransformer +(50) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(45) ProjectExecTransformer +(51) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(46) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(47) ColumnarExchange +(53) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(49) InputAdapter +(55) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(50) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(51) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(52) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(53) ColumnarExchange +(59) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(55) InputAdapter +(61) InputAdapter Input [2]: [n_name#X, revenue#X] -(56) InputIteratorTransformer +(62) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(57) SortExecTransformer +(63) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(60) Scan parquet +(66) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(61) Filter +(67) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(62) BroadcastExchange +(68) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(69) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(64) Filter +(70) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(65) Project +(71) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(66) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(67) Project +(73) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(68) Scan parquet +(74) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(69) Filter +(75) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(70) BroadcastExchange +(76) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(72) Project +(78) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(79) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(80) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(81) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(77) Project +(83) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(84) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(80) BroadcastExchange +(86) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(88) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(83) Scan parquet +(89) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(84) Filter +(90) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(85) Project +(91) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(86) BroadcastExchange +(92) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(87) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(88) Project +(94) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(89) HashAggregate +(95) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(90) Exchange +(96) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(97) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(92) Exchange +(98) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(99) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(94) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt index 2e51f22a6b0e..51c5836bdd11 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (18) +AdaptiveSparkPlan (19) +- == Final Plan == - VeloxColumnarToRowExec (11) - +- ^ RegularHashAggregateExecTransformer (9) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ ShuffleQueryStage (6), Statistics(X) - +- ColumnarExchange (5) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (17) - +- Exchange (16) - +- HashAggregate (15) - +- Project (14) - +- Filter (13) - +- Scan parquet (12) + HashAggregate (18) + +- Exchange (17) + +- HashAggregate (16) + +- Project (15) + +- Filter (14) + +- Scan parquet (13) (1) Scan parquet @@ -26,82 +27,86 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(5) ColumnarExchange +(6) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [sum#X, isEmpty#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(9) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(11) VeloxColumnarToRowExec +(12) VeloxColumnarToRowExec Input [1]: [revenue#X] -(12) Scan parquet +(13) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(13) Filter +(14) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(14) Project +(15) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) HashAggregate +(16) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(16) Exchange +(17) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(17) HashAggregate +(18) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(18) AdaptiveSparkPlan +(19) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt index 0a8c1111c851..37e9e5214256 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt @@ -1,87 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (88) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ SortExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ^ InputAdapter (51) - +- ^ ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ RegularHashAggregateExecTransformer (47) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ BroadcastHashJoinExecTransformer Inner (38) - :- ^ ProjectExecTransformer (33) - : +- ^ BroadcastHashJoinExecTransformer Inner (32) - : :- ^ ProjectExecTransformer (25) - : : +- ^ BroadcastHashJoinExecTransformer Inner (24) - : : :- ^ ProjectExecTransformer (17) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (16) - : : : :- ^ ProjectExecTransformer (9) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (8) - : : : : :- ^ InputIteratorTransformer (6) - : : : : : +- ^ InputAdapter (5) - : : : : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (3) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (15) - : : : +- ^ InputAdapter (14) - : : : +- ^ BroadcastQueryStage (13), Statistics(X) - : : : +- ColumnarBroadcastExchange (12) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ BroadcastQueryStage (21), Statistics(X) - : : +- ColumnarBroadcastExchange (20) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ BroadcastQueryStage (29), Statistics(X) - : +- ColumnarBroadcastExchange (28) - : +- ^ Scan parquet (26) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ BroadcastQueryStage (35), Statistics(X) - +- ReusedExchange (34) + VeloxColumnarToRowExec (60) + +- ^ SortExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ^ InputAdapter (56) + +- ^ ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ^ InputAdapter (50) + +- ^ ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- ^ InputAdapter (17) + : : : +- ^ BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- ^ InputAdapter (26) + : : +- ^ BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- ^ InputAdapter (41) + +- ^ BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (87) - +- Exchange (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- BroadcastHashJoin Inner BuildRight (81) - :- Project (77) - : +- BroadcastHashJoin Inner BuildRight (76) - : :- Project (72) - : : +- BroadcastHashJoin Inner BuildRight (71) - : : :- Project (67) - : : : +- BroadcastHashJoin Inner BuildRight (66) - : : : :- Project (62) - : : : : +- BroadcastHashJoin Inner BuildLeft (61) - : : : : :- BroadcastExchange (58) - : : : : : +- Filter (57) - : : : : : +- Scan parquet (56) - : : : : +- Filter (60) - : : : : +- Scan parquet (59) - : : : +- BroadcastExchange (65) - : : : +- Filter (64) - : : : +- Scan parquet (63) - : : +- BroadcastExchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (75) - : +- Filter (74) - : +- Scan parquet (73) - +- BroadcastExchange (80) - +- Filter (79) - +- Scan parquet (78) + Sort (92) + +- Exchange (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (77) + : : +- BroadcastHashJoin Inner BuildRight (76) + : : :- Project (72) + : : : +- BroadcastHashJoin Inner BuildRight (71) + : : : :- Project (67) + : : : : +- BroadcastHashJoin Inner BuildLeft (66) + : : : : :- BroadcastExchange (63) + : : : : : +- Filter (62) + : : : : : +- Scan parquet (61) + : : : : +- Filter (65) + : : : : +- Scan parquet (64) + : : : +- BroadcastExchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- BroadcastExchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (80) + : +- Filter (79) + : +- Scan parquet (78) + +- BroadcastExchange (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -91,386 +96,406 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(8) BroadcastHashJoinExecTransformer +(9) NoopFilter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(10) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(9) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) Scan parquet +(12) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(11) WholeStageCodegenTransformer (X) +(13) NoopFilter +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X, o_custkey#X] + +(14) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarBroadcastExchange +(15) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(16) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) BroadcastHashJoinExecTransformer +(19) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(17) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(18) Scan parquet +(21) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(19) WholeStageCodegenTransformer (X) +(22) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(23) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(20) ColumnarBroadcastExchange +(24) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(21) BroadcastQueryStage +(25) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(22) InputAdapter +(26) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(23) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(24) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(25) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(26) Scan parquet +(30) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(27) WholeStageCodegenTransformer (X) +(31) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(32) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(28) ColumnarBroadcastExchange +(33) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastQueryStage +(34) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(30) InputAdapter +(35) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(31) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(32) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(38) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(34) ReusedExchange [Reuses operator id: 28] +(39) ReusedExchange [Reuses operator id: 33] Output [2]: [n_nationkey#X, n_name#X] -(35) BroadcastQueryStage +(40) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(36) InputAdapter +(41) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(37) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(38) BroadcastHashJoinExecTransformer +(43) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(39) ProjectExecTransformer +(44) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(40) FlushableHashAggregateExecTransformer +(45) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(43) ColumnarExchange +(48) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(49) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(45) InputAdapter +(50) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(46) InputIteratorTransformer +(51) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(47) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(48) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(49) ColumnarExchange +(54) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(55) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(51) InputAdapter +(56) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(52) InputIteratorTransformer +(57) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(53) SortExecTransformer +(58) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(54) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(55) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(56) Scan parquet +(61) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(57) Filter +(62) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(58) BroadcastExchange +(63) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(59) Scan parquet +(64) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(60) Filter +(65) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(61) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(62) Project +(67) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(63) Scan parquet +(68) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(64) Filter +(69) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(65) BroadcastExchange +(70) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(66) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(67) Project +(72) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(68) Scan parquet +(73) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(70) BroadcastExchange +(75) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(72) Project +(77) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(73) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(74) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(75) BroadcastExchange +(80) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(77) Project +(82) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(78) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(79) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(80) BroadcastExchange +(85) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(82) Project +(87) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(88) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(84) Exchange +(89) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(90) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(86) Exchange +(91) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Sort +(92) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(88) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt index 6822e2becd51..9b490b9ba436 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt @@ -1,117 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (121) +AdaptiveSparkPlan (129) +- == Final Plan == - VeloxColumnarToRowExec (76) - +- ^ SortExecTransformer (74) - +- ^ InputIteratorTransformer (73) - +- ^ InputAdapter (72) - +- ^ ShuffleQueryStage (71), Statistics(X) - +- ColumnarExchange (70) - +- ^ ProjectExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (67) - +- ^ InputIteratorTransformer (66) - +- ^ InputAdapter (65) - +- ^ ShuffleQueryStage (64), Statistics(X) - +- ColumnarExchange (63) - +- ^ ProjectExecTransformer (61) - +- ^ FlushableHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ BroadcastHashJoinExecTransformer Inner (58) - :- ^ ProjectExecTransformer (50) - : +- ^ BroadcastHashJoinExecTransformer Inner (49) - : :- ^ ProjectExecTransformer (42) - : : +- ^ BroadcastHashJoinExecTransformer Inner (41) - : : :- ^ ProjectExecTransformer (34) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : : : :- ^ ProjectExecTransformer (26) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : : : :- ^ ProjectExecTransformer (18) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : : : :- ^ ProjectExecTransformer (10) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : : : :- ^ InputIteratorTransformer (7) - : : : : : : : +- ^ InputAdapter (6) - : : : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (4) - : : : : : : : +- ^ ProjectExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ Scan parquet (8) - : : : : : +- ^ InputIteratorTransformer (16) - : : : : : +- ^ InputAdapter (15) - : : : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (13) - : : : : : +- ^ Scan parquet (11) - : : : : +- ^ InputIteratorTransformer (24) - : : : : +- ^ InputAdapter (23) - : : : : +- ^ BroadcastQueryStage (22), Statistics(X) - : : : : +- ColumnarBroadcastExchange (21) - : : : : +- ^ Scan parquet (19) - : : : +- ^ InputIteratorTransformer (32) - : : : +- ^ InputAdapter (31) - : : : +- ^ BroadcastQueryStage (30), Statistics(X) - : : : +- ColumnarBroadcastExchange (29) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (40) - : : +- ^ InputAdapter (39) - : : +- ^ BroadcastQueryStage (38), Statistics(X) - : : +- ColumnarBroadcastExchange (37) - : : +- ^ Scan parquet (35) - : +- ^ InputIteratorTransformer (48) - : +- ^ InputAdapter (47) - : +- ^ BroadcastQueryStage (46), Statistics(X) - : +- ColumnarBroadcastExchange (45) - : +- ^ Scan parquet (43) - +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ BroadcastQueryStage (55), Statistics(X) - +- ColumnarBroadcastExchange (54) - +- ^ ProjectExecTransformer (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (84) + +- ^ SortExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ^ InputAdapter (80) + +- ^ ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ^ InputAdapter (73) + +- ^ ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ^ InputAdapter (7) + : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- ^ InputAdapter (18) + : : : : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- ^ InputAdapter (27) + : : : : +- ^ BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- ^ InputAdapter (36) + : : : +- ^ BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- ^ InputAdapter (54) + : +- ^ BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- ^ InputAdapter (64) + +- ^ BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (120) - +- Exchange (119) - +- HashAggregate (118) - +- Exchange (117) - +- HashAggregate (116) - +- Project (115) - +- BroadcastHashJoin Inner BuildRight (114) - :- Project (109) - : +- BroadcastHashJoin Inner BuildRight (108) - : :- Project (104) - : : +- BroadcastHashJoin Inner BuildRight (103) - : : :- Project (99) - : : : +- BroadcastHashJoin Inner BuildRight (98) - : : : :- Project (94) - : : : : +- BroadcastHashJoin Inner BuildRight (93) - : : : : :- Project (89) - : : : : : +- BroadcastHashJoin Inner BuildRight (88) - : : : : : :- Project (84) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (83) - : : : : : : :- BroadcastExchange (80) - : : : : : : : +- Project (79) - : : : : : : : +- Filter (78) - : : : : : : : +- Scan parquet (77) - : : : : : : +- Filter (82) - : : : : : : +- Scan parquet (81) - : : : : : +- BroadcastExchange (87) - : : : : : +- Filter (86) - : : : : : +- Scan parquet (85) - : : : : +- BroadcastExchange (92) - : : : : +- Filter (91) - : : : : +- Scan parquet (90) - : : : +- BroadcastExchange (97) - : : : +- Filter (96) - : : : +- Scan parquet (95) - : : +- BroadcastExchange (102) - : : +- Filter (101) - : : +- Scan parquet (100) - : +- BroadcastExchange (107) - : +- Filter (106) - : +- Scan parquet (105) - +- BroadcastExchange (113) - +- Project (112) - +- Filter (111) - +- Scan parquet (110) + Sort (128) + +- Exchange (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- BroadcastHashJoin Inner BuildRight (122) + :- Project (117) + : +- BroadcastHashJoin Inner BuildRight (116) + : :- Project (112) + : : +- BroadcastHashJoin Inner BuildRight (111) + : : :- Project (107) + : : : +- BroadcastHashJoin Inner BuildRight (106) + : : : :- Project (102) + : : : : +- BroadcastHashJoin Inner BuildRight (101) + : : : : :- Project (97) + : : : : : +- BroadcastHashJoin Inner BuildRight (96) + : : : : : :- Project (92) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) + : : : : : : :- BroadcastExchange (88) + : : : : : : : +- Project (87) + : : : : : : : +- Filter (86) + : : : : : : : +- Scan parquet (85) + : : : : : : +- Filter (90) + : : : : : : +- Scan parquet (89) + : : : : : +- BroadcastExchange (95) + : : : : : +- Filter (94) + : : : : : +- Scan parquet (93) + : : : : +- BroadcastExchange (100) + : : : : +- Filter (99) + : : : : +- Scan parquet (98) + : : : +- BroadcastExchange (105) + : : : +- Filter (104) + : : : +- Scan parquet (103) + : : +- BroadcastExchange (110) + : : +- Filter (109) + : : +- Scan parquet (108) + : +- BroadcastExchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- BroadcastExchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -121,534 +129,566 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(3) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) Scan parquet +(13) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(15) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(19) Scan parquet +(22) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(24) WholeStageCodegenTransformer (X) Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(27) Scan parquet +(31) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(33) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(36) WholeStageCodegenTransformer (X) +(41) NoopFilter +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_regionkey#X] + +(42) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: false -(37) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(44) BroadcastQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(39) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(40) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(41) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(42) ProjectExecTransformer +(48) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(43) Scan parquet +(49) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(44) WholeStageCodegenTransformer (X) +(50) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(51) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(45) ColumnarBroadcastExchange +(52) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastQueryStage +(53) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(47) InputAdapter +(54) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(48) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastHashJoinExecTransformer +(56) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(50) ProjectExecTransformer +(57) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(51) Scan parquet +(58) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(52) ProjectExecTransformer +(59) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(60) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(53) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(54) ColumnarBroadcastExchange +(62) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(55) BroadcastQueryStage +(63) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(56) InputAdapter +(64) InputAdapter Input [1]: [r_regionkey#X] -(57) InputIteratorTransformer +(65) InputIteratorTransformer Input [1]: [r_regionkey#X] -(58) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(59) ProjectExecTransformer +(67) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(60) FlushableHashAggregateExecTransformer +(68) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(61) ProjectExecTransformer +(69) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(62) WholeStageCodegenTransformer (X) +(70) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(63) ColumnarExchange +(71) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(64) ShuffleQueryStage +(72) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(65) InputAdapter +(73) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(66) InputIteratorTransformer +(74) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(67) RegularHashAggregateExecTransformer +(75) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(68) ProjectExecTransformer +(76) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(69) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(70) ColumnarExchange +(78) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(71) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(72) InputAdapter +(80) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(73) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(74) SortExecTransformer +(82) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(75) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(76) VeloxColumnarToRowExec +(84) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(77) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(78) Filter +(86) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(79) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(80) BroadcastExchange +(88) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(81) Scan parquet +(89) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(82) Filter +(90) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(83) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(84) Project +(92) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(85) Scan parquet +(93) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(94) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(95) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(96) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(89) Project +(97) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(98) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(91) Filter +(99) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(92) BroadcastExchange +(100) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(93) BroadcastHashJoin +(101) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(94) Project +(102) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(95) Scan parquet +(103) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(104) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) BroadcastExchange +(105) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(98) BroadcastHashJoin +(106) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(99) Project +(107) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(100) Scan parquet +(108) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(101) Filter +(109) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(102) BroadcastExchange +(110) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(103) BroadcastHashJoin +(111) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) Project +(112) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(105) Scan parquet +(113) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(106) Filter +(114) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(107) BroadcastExchange +(115) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(108) BroadcastHashJoin +(116) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(109) Project +(117) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(110) Scan parquet +(118) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(111) Filter +(119) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(112) Project +(120) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(113) BroadcastExchange +(121) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(114) BroadcastHashJoin +(122) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(115) Project +(123) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(116) HashAggregate +(124) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(117) Exchange +(125) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) HashAggregate +(126) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(119) Exchange +(127) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Sort +(128) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(121) AdaptiveSparkPlan +(129) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt index bec2b05cc2a3..8523cbeff2ab 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt @@ -1,90 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (98) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ SortExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (26) - : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : :- ^ ProjectExecTransformer (18) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ BroadcastQueryStage (22), Statistics(X) - : : +- ColumnarBroadcastExchange (21) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ColumnarBroadcastExchange (29) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (64) + +- ^ SortExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ^ InputAdapter (60) + +- ^ ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- ^ InputAdapter (18) + : : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- ^ InputAdapter (36) + : +- ^ BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (91) - +- Exchange (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (76) - : : +- BroadcastHashJoin Inner BuildRight (75) - : : :- Project (71) - : : : +- BroadcastHashJoin Inner BuildRight (70) - : : : :- Project (66) - : : : : +- BroadcastHashJoin Inner BuildLeft (65) - : : : : :- BroadcastExchange (62) - : : : : : +- Project (61) - : : : : : +- Filter (60) - : : : : : +- Scan parquet (59) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- BroadcastExchange (74) - : : +- Filter (73) - : : +- Scan parquet (72) - : +- BroadcastExchange (79) - : +- Filter (78) - : +- Scan parquet (77) - +- BroadcastExchange (84) - +- Filter (83) - +- Scan parquet (82) + Sort (97) + +- Exchange (96) + +- HashAggregate (95) + +- Exchange (94) + +- HashAggregate (93) + +- Project (92) + +- BroadcastHashJoin Inner BuildRight (91) + :- Project (87) + : +- BroadcastHashJoin Inner BuildRight (86) + : :- Project (82) + : : +- BroadcastHashJoin Inner BuildRight (81) + : : :- Project (77) + : : : +- BroadcastHashJoin Inner BuildRight (76) + : : : :- Project (72) + : : : : +- BroadcastHashJoin Inner BuildLeft (71) + : : : : :- BroadcastExchange (68) + : : : : : +- Project (67) + : : : : : +- Filter (66) + : : : : : +- Scan parquet (65) + : : : : +- Filter (70) + : : : : +- Scan parquet (69) + : : : +- BroadcastExchange (75) + : : : +- Filter (74) + : : : +- Scan parquet (73) + : : +- BroadcastExchange (80) + : : +- Filter (79) + : : +- Scan parquet (78) + : +- BroadcastExchange (85) + : +- Filter (84) + : +- Scan parquet (83) + +- BroadcastExchange (90) + +- Filter (89) + +- Scan parquet (88) (1) Scan parquet @@ -94,406 +100,430 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(3) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) Scan parquet +(13) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(15) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(19) Scan parquet +(22) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(24) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(27) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_orderdate#X] + +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(36) WholeStageCodegenTransformer (X) +(41) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(42) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(37) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(44) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(39) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(40) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(41) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(42) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(43) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(50) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(52) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(53) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(54) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(55) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(56) SortExecTransformer +(62) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(59) Scan parquet +(65) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(60) Filter +(66) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(61) Project +(67) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(62) BroadcastExchange +(68) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) Scan parquet +(69) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(64) Filter +(70) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(65) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(66) Project +(72) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(67) Scan parquet +(73) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(68) Filter +(74) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(69) BroadcastExchange +(75) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(70) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(71) Project +(77) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(72) Scan parquet +(78) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(73) Filter +(79) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(74) BroadcastExchange +(80) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(75) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(76) Project +(82) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(77) Scan parquet +(83) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(78) Filter +(84) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(79) BroadcastExchange +(85) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(81) Project +(87) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(82) Scan parquet +(88) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(83) Filter +(89) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(84) BroadcastExchange +(90) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(85) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) Project +(92) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(87) HashAggregate +(93) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(88) Exchange +(94) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(95) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(90) Exchange +(96) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Sort +(97) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(92) AdaptiveSparkPlan +(98) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt index 0217849e5de2..41613c36f7bd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (27) +AdaptiveSparkPlan (28) +- == Final Plan == - VeloxColumnarToRowExec (18) - +- ^ SortExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (19) + +- ^ SortExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (26) - +- Exchange (25) - +- HashAggregate (24) - +- Exchange (23) - +- HashAggregate (22) - +- Project (21) - +- Filter (20) - +- Scan parquet (19) + Sort (27) + +- Exchange (26) + +- HashAggregate (25) + +- Exchange (24) + +- HashAggregate (23) + +- Project (22) + +- Filter (21) + +- Scan parquet (20) (1) Scan parquet @@ -34,116 +35,120 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X, ((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(4) ProjectExecTransformer +(5) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(6) ColumnarExchange +(7) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(12) ColumnarExchange +(13) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(14) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) InputAdapter +(15) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(15) InputIteratorTransformer +(16) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) SortExecTransformer +(17) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(17) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(18) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(19) Scan parquet +(20) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(20) Filter +(21) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(21) Project +(22) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(22) HashAggregate +(23) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(23) Exchange +(24) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(24) HashAggregate +(25) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(25) Exchange +(26) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Sort +(27) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(27) AdaptiveSparkPlan +(28) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt index fc3503c42fef..276a885015d0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt @@ -1,64 +1,68 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- ^ ProjectExecTransformer (29) - +- ^ FlushableHashAggregateExecTransformer (28) - +- ^ ProjectExecTransformer (27) - +- ^ BroadcastHashJoinExecTransformer Inner (26) - :- ^ ProjectExecTransformer (19) - : +- ^ BroadcastHashJoinExecTransformer Inner (18) - : :- ^ ProjectExecTransformer (10) - : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : :- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ Scan parquet (2) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ BroadcastQueryStage (15), Statistics(X) - : +- ColumnarBroadcastExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ Scan parquet (11) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ BroadcastQueryStage (23), Statistics(X) - +- ColumnarBroadcastExchange (22) - +- ^ Scan parquet (20) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- ^ InputAdapter (9) + : : +- ^ BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- ^ InputAdapter (19) + : +- ^ BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- BroadcastHashJoin Inner BuildRight (57) - :- Project (53) - : +- BroadcastHashJoin Inner BuildRight (52) - : :- Project (47) - : : +- BroadcastHashJoin Inner BuildRight (46) - : : :- Filter (41) - : : : +- Scan parquet (40) - : : +- BroadcastExchange (45) - : : +- Project (44) - : : +- Filter (43) - : : +- Scan parquet (42) - : +- BroadcastExchange (51) - : +- Project (50) - : +- Filter (49) - : +- Scan parquet (48) - +- BroadcastExchange (56) - +- Filter (55) - +- Scan parquet (54) + TakeOrderedAndProject (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- BroadcastHashJoin Inner BuildRight (61) + :- Project (57) + : +- BroadcastHashJoin Inner BuildRight (56) + : :- Project (51) + : : +- BroadcastHashJoin Inner BuildRight (50) + : : :- Filter (45) + : : : +- Scan parquet (44) + : : +- BroadcastExchange (49) + : : +- Project (48) + : : +- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (55) + : +- Project (54) + : +- Filter (53) + : +- Scan parquet (52) + +- BroadcastExchange (60) + +- Filter (59) + +- Scan parquet (58) (1) Scan parquet @@ -68,286 +72,302 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(3) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(3) ProjectExecTransformer +(4) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(5) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(4) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(12) ProjectExecTransformer +(14) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(15) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(13) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(15) BroadcastQueryStage +(18) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(19) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(18) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) Scan parquet +(23) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(21) WholeStageCodegenTransformer (X) +(24) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(25) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(22) ColumnarBroadcastExchange +(26) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(27) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(24) InputAdapter +(28) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(25) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(26) BroadcastHashJoinExecTransformer +(30) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(27) ProjectExecTransformer +(31) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(28) FlushableHashAggregateExecTransformer +(32) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(29) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(30) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(31) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(33) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(34) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(40) Scan parquet +(44) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(42) Scan parquet +(46) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(43) Filter +(47) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(44) Project +(48) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(45) BroadcastExchange +(49) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(46) BroadcastHashJoin +(50) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(47) Project +(51) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(48) Scan parquet +(52) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(49) Filter +(53) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(50) Project +(54) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(51) BroadcastExchange +(55) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(52) BroadcastHashJoin +(56) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(53) Project +(57) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(54) Scan parquet +(58) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(55) Filter +(59) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(56) BroadcastExchange +(60) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) BroadcastHashJoin +(61) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(58) Project +(62) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(59) HashAggregate +(63) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(60) Exchange +(64) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(65) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(62) TakeOrderedAndProject +(66) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(63) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt index 943d59c56aaf..f8d59a67a22b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ FilterExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ BroadcastHashJoinExecTransformer Inner (17) - :- ^ ProjectExecTransformer (9) - : +- ^ BroadcastHashJoinExecTransformer Inner (8) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ Scan parquet (2) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ BroadcastQueryStage (14), Statistics(X) - +- ColumnarBroadcastExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ^ InputAdapter (34) + +- ^ ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ^ InputAdapter (27) + +- ^ ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- ^ InputAdapter (8) + : +- ^ BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- ^ InputAdapter (18) + +- ^ BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (54) - +- Exchange (53) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildRight (41) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (57) + +- Exchange (56) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Project (51) + +- BroadcastHashJoin Inner BuildRight (50) + :- Project (45) + : +- BroadcastHashJoin Inner BuildRight (44) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (49) + +- Project (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -60,472 +63,489 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(3) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(3) WholeStageCodegenTransformer (X) +(4) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(5) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(7) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(9) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(10) Scan parquet +(12) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(11) ProjectExecTransformer +(13) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(14) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(12) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [1]: [n_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [1]: [n_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(19) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(20) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(21) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(22) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(27) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(29) ColumnarExchange +(32) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(31) InputAdapter +(34) InputAdapter Input [2]: [ps_partkey#X, value#X] -(32) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(33) SortExecTransformer +(36) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(35) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(36) Scan parquet +(39) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(38) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(40) BroadcastExchange +(43) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(41) BroadcastHashJoin +(44) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(42) Project +(45) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(45) Project +(48) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(46) BroadcastExchange +(49) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(50) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(48) Project +(51) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(49) HashAggregate +(52) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(50) Exchange +(53) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(52) Filter +(55) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(53) Exchange +(56) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Sort +(57) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(55) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 27 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (95) +Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ^ InputAdapter (73) - +- ^ ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ FlushableHashAggregateExecTransformer (69) - +- ^ ProjectExecTransformer (68) - +- ^ BroadcastHashJoinExecTransformer Inner (67) - :- ^ ProjectExecTransformer (62) - : +- ^ BroadcastHashJoinExecTransformer Inner (61) - : :- ^ Scan parquet (56) - : +- ^ InputIteratorTransformer (60) - : +- ^ InputAdapter (59) - : +- ^ BroadcastQueryStage (58), Statistics(X) - : +- ReusedExchange (57) - +- ^ InputIteratorTransformer (66) - +- ^ InputAdapter (65) - +- ^ BroadcastQueryStage (64), Statistics(X) - +- ReusedExchange (63) + VeloxColumnarToRowExec (82) + +- ^ ProjectExecTransformer (80) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ^ InputAdapter (77) + +- ^ ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ BroadcastHashJoinExecTransformer Inner (71) + :- ^ ProjectExecTransformer (66) + : +- ^ BroadcastHashJoinExecTransformer Inner (65) + : :- ^ NoopFilter (60) + : : +- ^ Scan parquet (59) + : +- ^ InputIteratorTransformer (64) + : +- ^ InputAdapter (63) + : +- ^ BroadcastQueryStage (62), Statistics(X) + : +- ReusedExchange (61) + +- ^ InputIteratorTransformer (70) + +- ^ InputAdapter (69) + +- ^ BroadcastQueryStage (68), Statistics(X) + +- ReusedExchange (67) +- == Initial Plan == - HashAggregate (94) - +- Exchange (93) - +- HashAggregate (92) - +- Project (91) - +- BroadcastHashJoin Inner BuildRight (90) - :- Project (85) - : +- BroadcastHashJoin Inner BuildRight (84) - : :- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (83) - : +- Filter (82) - : +- Scan parquet (81) - +- BroadcastExchange (89) - +- Project (88) - +- Filter (87) - +- Scan parquet (86) - - -(56) Scan parquet + HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Filter (84) + : : +- Scan parquet (83) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) + + +(59) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(57) ReusedExchange [Reuses operator id: 4] +(60) NoopFilter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(61) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(58) BroadcastQueryStage +(62) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(59) InputAdapter +(63) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(60) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(65) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(62) ProjectExecTransformer +(66) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(63) ReusedExchange [Reuses operator id: 13] +(67) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(64) BroadcastQueryStage +(68) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(65) InputAdapter +(69) InputAdapter Input [1]: [n_nationkey#X] -(66) InputIteratorTransformer +(70) InputIteratorTransformer Input [1]: [n_nationkey#X] -(67) BroadcastHashJoinExecTransformer +(71) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(68) ProjectExecTransformer +(72) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(69) FlushableHashAggregateExecTransformer +(73) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(70) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(75) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(77) InputAdapter Input [2]: [sum#X, isEmpty#X] -(74) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(79) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(76) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(77) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(78) VeloxColumnarToRowExec +(82) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(79) Scan parquet +(83) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(80) Filter +(84) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(81) Scan parquet +(85) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(82) Filter +(86) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(83) BroadcastExchange +(87) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(84) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(85) Project +(89) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(86) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(87) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(88) Project +(92) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(89) BroadcastExchange +(93) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(90) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(91) Project +(95) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(92) HashAggregate +(96) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(93) Exchange +(97) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(98) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(95) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt index f53f974558bd..9e142ea961f5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (40) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ InputIteratorTransformer (6) - : +- ^ InputAdapter (5) - : +- ^ BroadcastQueryStage (4), Statistics(X) - : +- ColumnarBroadcastExchange (3) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (8) - +- ^ Scan parquet (7) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ^ InputAdapter (18) + +- ^ ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (39) - +- Exchange (38) - +- HashAggregate (37) - +- Exchange (36) - +- HashAggregate (35) - +- Project (34) - +- BroadcastHashJoin Inner BuildLeft (33) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- BroadcastHashJoin Inner BuildLeft (35) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -46,176 +48,184 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderpriority#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(10) ProjectExecTransformer Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(12) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(13) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(14) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(16) InputAdapter +(18) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(17) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(24) SortExecTransformer +(26) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(27) Scan parquet +(29) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(28) Filter +(30) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(29) BroadcastExchange +(31) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(32) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(31) Filter +(33) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(32) Project +(34) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(33) BroadcastHashJoin +(35) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(34) Project +(36) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(35) HashAggregate +(37) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(36) Exchange +(38) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) HashAggregate +(39) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(38) Exchange +(40) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Sort +(41) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(40) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt index 170698c552ac..9841ce1395e1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (51) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ ProjectExecTransformer (3) - +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (50) - +- Exchange (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Exchange (44) - +- HashAggregate (43) - +- Project (42) - +- BroadcastHashJoin LeftOuter BuildRight (41) - :- Scan parquet (36) - +- BroadcastExchange (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- BroadcastHashJoin LeftOuter BuildRight (42) + :- Scan parquet (37) + +- BroadcastExchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -62,222 +63,226 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(3) ProjectExecTransformer +(3) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] + +(4) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(7) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(11) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, count#X] Input [2]: [c_custkey#X, count#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(14) ColumnarExchange +(15) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [2]: [c_custkey#X, count#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(19) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(29) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(36) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(38) Filter +(39) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(39) Project +(40) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(40) BroadcastExchange +(41) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(42) Project +(43) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(43) HashAggregate +(44) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(44) Exchange +(45) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) HashAggregate +(46) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(46) HashAggregate +(47) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(47) Exchange +(48) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(49) Exchange +(50) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Sort +(51) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(51) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt index b4a96c56cfb8..fe13deb40500 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (32) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ ProjectExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (22) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (31) - +- Exchange (30) - +- HashAggregate (29) - +- Project (28) - +- BroadcastHashJoin Inner BuildRight (27) - :- Project (23) - : +- Filter (22) - : +- Scan parquet (21) - +- BroadcastExchange (26) - +- Filter (25) - +- Scan parquet (24) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -39,146 +41,154 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) Scan parquet +(4) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(5) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(6) WholeStageCodegenTransformer (X) Input [2]: [p_partkey#X, p_type#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(20) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(21) Scan parquet +(23) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(22) Filter +(24) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(23) Project +(25) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(26) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(26) BroadcastExchange +(28) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(27) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(28) Project +(30) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(29) HashAggregate +(31) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(30) Exchange +(32) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(31) HashAggregate +(33) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(32) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt index 75af0c33004f..4b761198f3e3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (39) +AdaptiveSparkPlan (41) +- == Final Plan == - VeloxColumnarToRowExec (24) - +- AQEShuffleRead (23) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ BroadcastHashJoinExecTransformer Inner (18) - :- ^ InputIteratorTransformer (6) - : +- ^ InputAdapter (5) - : +- ^ BroadcastQueryStage (4), Statistics(X) - : +- ColumnarBroadcastExchange (3) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (17) - +- ^ RegularHashAggregateExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FlushableHashAggregateExecTransformer (9) - +- ^ ProjectExecTransformer (8) - +- ^ Scan parquet (7) + VeloxColumnarToRowExec (26) + +- AQEShuffleRead (25) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner (20) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (38) - +- Exchange (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Filter (34) - +- HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- Filter (29) - +- Scan parquet (28) + Sort (40) + +- Exchange (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (29) + : +- Filter (28) + : +- Scan parquet (27) + +- Filter (36) + +- HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -46,326 +48,339 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(3) WholeStageCodegenTransformer (X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(10) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(9) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(11) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(12) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(16) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(17) FilterExecTransformer +(19) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(18) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(23) AQEShuffleRead +(25) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(24) VeloxColumnarToRowExec +(26) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(25) Scan parquet +(27) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(26) Filter +(28) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(27) BroadcastExchange +(29) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(30) Project +(32) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(31) HashAggregate +(33) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(32) Exchange +(34) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(34) Filter +(36) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(36) Project +(38) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(37) Exchange +(39) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Sort +(40) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(39) AdaptiveSparkPlan +(41) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 17 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (63) +Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (66) +- == Final Plan == - VeloxColumnarToRowExec (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ ProjectExecTransformer (50) - +- ^ RegularHashAggregateExecTransformer (49) - +- ^ InputIteratorTransformer (48) - +- ^ InputAdapter (47) - +- ^ ShuffleQueryStage (46), Statistics(X) - +- ColumnarExchange (45) - +- ^ ProjectExecTransformer (43) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (57) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ^ InputAdapter (50) + +- ^ ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ NoopFilter (43) + +- ^ Scan parquet (42) +- == Initial Plan == - HashAggregate (62) - +- HashAggregate (61) - +- HashAggregate (60) - +- Exchange (59) - +- HashAggregate (58) - +- Project (57) - +- Filter (56) - +- Scan parquet (55) + HashAggregate (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- Filter (59) + +- Scan parquet (58) -(40) Scan parquet +(42) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(41) ProjectExecTransformer +(43) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(44) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(42) FlushableHashAggregateExecTransformer +(45) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(43) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(44) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(45) ColumnarExchange +(48) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(46) ShuffleQueryStage +(49) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(47) InputAdapter +(50) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) InputIteratorTransformer +(51) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(49) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(50) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(51) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(52) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(53) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(54) VeloxColumnarToRowExec +(57) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(55) Scan parquet +(58) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(56) Filter +(59) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(57) Project +(60) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(58) HashAggregate +(61) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(59) Exchange +(62) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) HashAggregate +(63) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(61) HashAggregate +(64) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(63) AdaptiveSparkPlan +(66) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt index f87640ad9456..f178803b43e9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt @@ -1,62 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (62) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (41) - +- ^ SortExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ ProjectExecTransformer (27) - +- ^ FlushableHashAggregateExecTransformer (26) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ BroadcastHashJoinExecTransformer Inner (16) - :- ^ BroadcastHashJoinExecTransformer LeftAnti (9) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ BroadcastQueryStage (6), Statistics(X) - : +- ColumnarBroadcastExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ Scan parquet (2) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ BroadcastQueryStage (13), Statistics(X) - +- ColumnarBroadcastExchange (12) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (61) - +- Exchange (60) - +- HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- BroadcastHashJoin Inner BuildRight (52) - :- BroadcastHashJoin LeftAnti BuildRight (48) - : :- Filter (43) - : : +- Scan parquet (42) - : +- BroadcastExchange (47) - : +- Project (46) - : +- Filter (45) - : +- Scan parquet (44) - +- BroadcastExchange (51) - +- Filter (50) - +- Scan parquet (49) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- BroadcastHashJoin Inner BuildRight (46) + :- BroadcastHashJoin LeftAnti BuildRight (42) + : :- Filter (37) + : : +- Scan parquet (36) + : +- BroadcastExchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- BroadcastExchange (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -66,282 +61,255 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) Scan parquet -Output [2]: [s_suppkey#X, s_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_comment)] -ReadSchema: struct - -(3) ProjectExecTransformer -Output [1]: [s_suppkey#X] -Input [2]: [s_suppkey#X, s_comment#X] - -(4) WholeStageCodegenTransformer (X) -Input [1]: [s_suppkey#X] -Arguments: false - -(5) ColumnarBroadcastExchange -Input [1]: [s_suppkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] - -(6) BroadcastQueryStage -Output [1]: [s_suppkey#X] -Arguments: X - -(7) InputAdapter -Input [1]: [s_suppkey#X] - -(8) InputIteratorTransformer -Input [1]: [s_suppkey#X] - -(9) BroadcastHashJoinExecTransformer -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join type: LeftAnti -Join condition: None +(2) NoopFilter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X, ps_suppkey#X] -(10) Scan parquet +(3) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(11) WholeStageCodegenTransformer (X) +(4) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(5) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(12) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(7) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(14) InputAdapter +(8) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(15) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(17) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(18) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) ProjectExecTransformer +(13) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(21) ColumnarExchange +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(23) InputAdapter +(17) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(24) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(29) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(31) InputAdapter +(25) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(32) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(34) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) ColumnarExchange +(29) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(30) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(37) InputAdapter +(31) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(38) InputIteratorTransformer +(32) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) SortExecTransformer +(33) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(40) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(41) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) Scan parquet +(36) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(43) Filter +(37) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(44) Scan parquet +(38) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(45) Filter +(39) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(46) Project +(40) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(47) BroadcastExchange +(41) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(48) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(49) Scan parquet +(43) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(50) Filter +(44) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(51) BroadcastExchange +(45) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(52) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(53) Project +(47) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(54) HashAggregate +(48) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(55) Exchange +(49) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(50) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(58) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(60) Exchange +(54) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(55) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(62) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt index 9c4a028c442c..2484955032d0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt @@ -1,83 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (83) +AdaptiveSparkPlan (86) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- TakeOrderedAndProjectExecTransformer (49) - +- ^ RegularHashAggregateExecTransformer (47) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ BroadcastHashJoinExecTransformer Inner (38) - :- ^ ProjectExecTransformer (26) - : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : :- ^ InputIteratorTransformer (6) - : : +- ^ InputAdapter (5) - : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : +- ColumnarBroadcastExchange (3) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (24) - : :- ^ Scan parquet (7) - : +- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ BroadcastQueryStage (21), Statistics(X) - : +- ColumnarBroadcastExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ FilterExecTransformer (17) - : +- ^ RegularHashAggregateExecTransformer (16) - : +- ^ InputIteratorTransformer (15) - : +- ^ InputAdapter (14) - : +- ^ ShuffleQueryStage (13), Statistics(X) - : +- ColumnarExchange (12) - : +- ^ ProjectExecTransformer (10) - : +- ^ FlushableHashAggregateExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ BroadcastQueryStage (35), Statistics(X) - +- ColumnarBroadcastExchange (34) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (32) - :- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ BroadcastQueryStage (29), Statistics(X) - +- ReusedExchange (28) + VeloxColumnarToRowExec (53) + +- TakeOrderedAndProjectExecTransformer (52) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23), Statistics(X) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ ShuffleQueryStage (15), Statistics(X) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + :- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ BroadcastQueryStage (32), Statistics(X) + +- ReusedExchange (31) +- == Initial Plan == - TakeOrderedAndProject (82) - +- HashAggregate (81) - +- Exchange (80) - +- HashAggregate (79) - +- Project (78) - +- BroadcastHashJoin Inner BuildRight (77) - :- Project (65) - : +- BroadcastHashJoin Inner BuildLeft (64) - : :- BroadcastExchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- BroadcastHashJoin LeftSemi BuildRight (63) - : :- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastExchange (62) - : +- Project (61) - : +- Filter (60) - : +- HashAggregate (59) - : +- Exchange (58) - : +- HashAggregate (57) - : +- Scan parquet (56) - +- BroadcastExchange (76) - +- BroadcastHashJoin LeftSemi BuildRight (75) - :- Filter (67) - : +- Scan parquet (66) - +- BroadcastExchange (74) - +- Project (73) - +- Filter (72) - +- HashAggregate (71) - +- Exchange (70) - +- HashAggregate (69) - +- Scan parquet (68) + TakeOrderedAndProject (85) + +- HashAggregate (84) + +- Exchange (83) + +- HashAggregate (82) + +- Project (81) + +- BroadcastHashJoin Inner BuildRight (80) + :- Project (68) + : +- BroadcastHashJoin Inner BuildLeft (67) + : :- BroadcastExchange (56) + : : +- Filter (55) + : : +- Scan parquet (54) + : +- BroadcastHashJoin LeftSemi BuildRight (66) + : :- Filter (58) + : : +- Scan parquet (57) + : +- BroadcastExchange (65) + : +- Project (64) + : +- Filter (63) + : +- HashAggregate (62) + : +- Exchange (61) + : +- HashAggregate (60) + : +- Scan parquet (59) + +- BroadcastExchange (79) + +- BroadcastHashJoin LeftSemi BuildRight (78) + :- Filter (70) + : +- Scan parquet (69) + +- BroadcastExchange (77) + +- Project (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Scan parquet (71) (1) Scan parquet @@ -87,383 +90,395 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X, c_name#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_name#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(8) Scan parquet +(9) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(10) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(9) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(11) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(12) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(16) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(17) FilterExecTransformer +(19) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(20) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastQueryStage +(23) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [1]: [l_orderkey#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [1]: [l_orderkey#X] -(24) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(25) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(26) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(27) Scan parquet +(29) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(28) ReusedExchange [Reuses operator id: 20] +(30) NoopFilter +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X, l_quantity#X] + +(31) ReusedExchange [Reuses operator id: 22] Output [1]: [l_orderkey#X] -(29) BroadcastQueryStage +(32) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [1]: [l_orderkey#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [1]: [l_orderkey#X] -(32) BroadcastHashJoinExecTransformer +(35) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(34) ColumnarBroadcastExchange +(37) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(38) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(36) InputAdapter +(39) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(37) InputIteratorTransformer +(40) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(38) BroadcastHashJoinExecTransformer +(41) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(39) ProjectExecTransformer +(42) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(40) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(41) ProjectExecTransformer +(44) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(42) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(43) ColumnarExchange +(46) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(47) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(45) InputAdapter +(48) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(46) InputIteratorTransformer +(49) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(47) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(48) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(49) TakeOrderedAndProjectExecTransformer +(52) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(50) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(51) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(53) BroadcastExchange +(56) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(54) Scan parquet +(57) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(55) Filter +(58) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(56) Scan parquet +(59) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(57) HashAggregate +(60) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(58) Exchange +(61) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(62) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(60) Filter +(63) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(61) Project +(64) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(62) BroadcastExchange +(65) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(64) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(65) Project +(68) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(66) Scan parquet +(69) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(67) Filter +(70) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(68) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(69) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(70) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(72) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(73) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(74) BroadcastExchange +(77) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(75) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(76) BroadcastExchange +(79) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(78) Project +(81) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(79) HashAggregate +(82) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(80) Exchange +(83) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(84) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(82) TakeOrderedAndProject +(85) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(83) AdaptiveSparkPlan +(86) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt index 16e791cab81b..1a2a13c21db8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt @@ -1,34 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (31) +AdaptiveSparkPlan (33) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (30) - +- Exchange (29) - +- HashAggregate (28) - +- Project (27) - +- BroadcastHashJoin Inner BuildRight (26) - :- Project (22) - : +- Filter (21) - : +- Scan parquet (20) - +- BroadcastExchange (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (32) + +- Exchange (31) + +- HashAggregate (30) + +- Project (29) + +- BroadcastHashJoin Inner BuildRight (28) + :- Project (24) + : +- Filter (23) + : +- Scan parquet (22) + +- BroadcastExchange (27) + +- Filter (26) + +- Scan parquet (25) (1) Scan parquet @@ -38,142 +40,150 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(3) ProjectExecTransformer Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(3) Scan parquet +(4) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(5) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(6) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [1]: [revenue#X] -(20) Scan parquet +(22) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(21) Filter +(23) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(22) Project +(24) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(23) Scan parquet +(25) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(24) Filter +(26) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(25) BroadcastExchange +(27) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(26) BroadcastHashJoin +(28) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(27) Project +(29) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(28) HashAggregate +(30) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(29) Exchange +(31) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(30) HashAggregate +(32) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(31) AdaptiveSparkPlan +(33) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt index 70067664658e..ed40d3bbe23d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt @@ -1,91 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (96) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- AQEShuffleRead (54) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ BroadcastHashJoinExecTransformer Inner (49) - :- ^ ProjectExecTransformer (41) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (40) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ BroadcastQueryStage (37), Statistics(X) - : +- ColumnarBroadcastExchange (36) - : +- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ InputIteratorTransformer (15) - : : +- ^ InputAdapter (14) - : : +- ^ BroadcastQueryStage (13), Statistics(X) - : : +- ColumnarBroadcastExchange (12) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (10) - : : :- ^ Scan parquet (2) - : : +- ^ InputIteratorTransformer (9) - : : +- ^ InputAdapter (8) - : : +- ^ BroadcastQueryStage (7), Statistics(X) - : : +- ColumnarBroadcastExchange (6) - : : +- ^ ProjectExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ FilterExecTransformer (32) - : +- ^ ProjectExecTransformer (31) - : +- ^ RegularHashAggregateExecTransformer (30) - : +- ^ InputIteratorTransformer (29) - : +- ^ InputAdapter (28) - : +- ^ ShuffleQueryStage (27), Statistics(X) - : +- ColumnarExchange (26) - : +- ^ ProjectExecTransformer (24) - : +- ^ FlushableHashAggregateExecTransformer (23) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (22) - : :- ^ ProjectExecTransformer (17) - : : +- ^ Scan parquet (16) - : +- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ BroadcastQueryStage (19), Statistics(X) - : +- ReusedExchange (18) - +- ^ InputIteratorTransformer (48) - +- ^ InputAdapter (47) - +- ^ BroadcastQueryStage (46), Statistics(X) - +- ColumnarBroadcastExchange (45) - +- ^ ProjectExecTransformer (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (60) + +- AQEShuffleRead (59) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- ^ ProjectExecTransformer (55) + +- ^ BroadcastHashJoinExecTransformer Inner (54) + :- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (44) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (43) + : +- ^ InputAdapter (42) + : +- ^ BroadcastQueryStage (41), Statistics(X) + : +- ColumnarBroadcastExchange (40) + : +- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ InputIteratorTransformer (18) + : : +- ^ InputAdapter (17) + : : +- ^ BroadcastQueryStage (16), Statistics(X) + : : +- ColumnarBroadcastExchange (15) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (13) + : : :- ^ NoopFilter (4) + : : : +- ^ Scan parquet (3) + : : +- ^ InputIteratorTransformer (12) + : : +- ^ InputAdapter (11) + : : +- ^ BroadcastQueryStage (10), Statistics(X) + : : +- ColumnarBroadcastExchange (9) + : : +- ^ ProjectExecTransformer (7) + : : +- ^ NoopFilter (6) + : : +- ^ Scan parquet (5) + : +- ^ FilterExecTransformer (36) + : +- ^ ProjectExecTransformer (35) + : +- ^ RegularHashAggregateExecTransformer (34) + : +- ^ InputIteratorTransformer (33) + : +- ^ InputAdapter (32) + : +- ^ ShuffleQueryStage (31), Statistics(X) + : +- ColumnarExchange (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ BroadcastQueryStage (51), Statistics(X) + +- ColumnarBroadcastExchange (50) + +- ^ ProjectExecTransformer (48) + +- ^ NoopFilter (47) + +- ^ Scan parquet (46) +- == Initial Plan == - Sort (90) - +- Exchange (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (82) - : +- BroadcastHashJoin LeftSemi BuildRight (81) - : :- Filter (57) - : : +- Scan parquet (56) - : +- BroadcastExchange (80) - : +- Project (79) - : +- BroadcastHashJoin Inner BuildLeft (78) - : :- BroadcastExchange (65) - : : +- BroadcastHashJoin LeftSemi BuildRight (64) - : : :- Filter (59) - : : : +- Scan parquet (58) - : : +- BroadcastExchange (63) - : : +- Project (62) - : : +- Filter (61) - : : +- Scan parquet (60) - : +- Filter (77) - : +- HashAggregate (76) - : +- Exchange (75) - : +- HashAggregate (74) - : +- BroadcastHashJoin LeftSemi BuildRight (73) - : :- Project (68) - : : +- Filter (67) - : : +- Scan parquet (66) - : +- BroadcastExchange (72) - : +- Project (71) - : +- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (86) - +- Project (85) - +- Filter (84) - +- Scan parquet (83) + Sort (95) + +- Exchange (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (87) + : +- BroadcastHashJoin LeftSemi BuildRight (86) + : :- Filter (62) + : : +- Scan parquet (61) + : +- BroadcastExchange (85) + : +- Project (84) + : +- BroadcastHashJoin Inner BuildLeft (83) + : :- BroadcastExchange (70) + : : +- BroadcastHashJoin LeftSemi BuildRight (69) + : : :- Filter (64) + : : : +- Scan parquet (63) + : : +- BroadcastExchange (68) + : : +- Project (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Filter (82) + : +- HashAggregate (81) + : +- Exchange (80) + : +- HashAggregate (79) + : +- BroadcastHashJoin LeftSemi BuildRight (78) + : :- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- BroadcastExchange (77) + : +- Project (76) + : +- Filter (75) + : +- Scan parquet (74) + +- BroadcastExchange (91) + +- Project (90) + +- Filter (89) + +- Scan parquet (88) (1) Scan parquet @@ -95,410 +100,430 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(3) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(3) Scan parquet +(4) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(5) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(4) ProjectExecTransformer +(6) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(7) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(5) WholeStageCodegenTransformer (X) +(8) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(9) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(7) BroadcastQueryStage +(10) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(8) InputAdapter +(11) InputAdapter Input [1]: [p_partkey#X] -(9) InputIteratorTransformer +(12) InputIteratorTransformer Input [1]: [p_partkey#X] -(10) BroadcastHashJoinExecTransformer +(13) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(11) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(12) ColumnarBroadcastExchange +(15) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(16) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(14) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(15) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) Scan parquet +(19) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(17) ProjectExecTransformer +(20) NoopFilter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(21) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(18) ReusedExchange [Reuses operator id: 6] +(22) ReusedExchange [Reuses operator id: 9] Output [1]: [p_partkey#X] -(19) BroadcastQueryStage +(23) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(20) InputAdapter +(24) InputAdapter Input [1]: [p_partkey#X] -(21) InputIteratorTransformer +(25) InputIteratorTransformer Input [1]: [p_partkey#X] -(22) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(23) FlushableHashAggregateExecTransformer +(27) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(24) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(25) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(30) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(32) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(31) ProjectExecTransformer +(35) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(32) FilterExecTransformer +(36) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(33) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(36) ColumnarBroadcastExchange +(40) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastQueryStage +(41) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(38) InputAdapter +(42) InputAdapter Input [1]: [ps_suppkey#X] -(39) InputIteratorTransformer +(43) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(40) BroadcastHashJoinExecTransformer +(44) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(41) ProjectExecTransformer +(45) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(43) ProjectExecTransformer +(47) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(48) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(44) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(45) ColumnarBroadcastExchange +(50) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(46) BroadcastQueryStage +(51) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(47) InputAdapter +(52) InputAdapter Input [1]: [n_nationkey#X] -(48) InputIteratorTransformer +(53) InputIteratorTransformer Input [1]: [n_nationkey#X] -(49) BroadcastHashJoinExecTransformer +(54) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(50) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(51) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(52) ColumnarExchange +(57) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(58) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(54) AQEShuffleRead +(59) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(55) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(56) Scan parquet +(61) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(57) Filter +(62) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(58) Scan parquet +(63) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(59) Filter +(64) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(60) Scan parquet +(65) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(62) Project +(67) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(63) BroadcastExchange +(68) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(64) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(65) BroadcastExchange +(70) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(66) Scan parquet +(71) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(67) Filter +(72) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(68) Project +(73) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(69) Scan parquet +(74) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(70) Filter +(75) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(71) Project +(76) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(72) BroadcastExchange +(77) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(73) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(74) HashAggregate +(79) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(75) Exchange +(80) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) HashAggregate +(81) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(77) Filter +(82) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(78) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(79) Project +(84) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(80) BroadcastExchange +(85) BroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(81) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(82) Project +(87) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(83) Scan parquet +(88) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(84) Filter +(89) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(85) Project +(90) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(86) BroadcastExchange +(91) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(87) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) Project +(93) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(89) Exchange +(94) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Sort +(95) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(91) AdaptiveSparkPlan +(96) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt index 80e1d1dc5f36..d0b14137ca5a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt @@ -1,85 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (91) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48), Statistics(X) - +- ColumnarExchange (47) - +- ^ ProjectExecTransformer (45) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ BroadcastHashJoinExecTransformer Inner (42) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (25) - : : +- ^ BroadcastHashJoinExecTransformer Inner (24) - : : :- ^ InputIteratorTransformer (6) - : : : +- ^ InputAdapter (5) - : : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : : +- ColumnarBroadcastExchange (3) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (23) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (15) - : : : :- ^ ProjectExecTransformer (8) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (14) - : : : +- ^ InputAdapter (13) - : : : +- ^ BroadcastQueryStage (12), Statistics(X) - : : : +- ColumnarBroadcastExchange (11) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (22) - : : +- ^ InputAdapter (21) - : : +- ^ BroadcastQueryStage (20), Statistics(X) - : : +- ColumnarBroadcastExchange (19) - : : +- ^ ProjectExecTransformer (17) - : : +- ^ Scan parquet (16) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ColumnarBroadcastExchange (29) - : +- ^ ProjectExecTransformer (27) - : +- ^ Scan parquet (26) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ BroadcastQueryStage (39), Statistics(X) - +- ColumnarBroadcastExchange (38) - +- ^ ProjectExecTransformer (36) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- ^ InputAdapter (24) + : : +- ^ BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (75) - : +- BroadcastHashJoin Inner BuildRight (74) - : :- Project (69) - : : +- BroadcastHashJoin Inner BuildLeft (68) - : : :- BroadcastExchange (56) - : : : +- Filter (55) - : : : +- Scan parquet (54) - : : +- BroadcastHashJoin LeftAnti BuildRight (67) - : : :- BroadcastHashJoin LeftSemi BuildRight (62) - : : : :- Project (59) - : : : : +- Filter (58) - : : : : +- Scan parquet (57) - : : : +- BroadcastExchange (61) - : : : +- Scan parquet (60) - : : +- BroadcastExchange (66) - : : +- Project (65) - : : +- Filter (64) - : : +- Scan parquet (63) - : +- BroadcastExchange (73) - : +- Project (72) - : +- Filter (71) - : +- Scan parquet (70) - +- BroadcastExchange (79) - +- Project (78) - +- Filter (77) - +- Scan parquet (76) + TakeOrderedAndProject (90) + +- HashAggregate (89) + +- Exchange (88) + +- HashAggregate (87) + +- Project (86) + +- BroadcastHashJoin Inner BuildRight (85) + :- Project (80) + : +- BroadcastHashJoin Inner BuildRight (79) + : :- Project (74) + : : +- BroadcastHashJoin Inner BuildLeft (73) + : : :- BroadcastExchange (61) + : : : +- Filter (60) + : : : +- Scan parquet (59) + : : +- BroadcastHashJoin LeftAnti BuildRight (72) + : : :- BroadcastHashJoin LeftSemi BuildRight (67) + : : : :- Project (64) + : : : : +- Filter (63) + : : : : +- Scan parquet (62) + : : : +- BroadcastExchange (66) + : : : +- Scan parquet (65) + : : +- BroadcastExchange (71) + : : +- Project (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- BroadcastExchange (78) + : +- Project (77) + : +- Filter (76) + : +- Scan parquet (75) + +- BroadcastExchange (84) + +- Project (83) + +- Filter (82) + +- Scan parquet (81) (1) Scan parquet @@ -89,392 +94,412 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(10) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(9) Scan parquet +(11) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(11) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(12) BroadcastQueryStage +(14) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(15) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(16) Scan parquet +(18) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(17) ProjectExecTransformer +(19) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(20) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(18) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(19) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(20) BroadcastQueryStage +(23) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(21) InputAdapter +(24) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(22) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(23) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(24) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(25) ProjectExecTransformer +(28) ProjectExecTransformer Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(26) Scan parquet +(29) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(27) ProjectExecTransformer +(30) NoopFilter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: [o_orderkey#X, o_orderstatus#X] + +(31) ProjectExecTransformer Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(28) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [1]: [o_orderkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(33) ColumnarBroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(30) BroadcastQueryStage +(34) BroadcastQueryStage Output [1]: [o_orderkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [1]: [o_orderkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [1]: [o_orderkey#X] -(33) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(35) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(36) ProjectExecTransformer +(40) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(41) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(37) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(38) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) BroadcastQueryStage +(44) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(40) InputAdapter +(45) InputAdapter Input [1]: [n_nationkey#X] -(41) InputIteratorTransformer +(46) InputIteratorTransformer Input [1]: [n_nationkey#X] -(42) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(43) ProjectExecTransformer +(48) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(44) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(45) ProjectExecTransformer +(50) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(46) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(47) ColumnarExchange +(52) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(49) InputAdapter +(54) InputAdapter Input [2]: [s_name#X, count#X] -(50) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(51) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(53) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(54) Scan parquet +(59) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) BroadcastExchange +(61) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(62) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(58) Filter +(63) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(59) Project +(64) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(60) Scan parquet +(65) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(61) BroadcastExchange +(66) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(62) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(63) Scan parquet +(68) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(64) Filter +(69) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(65) Project +(70) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(66) BroadcastExchange +(71) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(69) Project +(74) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(70) Scan parquet +(75) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(76) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(72) Project +(77) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(73) BroadcastExchange +(78) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(74) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(75) Project +(80) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(76) Scan parquet +(81) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(77) Filter +(82) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(78) Project +(83) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(79) BroadcastExchange +(84) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(85) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(81) Project +(86) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(82) HashAggregate +(87) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(83) Exchange +(88) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(89) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(85) TakeOrderedAndProject +(90) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(86) AdaptiveSparkPlan +(91) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt index 503fe27ab555..698879c473b6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt @@ -1,39 +1,40 @@ == Physical Plan == -AdaptiveSparkPlan (37) +AdaptiveSparkPlan (38) +- == Final Plan == - VeloxColumnarToRowExec (25) - +- ^ SortExecTransformer (23) - +- ^ InputIteratorTransformer (22) - +- ^ InputAdapter (21) - +- ^ ShuffleQueryStage (20), Statistics(X) - +- ColumnarExchange (19) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FlushableHashAggregateExecTransformer (10) - +- ^ ProjectExecTransformer (9) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (8) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (7) - +- ^ InputAdapter (6) - +- ^ BroadcastQueryStage (5), Statistics(X) - +- ColumnarBroadcastExchange (4) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (26) + +- ^ SortExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (36) - +- Exchange (35) - +- HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin LeftAnti BuildRight (30) - :- Filter (27) - : +- Scan parquet (26) - +- BroadcastExchange (29) - +- Scan parquet (28) + Sort (37) + +- Exchange (36) + +- HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin LeftAnti BuildRight (31) + :- Filter (28) + : +- Scan parquet (27) + +- BroadcastExchange (30) + +- Scan parquet (29) (1) Scan parquet @@ -43,272 +44,281 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X, c_phone#X, c_acctbal#X] + +(3) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [o_custkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [o_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [o_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [o_custkey#X] -(8) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(9) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(10) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(14) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(19) ColumnarExchange +(20) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(20) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(21) InputAdapter +(22) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(22) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) SortExecTransformer +(24) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(24) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(25) VeloxColumnarToRowExec +(26) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(26) Scan parquet +(27) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(27) Filter +(28) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(28) Scan parquet +(29) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(29) BroadcastExchange +(30) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(31) Project +(32) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(32) HashAggregate +(33) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(35) Exchange +(36) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Sort +(37) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(37) AdaptiveSparkPlan +(38) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (48) - +- ^ RegularHashAggregateExecTransformer (46) - +- ^ InputIteratorTransformer (45) - +- ^ InputAdapter (44) - +- ^ ShuffleQueryStage (43), Statistics(X) - +- ColumnarExchange (42) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ Scan parquet (38) + VeloxColumnarToRowExec (50) + +- ^ RegularHashAggregateExecTransformer (48) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FlushableHashAggregateExecTransformer (42) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- Filter (50) - +- Scan parquet (49) + HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- Filter (52) + +- Scan parquet (51) -(38) Scan parquet +(39) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(39) ProjectExecTransformer +(40) NoopFilter +Input [2]: [c_phone#X, c_acctbal#X] +Arguments: [c_phone#X, c_acctbal#X] + +(41) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(40) FlushableHashAggregateExecTransformer +(42) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(41) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(42) ColumnarExchange +(44) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(43) ShuffleQueryStage +(45) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(44) InputAdapter +(46) InputAdapter Input [2]: [sum#X, count#X] -(45) InputIteratorTransformer +(47) InputIteratorTransformer Input [2]: [sum#X, count#X] -(46) RegularHashAggregateExecTransformer +(48) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(47) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(48) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(49) Scan parquet +(51) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(50) Filter +(52) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(51) Project +(53) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(52) HashAggregate +(54) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(53) Exchange +(55) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(55) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt index cd21047a80a5..98a7cd0e0145 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt @@ -1,52 +1,55 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (31) - +- TakeOrderedAndProjectExecTransformer (30) - +- ^ ProjectExecTransformer (28) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ BroadcastHashJoinExecTransformer Inner (18) - :- ^ ProjectExecTransformer (10) - : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ BroadcastQueryStage (15), Statistics(X) - +- ColumnarBroadcastExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ Scan parquet (11) + VeloxColumnarToRowExec (34) + +- TakeOrderedAndProjectExecTransformer (33) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- Project (45) - +- BroadcastHashJoin Inner BuildRight (44) - :- Project (39) - : +- BroadcastHashJoin Inner BuildLeft (38) - : :- BroadcastExchange (35) - : : +- Project (34) - : : +- Filter (33) - : : +- Scan parquet (32) - : +- Filter (37) - : +- Scan parquet (36) - +- BroadcastExchange (43) - +- Project (42) - +- Filter (41) - +- Scan parquet (40) + TakeOrderedAndProject (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- Project (42) + : +- BroadcastHashJoin Inner BuildLeft (41) + : :- BroadcastExchange (38) + : : +- Project (37) + : : +- Filter (36) + : : +- Scan parquet (35) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -56,226 +59,238 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: [c_custkey#X, c_mktsegment#X] + +(3) ProjectExecTransformer Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [c_custkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(12) ProjectExecTransformer +(14) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(15) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(13) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(15) BroadcastQueryStage +(18) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(19) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(18) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(21) ProjectExecTransformer +(24) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, o_orderdate#X, o_shippriority#X, 42) AS hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(22) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(26) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(28) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(29) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(28) ProjectExecTransformer +(31) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(30) TakeOrderedAndProjectExecTransformer +(33) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(31) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(32) Scan parquet +(35) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(33) Filter +(36) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(34) Project +(37) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(35) BroadcastExchange +(38) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(36) Scan parquet +(39) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(38) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(39) Project +(42) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(43) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(41) Filter +(44) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(42) Project +(45) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(43) BroadcastExchange +(46) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(44) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(45) Project +(48) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(46) HashAggregate +(49) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(47) Exchange +(50) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(51) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(49) TakeOrderedAndProject +(52) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt index 95367083053d..b5fefc6bef3c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt @@ -1,44 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (10) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- ^ InputAdapter (10) + +- ^ BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin LeftSemi BuildRight (35) - :- Project (30) - : +- Filter (29) - : +- Scan parquet (28) - +- BroadcastExchange (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin LeftSemi BuildRight (37) + :- Project (32) + : +- Filter (31) + : +- Scan parquet (30) + +- BroadcastExchange (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -48,184 +50,192 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(3) Scan parquet +(4) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(4) ProjectExecTransformer +(5) NoopFilter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(6) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(5) WholeStageCodegenTransformer (X) +(7) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(8) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(7) BroadcastQueryStage +(9) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(8) InputAdapter +(10) InputAdapter Input [1]: [l_orderkey#X] -(9) InputIteratorTransformer +(11) InputIteratorTransformer Input [1]: [l_orderkey#X] -(10) BroadcastHashJoinExecTransformer +(12) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(11) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(12) FlushableHashAggregateExecTransformer +(14) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(13) ProjectExecTransformer +(15) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(14) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(15) ColumnarExchange +(17) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(18) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(17) InputAdapter +(19) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(18) InputIteratorTransformer +(20) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(25) SortExecTransformer +(27) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(28) Scan parquet +(30) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(29) Filter +(31) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(30) Project +(32) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(31) Scan parquet +(33) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(32) Filter +(34) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(33) Project +(35) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(34) BroadcastExchange +(36) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(36) Project +(38) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(37) HashAggregate +(39) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(38) Exchange +(40) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(40) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt index 71c8049fa4dd..be0403f587f5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt @@ -1,92 +1,98 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- ^ SortExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48), Statistics(X) - +- ColumnarExchange (47) - +- ^ ProjectExecTransformer (45) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ BroadcastHashJoinExecTransformer Inner (42) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (26) - : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : :- ^ ProjectExecTransformer (18) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : :- ^ InputIteratorTransformer (6) - : : : : : +- ^ InputAdapter (5) - : : : : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (3) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (8) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ BroadcastQueryStage (22), Statistics(X) - : : +- ColumnarBroadcastExchange (21) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ColumnarBroadcastExchange (29) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ BroadcastQueryStage (39), Statistics(X) - +- ColumnarBroadcastExchange (38) - +- ^ ProjectExecTransformer (36) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ^ InputAdapter (61) + +- ^ ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ^ InputAdapter (55) + +- ^ ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- ^ InputAdapter (18) + : : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- ^ InputAdapter (36) + : +- ^ BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (93) - +- Exchange (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (62) - : : : : : +- Filter (61) - : : : : : +- Scan parquet (60) - : : : : +- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (86) - +- Project (85) - +- Filter (84) - +- Scan parquet (83) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Project (71) + : : : : +- Filter (70) + : : : : +- Scan parquet (69) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (92) + +- Project (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -96,424 +102,448 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(10) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(15) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) Scan parquet +(22) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(24) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(27) Scan parquet +(31) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] + +(33) WholeStageCodegenTransformer (X) Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(36) ProjectExecTransformer +(41) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(42) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(37) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(38) ColumnarBroadcastExchange +(44) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) BroadcastQueryStage +(45) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(40) InputAdapter +(46) InputAdapter Input [1]: [r_regionkey#X] -(41) InputIteratorTransformer +(47) InputIteratorTransformer Input [1]: [r_regionkey#X] -(42) BroadcastHashJoinExecTransformer +(48) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(43) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(44) FlushableHashAggregateExecTransformer +(50) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(45) ProjectExecTransformer +(51) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(46) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(47) ColumnarExchange +(53) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(49) InputAdapter +(55) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(50) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(51) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(52) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(53) ColumnarExchange +(59) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(55) InputAdapter +(61) InputAdapter Input [2]: [n_name#X, revenue#X] -(56) InputIteratorTransformer +(62) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(57) SortExecTransformer +(63) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(60) Scan parquet +(66) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(61) Filter +(67) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(62) BroadcastExchange +(68) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(69) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(64) Filter +(70) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(65) Project +(71) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(66) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(67) Project +(73) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(68) Scan parquet +(74) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(69) Filter +(75) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(70) BroadcastExchange +(76) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(72) Project +(78) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(79) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(80) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(81) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(77) Project +(83) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(84) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(80) BroadcastExchange +(86) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(88) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(83) Scan parquet +(89) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(84) Filter +(90) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(85) Project +(91) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(86) BroadcastExchange +(92) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(87) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(88) Project +(94) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(89) HashAggregate +(95) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(90) Exchange +(96) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(97) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(92) Exchange +(98) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(99) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(94) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt index 733adf0d0b4a..9333e2a8ad59 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (18) +AdaptiveSparkPlan (19) +- == Final Plan == - VeloxColumnarToRowExec (11) - +- ^ RegularHashAggregateExecTransformer (9) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ ShuffleQueryStage (6), Statistics(X) - +- ColumnarExchange (5) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (17) - +- Exchange (16) - +- HashAggregate (15) - +- Project (14) - +- Filter (13) - +- Scan parquet (12) + HashAggregate (18) + +- Exchange (17) + +- HashAggregate (16) + +- Project (15) + +- Filter (14) + +- Scan parquet (13) (1) Scan parquet @@ -26,82 +27,86 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(5) ColumnarExchange +(6) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [sum#X, isEmpty#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(9) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(11) VeloxColumnarToRowExec +(12) VeloxColumnarToRowExec Input [1]: [revenue#X] -(12) Scan parquet +(13) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(13) Filter +(14) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(14) Project +(15) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) HashAggregate +(16) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(16) Exchange +(17) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(17) HashAggregate +(18) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(18) AdaptiveSparkPlan +(19) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt index 93a86cb78a95..2ac2968387fd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt @@ -1,87 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (88) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ SortExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ^ InputAdapter (51) - +- ^ ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ RegularHashAggregateExecTransformer (47) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ BroadcastHashJoinExecTransformer Inner (38) - :- ^ ProjectExecTransformer (33) - : +- ^ BroadcastHashJoinExecTransformer Inner (32) - : :- ^ ProjectExecTransformer (25) - : : +- ^ BroadcastHashJoinExecTransformer Inner (24) - : : :- ^ ProjectExecTransformer (17) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (16) - : : : :- ^ ProjectExecTransformer (9) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (8) - : : : : :- ^ InputIteratorTransformer (6) - : : : : : +- ^ InputAdapter (5) - : : : : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (3) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (15) - : : : +- ^ InputAdapter (14) - : : : +- ^ BroadcastQueryStage (13), Statistics(X) - : : : +- ColumnarBroadcastExchange (12) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ BroadcastQueryStage (21), Statistics(X) - : : +- ColumnarBroadcastExchange (20) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ BroadcastQueryStage (29), Statistics(X) - : +- ColumnarBroadcastExchange (28) - : +- ^ Scan parquet (26) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ BroadcastQueryStage (35), Statistics(X) - +- ReusedExchange (34) + VeloxColumnarToRowExec (60) + +- ^ SortExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ^ InputAdapter (56) + +- ^ ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ^ InputAdapter (50) + +- ^ ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- ^ InputAdapter (17) + : : : +- ^ BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- ^ InputAdapter (26) + : : +- ^ BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- ^ InputAdapter (41) + +- ^ BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (87) - +- Exchange (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- BroadcastHashJoin Inner BuildRight (81) - :- Project (77) - : +- BroadcastHashJoin Inner BuildRight (76) - : :- Project (72) - : : +- BroadcastHashJoin Inner BuildRight (71) - : : :- Project (67) - : : : +- BroadcastHashJoin Inner BuildRight (66) - : : : :- Project (62) - : : : : +- BroadcastHashJoin Inner BuildLeft (61) - : : : : :- BroadcastExchange (58) - : : : : : +- Filter (57) - : : : : : +- Scan parquet (56) - : : : : +- Filter (60) - : : : : +- Scan parquet (59) - : : : +- BroadcastExchange (65) - : : : +- Filter (64) - : : : +- Scan parquet (63) - : : +- BroadcastExchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (75) - : +- Filter (74) - : +- Scan parquet (73) - +- BroadcastExchange (80) - +- Filter (79) - +- Scan parquet (78) + Sort (92) + +- Exchange (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (77) + : : +- BroadcastHashJoin Inner BuildRight (76) + : : :- Project (72) + : : : +- BroadcastHashJoin Inner BuildRight (71) + : : : :- Project (67) + : : : : +- BroadcastHashJoin Inner BuildLeft (66) + : : : : :- BroadcastExchange (63) + : : : : : +- Filter (62) + : : : : : +- Scan parquet (61) + : : : : +- Filter (65) + : : : : +- Scan parquet (64) + : : : +- BroadcastExchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- BroadcastExchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (80) + : +- Filter (79) + : +- Scan parquet (78) + +- BroadcastExchange (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -91,396 +96,416 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(8) BroadcastHashJoinExecTransformer +(9) NoopFilter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(10) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(9) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) Scan parquet +(12) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(11) WholeStageCodegenTransformer (X) +(13) NoopFilter +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X, o_custkey#X] + +(14) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarBroadcastExchange +(15) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(16) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) BroadcastHashJoinExecTransformer +(19) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(17) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(18) Scan parquet +(21) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(19) WholeStageCodegenTransformer (X) +(22) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(23) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(20) ColumnarBroadcastExchange +(24) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(21) BroadcastQueryStage +(25) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(22) InputAdapter +(26) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(23) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(24) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(25) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(26) Scan parquet +(30) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(27) WholeStageCodegenTransformer (X) +(31) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(32) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(28) ColumnarBroadcastExchange +(33) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastQueryStage +(34) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(30) InputAdapter +(35) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(31) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(32) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(38) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(34) ReusedExchange [Reuses operator id: 28] +(39) ReusedExchange [Reuses operator id: 33] Output [2]: [n_nationkey#X, n_name#X] -(35) BroadcastQueryStage +(40) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(36) InputAdapter +(41) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(37) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(38) BroadcastHashJoinExecTransformer +(43) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(39) ProjectExecTransformer +(44) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(40) FlushableHashAggregateExecTransformer +(45) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(43) ColumnarExchange +(48) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(49) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(45) InputAdapter +(50) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(46) InputIteratorTransformer +(51) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(47) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(48) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(49) ColumnarExchange +(54) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(55) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(51) InputAdapter +(56) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(52) InputIteratorTransformer +(57) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(53) SortExecTransformer +(58) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(54) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(55) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(56) Scan parquet +(61) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(57) Filter +(62) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(58) BroadcastExchange +(63) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(59) Scan parquet +(64) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(60) Filter +(65) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(61) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(62) Project +(67) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(63) Scan parquet +(68) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(64) Filter +(69) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(65) BroadcastExchange +(70) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(66) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(67) Project +(72) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(68) Scan parquet +(73) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(70) BroadcastExchange +(75) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(72) Project +(77) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(73) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(74) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(75) BroadcastExchange +(80) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(77) Project +(82) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(78) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(79) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(80) BroadcastExchange +(85) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(82) Project +(87) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(88) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(84) Exchange +(89) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(90) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(86) Exchange +(91) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Sort +(92) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(88) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt index caddcc769034..65906bb96691 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt @@ -1,117 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (121) +AdaptiveSparkPlan (129) +- == Final Plan == - VeloxColumnarToRowExec (76) - +- ^ SortExecTransformer (74) - +- ^ InputIteratorTransformer (73) - +- ^ InputAdapter (72) - +- ^ ShuffleQueryStage (71), Statistics(X) - +- ColumnarExchange (70) - +- ^ ProjectExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (67) - +- ^ InputIteratorTransformer (66) - +- ^ InputAdapter (65) - +- ^ ShuffleQueryStage (64), Statistics(X) - +- ColumnarExchange (63) - +- ^ ProjectExecTransformer (61) - +- ^ FlushableHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ BroadcastHashJoinExecTransformer Inner (58) - :- ^ ProjectExecTransformer (50) - : +- ^ BroadcastHashJoinExecTransformer Inner (49) - : :- ^ ProjectExecTransformer (42) - : : +- ^ BroadcastHashJoinExecTransformer Inner (41) - : : :- ^ ProjectExecTransformer (34) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : : : :- ^ ProjectExecTransformer (26) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : : : :- ^ ProjectExecTransformer (18) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : : : :- ^ ProjectExecTransformer (10) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : : : :- ^ InputIteratorTransformer (7) - : : : : : : : +- ^ InputAdapter (6) - : : : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (4) - : : : : : : : +- ^ ProjectExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ Scan parquet (8) - : : : : : +- ^ InputIteratorTransformer (16) - : : : : : +- ^ InputAdapter (15) - : : : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (13) - : : : : : +- ^ Scan parquet (11) - : : : : +- ^ InputIteratorTransformer (24) - : : : : +- ^ InputAdapter (23) - : : : : +- ^ BroadcastQueryStage (22), Statistics(X) - : : : : +- ColumnarBroadcastExchange (21) - : : : : +- ^ Scan parquet (19) - : : : +- ^ InputIteratorTransformer (32) - : : : +- ^ InputAdapter (31) - : : : +- ^ BroadcastQueryStage (30), Statistics(X) - : : : +- ColumnarBroadcastExchange (29) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (40) - : : +- ^ InputAdapter (39) - : : +- ^ BroadcastQueryStage (38), Statistics(X) - : : +- ColumnarBroadcastExchange (37) - : : +- ^ Scan parquet (35) - : +- ^ InputIteratorTransformer (48) - : +- ^ InputAdapter (47) - : +- ^ BroadcastQueryStage (46), Statistics(X) - : +- ColumnarBroadcastExchange (45) - : +- ^ Scan parquet (43) - +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ BroadcastQueryStage (55), Statistics(X) - +- ColumnarBroadcastExchange (54) - +- ^ ProjectExecTransformer (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (84) + +- ^ SortExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ^ InputAdapter (80) + +- ^ ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ^ InputAdapter (73) + +- ^ ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ^ InputAdapter (7) + : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- ^ InputAdapter (18) + : : : : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- ^ InputAdapter (27) + : : : : +- ^ BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- ^ InputAdapter (36) + : : : +- ^ BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- ^ InputAdapter (54) + : +- ^ BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- ^ InputAdapter (64) + +- ^ BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (120) - +- Exchange (119) - +- HashAggregate (118) - +- Exchange (117) - +- HashAggregate (116) - +- Project (115) - +- BroadcastHashJoin Inner BuildRight (114) - :- Project (109) - : +- BroadcastHashJoin Inner BuildRight (108) - : :- Project (104) - : : +- BroadcastHashJoin Inner BuildRight (103) - : : :- Project (99) - : : : +- BroadcastHashJoin Inner BuildRight (98) - : : : :- Project (94) - : : : : +- BroadcastHashJoin Inner BuildRight (93) - : : : : :- Project (89) - : : : : : +- BroadcastHashJoin Inner BuildRight (88) - : : : : : :- Project (84) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (83) - : : : : : : :- BroadcastExchange (80) - : : : : : : : +- Project (79) - : : : : : : : +- Filter (78) - : : : : : : : +- Scan parquet (77) - : : : : : : +- Filter (82) - : : : : : : +- Scan parquet (81) - : : : : : +- BroadcastExchange (87) - : : : : : +- Filter (86) - : : : : : +- Scan parquet (85) - : : : : +- BroadcastExchange (92) - : : : : +- Filter (91) - : : : : +- Scan parquet (90) - : : : +- BroadcastExchange (97) - : : : +- Filter (96) - : : : +- Scan parquet (95) - : : +- BroadcastExchange (102) - : : +- Filter (101) - : : +- Scan parquet (100) - : +- BroadcastExchange (107) - : +- Filter (106) - : +- Scan parquet (105) - +- BroadcastExchange (113) - +- Project (112) - +- Filter (111) - +- Scan parquet (110) + Sort (128) + +- Exchange (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- BroadcastHashJoin Inner BuildRight (122) + :- Project (117) + : +- BroadcastHashJoin Inner BuildRight (116) + : :- Project (112) + : : +- BroadcastHashJoin Inner BuildRight (111) + : : :- Project (107) + : : : +- BroadcastHashJoin Inner BuildRight (106) + : : : :- Project (102) + : : : : +- BroadcastHashJoin Inner BuildRight (101) + : : : : :- Project (97) + : : : : : +- BroadcastHashJoin Inner BuildRight (96) + : : : : : :- Project (92) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) + : : : : : : :- BroadcastExchange (88) + : : : : : : : +- Project (87) + : : : : : : : +- Filter (86) + : : : : : : : +- Scan parquet (85) + : : : : : : +- Filter (90) + : : : : : : +- Scan parquet (89) + : : : : : +- BroadcastExchange (95) + : : : : : +- Filter (94) + : : : : : +- Scan parquet (93) + : : : : +- BroadcastExchange (100) + : : : : +- Filter (99) + : : : : +- Scan parquet (98) + : : : +- BroadcastExchange (105) + : : : +- Filter (104) + : : : +- Scan parquet (103) + : : +- BroadcastExchange (110) + : : +- Filter (109) + : : +- Scan parquet (108) + : +- BroadcastExchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- BroadcastExchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -121,548 +129,580 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(3) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) Scan parquet +(13) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(15) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(19) Scan parquet +(22) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(24) WholeStageCodegenTransformer (X) Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(27) Scan parquet +(31) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(33) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(36) WholeStageCodegenTransformer (X) +(41) NoopFilter +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_regionkey#X] + +(42) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: false -(37) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(44) BroadcastQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(39) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(40) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(41) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(42) ProjectExecTransformer +(48) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(43) Scan parquet +(49) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(44) WholeStageCodegenTransformer (X) +(50) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(51) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(45) ColumnarBroadcastExchange +(52) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastQueryStage +(53) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(47) InputAdapter +(54) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(48) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastHashJoinExecTransformer +(56) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(50) ProjectExecTransformer +(57) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(51) Scan parquet +(58) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(52) ProjectExecTransformer +(59) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(60) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(53) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(54) ColumnarBroadcastExchange +(62) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(55) BroadcastQueryStage +(63) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(56) InputAdapter +(64) InputAdapter Input [1]: [r_regionkey#X] -(57) InputIteratorTransformer +(65) InputIteratorTransformer Input [1]: [r_regionkey#X] -(58) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(59) ProjectExecTransformer +(67) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(60) FlushableHashAggregateExecTransformer +(68) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(61) ProjectExecTransformer +(69) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(62) WholeStageCodegenTransformer (X) +(70) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(63) ColumnarExchange +(71) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(64) ShuffleQueryStage +(72) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(65) InputAdapter +(73) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(66) InputIteratorTransformer +(74) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(67) RegularHashAggregateExecTransformer +(75) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(68) ProjectExecTransformer +(76) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(69) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(70) ColumnarExchange +(78) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(71) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(72) InputAdapter +(80) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(73) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(74) SortExecTransformer +(82) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(75) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(76) VeloxColumnarToRowExec +(84) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(77) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(78) Filter +(86) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(79) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(80) BroadcastExchange +(88) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(81) Scan parquet +(89) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(82) Filter +(90) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(83) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(84) Project +(92) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(85) Scan parquet +(93) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(94) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(95) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(96) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(89) Project +(97) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(98) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(91) Filter +(99) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(92) BroadcastExchange +(100) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(93) BroadcastHashJoin +(101) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(94) Project +(102) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(95) Scan parquet +(103) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(104) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) BroadcastExchange +(105) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(98) BroadcastHashJoin +(106) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(99) Project +(107) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(100) Scan parquet +(108) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(101) Filter +(109) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(102) BroadcastExchange +(110) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(103) BroadcastHashJoin +(111) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(104) Project +(112) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(105) Scan parquet +(113) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(106) Filter +(114) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(107) BroadcastExchange +(115) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(108) BroadcastHashJoin +(116) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(109) Project +(117) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(110) Scan parquet +(118) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(111) Filter +(119) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(112) Project +(120) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(113) BroadcastExchange +(121) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(114) BroadcastHashJoin +(122) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(115) Project +(123) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(116) HashAggregate +(124) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(117) Exchange +(125) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) HashAggregate +(126) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(119) Exchange +(127) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Sort +(128) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(121) AdaptiveSparkPlan +(129) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt index 7f9173ddd5af..e7abd01744e5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt @@ -1,90 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (98) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ SortExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (26) - : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : :- ^ ProjectExecTransformer (18) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ BroadcastQueryStage (22), Statistics(X) - : : +- ColumnarBroadcastExchange (21) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ColumnarBroadcastExchange (29) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (64) + +- ^ SortExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ^ InputAdapter (60) + +- ^ ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- ^ InputAdapter (18) + : : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- ^ InputAdapter (36) + : +- ^ BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (91) - +- Exchange (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (76) - : : +- BroadcastHashJoin Inner BuildRight (75) - : : :- Project (71) - : : : +- BroadcastHashJoin Inner BuildRight (70) - : : : :- Project (66) - : : : : +- BroadcastHashJoin Inner BuildLeft (65) - : : : : :- BroadcastExchange (62) - : : : : : +- Project (61) - : : : : : +- Filter (60) - : : : : : +- Scan parquet (59) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- BroadcastExchange (74) - : : +- Filter (73) - : : +- Scan parquet (72) - : +- BroadcastExchange (79) - : +- Filter (78) - : +- Scan parquet (77) - +- BroadcastExchange (84) - +- Filter (83) - +- Scan parquet (82) + Sort (97) + +- Exchange (96) + +- HashAggregate (95) + +- Exchange (94) + +- HashAggregate (93) + +- Project (92) + +- BroadcastHashJoin Inner BuildRight (91) + :- Project (87) + : +- BroadcastHashJoin Inner BuildRight (86) + : :- Project (82) + : : +- BroadcastHashJoin Inner BuildRight (81) + : : :- Project (77) + : : : +- BroadcastHashJoin Inner BuildRight (76) + : : : :- Project (72) + : : : : +- BroadcastHashJoin Inner BuildLeft (71) + : : : : :- BroadcastExchange (68) + : : : : : +- Project (67) + : : : : : +- Filter (66) + : : : : : +- Scan parquet (65) + : : : : +- Filter (70) + : : : : +- Scan parquet (69) + : : : +- BroadcastExchange (75) + : : : +- Filter (74) + : : : +- Scan parquet (73) + : : +- BroadcastExchange (80) + : : +- Filter (79) + : : +- Scan parquet (78) + : +- BroadcastExchange (85) + : +- Filter (84) + : +- Scan parquet (83) + +- BroadcastExchange (90) + +- Filter (89) + +- Scan parquet (88) (1) Scan parquet @@ -94,416 +100,440 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(3) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) Scan parquet +(13) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(15) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(19) Scan parquet +(22) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(24) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(27) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_orderdate#X] + +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(36) WholeStageCodegenTransformer (X) +(41) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(42) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(37) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(44) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(39) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(40) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(41) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(42) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(43) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(50) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(52) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(53) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(54) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(55) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(56) SortExecTransformer +(62) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(59) Scan parquet +(65) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(60) Filter +(66) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(61) Project +(67) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(62) BroadcastExchange +(68) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) Scan parquet +(69) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(64) Filter +(70) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(65) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(66) Project +(72) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(67) Scan parquet +(73) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(68) Filter +(74) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(69) BroadcastExchange +(75) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(70) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(71) Project +(77) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(72) Scan parquet +(78) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(73) Filter +(79) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(74) BroadcastExchange +(80) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(75) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(76) Project +(82) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(77) Scan parquet +(83) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(78) Filter +(84) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(79) BroadcastExchange +(85) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(81) Project +(87) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(82) Scan parquet +(88) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(83) Filter +(89) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(84) BroadcastExchange +(90) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(85) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(86) Project +(92) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(87) HashAggregate +(93) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(88) Exchange +(94) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(95) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(90) Exchange +(96) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Sort +(97) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(92) AdaptiveSparkPlan +(98) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt index 699c17ea4562..8d67aad16c3d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (27) +AdaptiveSparkPlan (28) +- == Final Plan == - VeloxColumnarToRowExec (18) - +- ^ SortExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (19) + +- ^ SortExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (26) - +- Exchange (25) - +- HashAggregate (24) - +- Exchange (23) - +- HashAggregate (22) - +- Project (21) - +- Filter (20) - +- Scan parquet (19) + Sort (27) + +- Exchange (26) + +- HashAggregate (25) + +- Exchange (24) + +- HashAggregate (23) + +- Project (22) + +- Filter (21) + +- Scan parquet (20) (1) Scan parquet @@ -34,116 +35,120 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(4) ProjectExecTransformer +(5) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(6) ColumnarExchange +(7) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(12) ColumnarExchange +(13) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(14) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) InputAdapter +(15) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(15) InputIteratorTransformer +(16) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) SortExecTransformer +(17) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(17) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(18) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(19) Scan parquet +(20) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(20) Filter +(21) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(21) Project +(22) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(22) HashAggregate +(23) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(23) Exchange +(24) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(24) HashAggregate +(25) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(25) Exchange +(26) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Sort +(27) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(27) AdaptiveSparkPlan +(28) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt index 35c0d13f1e1e..9af231ec1bb8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt @@ -1,81 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (83) +AdaptiveSparkPlan (87) +- == Final Plan == - VeloxColumnarToRowExec (56) - +- TakeOrderedAndProjectExecTransformer (55) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ ShuffledHashJoinExecTransformer Inner (43) - :- ^ InputIteratorTransformer (35) - : +- ^ InputAdapter (34) - : +- ^ ShuffleQueryStage (33) - : +- ColumnarExchange (32) - : +- ^ ProjectExecTransformer (30) - : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : :- ^ InputIteratorTransformer (21) - : : +- ^ InputAdapter (20) - : : +- ^ ShuffleQueryStage (19) - : : +- ColumnarExchange (18) - : : +- ^ ProjectExecTransformer (16) - : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ ShuffleQueryStage (5) - : : : +- ColumnarExchange (4) - : : : +- ^ ProjectExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (14) - : : +- ^ InputAdapter (13) - : : +- ^ ShuffleQueryStage (12) - : : +- ColumnarExchange (11) - : : +- ^ ProjectExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (28) - : +- ^ InputAdapter (27) - : +- ^ ShuffleQueryStage (26) - : +- ColumnarExchange (25) - : +- ^ ProjectExecTransformer (23) - : +- ^ Scan parquet (22) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ ShuffleQueryStage (40) - +- ColumnarExchange (39) - +- ^ ProjectExecTransformer (37) - +- ^ Scan parquet (36) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner (47) + :- ^ InputIteratorTransformer (38) + : +- ^ InputAdapter (37) + : +- ^ ShuffleQueryStage (36) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : :- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ ShuffleQueryStage (21) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ ShuffleQueryStage (29) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (82) - +- HashAggregate (81) - +- Exchange (80) - +- HashAggregate (79) - +- Project (78) - +- ShuffledHashJoin Inner BuildRight (77) - :- Exchange (73) - : +- Project (72) - : +- ShuffledHashJoin Inner BuildRight (71) - : :- Exchange (66) - : : +- Project (65) - : : +- ShuffledHashJoin Inner BuildRight (64) - : : :- Exchange (59) - : : : +- Filter (58) - : : : +- Scan parquet (57) - : : +- Exchange (63) - : : +- Project (62) - : : +- Filter (61) - : : +- Scan parquet (60) - : +- Exchange (70) - : +- Project (69) - : +- Filter (68) - : +- Scan parquet (67) - +- Exchange (76) - +- Filter (75) - +- Scan parquet (74) + TakeOrderedAndProject (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- ShuffledHashJoin Inner BuildRight (81) + :- Exchange (77) + : +- Project (76) + : +- ShuffledHashJoin Inner BuildRight (75) + : :- Exchange (70) + : : +- Project (69) + : : +- ShuffledHashJoin Inner BuildRight (68) + : : :- Exchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- Exchange (67) + : : +- Project (66) + : : +- Filter (65) + : : +- Scan parquet (64) + : +- Exchange (74) + : +- Project (73) + : +- Filter (72) + : +- Scan parquet (71) + +- Exchange (80) + +- Filter (79) + +- Scan parquet (78) (1) Scan parquet @@ -85,354 +89,370 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(3) ProjectExecTransformer Output [8]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(41) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(45) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(50) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(52) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(53) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(54) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(55) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(53) ProjectExecTransformer +(57) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(54) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(55) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(56) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(57) Scan parquet +(61) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(58) Filter +(62) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(59) Exchange +(63) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(64) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(61) Filter +(65) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(62) Project +(66) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(63) Exchange +(67) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(68) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(65) Project +(69) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(66) Exchange +(70) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(67) Scan parquet +(71) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(68) Filter +(72) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(69) Project +(73) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(70) Exchange +(74) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(72) Project +(76) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(73) Exchange +(77) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(75) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(76) Exchange +(80) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(79) HashAggregate +(83) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(80) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(85) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(82) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(83) AdaptiveSparkPlan +(87) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt index 23cd40b681e6..f028b8daa9b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt @@ -1,68 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (72) +- == Final Plan == - VeloxColumnarToRowExec (47) - +- ^ SortExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ^ InputAdapter (43) - +- ^ ShuffleQueryStage (42) - +- ColumnarExchange (41) - +- ^ FilterExecTransformer (39) - +- ^ RegularHashAggregateExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ ShuffleQueryStage (35) - +- ColumnarExchange (34) - +- ^ ProjectExecTransformer (32) - +- ^ FlushableHashAggregateExecTransformer (31) - +- ^ ProjectExecTransformer (30) - +- ^ ShuffledHashJoinExecTransformer Inner (29) - :- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ ShuffleQueryStage (19) - : +- ColumnarExchange (18) - : +- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ Scan parquet (22) + VeloxColumnarToRowExec (50) + +- ^ SortExecTransformer (48) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ ShuffleQueryStage (45) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ ShuffleQueryStage (38) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - Sort (68) - +- Exchange (67) - +- Filter (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- ShuffledHashJoin Inner BuildRight (61) - :- Exchange (56) - : +- Project (55) - : +- ShuffledHashJoin Inner BuildRight (54) - : :- Exchange (50) - : : +- Filter (49) - : : +- Scan parquet (48) - : +- Exchange (53) - : +- Filter (52) - : +- Scan parquet (51) - +- Exchange (60) - +- Project (59) - +- Filter (58) - +- Scan parquet (57) + Sort (71) + +- Exchange (70) + +- Filter (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- ShuffledHashJoin Inner BuildRight (64) + :- Exchange (59) + : +- Project (58) + : +- ShuffledHashJoin Inner BuildRight (57) + : :- Exchange (53) + : : +- Filter (52) + : : +- Scan parquet (51) + : +- Exchange (56) + : +- Filter (55) + : +- Scan parquet (54) + +- Exchange (63) + +- Project (62) + +- Filter (61) + +- Scan parquet (60) (1) Scan parquet @@ -72,292 +75,304 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(3) ProjectExecTransformer Output [5]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) Scan parquet +(9) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(11) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(26) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [1]: [n_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [1]: [n_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(31) FlushableHashAggregateExecTransformer +(34) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(34) ColumnarExchange +(37) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(38) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(36) InputAdapter +(39) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(37) InputIteratorTransformer +(40) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(38) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(39) FilterExecTransformer +(42) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(40) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(41) ColumnarExchange +(44) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(45) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(43) InputAdapter +(46) InputAdapter Input [2]: [ps_partkey#X, value#X] -(44) InputIteratorTransformer +(47) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(45) SortExecTransformer +(48) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(46) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(47) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(48) Scan parquet +(51) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(49) Filter +(52) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(50) Exchange +(53) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(53) Exchange +(56) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(55) Project +(58) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(56) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Scan parquet +(60) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(58) Filter +(61) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(59) Project +(62) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(60) Exchange +(63) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) ShuffledHashJoin +(64) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(65) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(63) HashAggregate +(66) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(64) Exchange +(67) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(68) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(66) Filter +(69) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(67) Exchange +(70) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Sort +(71) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(69) AdaptiveSparkPlan +(72) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt index c5f7a17f4286..914b1813df44 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt @@ -1,48 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (47) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (46) - +- Exchange (45) - +- HashAggregate (44) - +- Exchange (43) - +- HashAggregate (42) - +- Project (41) - +- ShuffledHashJoin Inner BuildLeft (40) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (39) - +- Project (38) - +- Filter (37) - +- Scan parquet (36) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin Inner BuildLeft (42) + :- Exchange (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Exchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -52,200 +54,208 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(26) ColumnarExchange +(28) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(28) InputAdapter +(30) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(30) SortExecTransformer +(32) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(32) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(33) Scan parquet +(35) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(34) Filter +(36) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(35) Exchange +(37) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(38) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(37) Filter +(39) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(38) Project +(40) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(39) Exchange +(41) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(41) Project +(43) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(42) HashAggregate +(44) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(43) Exchange +(45) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) HashAggregate +(46) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(45) Exchange +(47) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(48) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(47) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt index 507966beca53..408b8a0f7f97 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (51) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (15) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5) : +- ColumnarExchange (4) : +- ^ ProjectExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (50) - +- Exchange (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin LeftOuter BuildRight (42) - :- Exchange (37) - : +- Scan parquet (36) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- HashAggregate (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftOuter BuildRight (43) + :- Exchange (38) + : +- Scan parquet (37) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -84,198 +85,202 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) ProjectExecTransformer +(9) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] + +(10) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(12) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(13) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(14) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(15) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(17) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(19) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(29) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(36) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Exchange +(38) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) HashAggregate +(46) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(46) HashAggregate +(47) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(47) Exchange +(48) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(49) Exchange +(50) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Sort +(51) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(51) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt index fd6ee41dbfa9..c1e2a1e52130 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (32) - +- HashAggregate (31) - +- Project (30) - +- ShuffledHashJoin Inner BuildRight (29) - :- Exchange (25) - : +- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- Exchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- HashAggregate (33) + +- Project (32) + +- ShuffledHashJoin Inner BuildRight (31) + :- Exchange (27) + : +- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- Exchange (30) + +- Filter (29) + +- Scan parquet (28) (1) Scan parquet @@ -40,148 +42,156 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) Scan parquet +(9) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(11) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(23) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(24) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(25) Exchange +(27) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Scan parquet +(28) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(29) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) Exchange +(30) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) ShuffledHashJoin +(31) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(32) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(33) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(33) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt index 66d138fef6d1..1da11e4fffea 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt @@ -1,45 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (43) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner (19) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FlushableHashAggregateExecTransformer (10) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ ShuffledHashJoinExecTransformer Inner (21) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (42) - +- Exchange (41) - +- Project (40) - +- ShuffledHashJoin Inner BuildLeft (39) - :- Exchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Filter (38) - +- HashAggregate (37) - +- Exchange (36) - +- HashAggregate (35) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (44) + +- Exchange (43) + +- Project (42) + +- ShuffledHashJoin Inner BuildLeft (41) + :- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -49,186 +51,194 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(11) ProjectExecTransformer +(13) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(19) ShuffledHashJoinExecTransformer +(21) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(26) SortExecTransformer +(28) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(28) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(29) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(30) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(31) Exchange +(33) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(32) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(33) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(34) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(35) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(36) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(38) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(39) ShuffledHashJoin +(41) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(40) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(41) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(43) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt index 8acd437dd729..4a8b56e41f36 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt @@ -1,62 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (62) +AdaptiveSparkPlan (64) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ SortExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ ShuffleQueryStage (35) - +- ColumnarExchange (34) - +- ^ RegularHashAggregateExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FlushableHashAggregateExecTransformer (25) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (42) + +- ^ SortExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ^ InputAdapter (38) + +- ^ ShuffleQueryStage (37) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (61) - +- Exchange (60) - +- HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- ShuffledHashJoin Inner BuildRight (52) - :- Exchange (48) - : +- BroadcastHashJoin LeftAnti BuildRight (47) - : :- Filter (42) - : : +- Scan parquet (41) - : +- BroadcastExchange (46) - : +- Project (45) - : +- Filter (44) - : +- Scan parquet (43) - +- Exchange (51) - +- Filter (50) - +- Scan parquet (49) + Sort (63) + +- Exchange (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- HashAggregate (58) + +- Exchange (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (50) + : +- BroadcastHashJoin LeftAnti BuildRight (49) + : :- Filter (44) + : : +- Scan parquet (43) + : +- BroadcastExchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Exchange (53) + +- Filter (52) + +- Scan parquet (51) (1) Scan parquet @@ -66,274 +68,282 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X, ps_suppkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Input [2]: [ps_partkey#X, ps_suppkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) FlushableHashAggregateExecTransformer +(27) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(28) ColumnarExchange +(30) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(30) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(32) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(33) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(34) ColumnarExchange +(36) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(37) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(36) InputAdapter +(38) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(37) InputIteratorTransformer +(39) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(38) SortExecTransformer +(40) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(39) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(40) VeloxColumnarToRowExec +(42) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(41) Scan parquet +(43) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(43) Scan parquet +(45) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(44) Filter +(46) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(45) Project +(47) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(46) BroadcastExchange +(48) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(47) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(48) Exchange +(50) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Scan parquet +(51) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(50) Filter +(52) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(51) Exchange +(53) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(53) Project +(55) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(54) HashAggregate +(56) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(55) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(58) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) HashAggregate +(59) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(58) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(60) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(63) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(62) AdaptiveSparkPlan +(64) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt index a759dc4f95c1..523c081b1d19 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ ProjectExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ ShuffledHashJoinExecTransformer Inner (28) - :- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ FilterExecTransformer (27) - +- ^ ProjectExecTransformer (26) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ Scan parquet (17) + VeloxColumnarToRowExec (37) + +- ^ ProjectExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ RegularHashAggregateExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ ShuffledHashJoinExecTransformer Inner (31) + :- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ FilterExecTransformer (30) + +- ^ ProjectExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ NoopFilter (20) + +- ^ Scan parquet (19) +- == Initial Plan == - HashAggregate (53) - +- HashAggregate (52) - +- Project (51) - +- ShuffledHashJoin Inner BuildRight (50) - :- Project (43) - : +- ShuffledHashJoin Inner BuildRight (42) - : :- Exchange (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Exchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Filter (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- Filter (45) - +- Scan parquet (44) + HashAggregate (56) + +- HashAggregate (55) + +- Project (54) + +- ShuffledHashJoin Inner BuildRight (53) + :- Project (46) + : +- ShuffledHashJoin Inner BuildRight (45) + : :- Exchange (40) + : : +- Filter (39) + : : +- Scan parquet (38) + : +- Exchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- Filter (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -60,250 +63,262 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X] + +(3) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [p_partkey#X, p_brand#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_container#X] + +(11) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [1]: [p_partkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [p_partkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) NoopFilter +Input [2]: [l_partkey#X, l_quantity#X] +Arguments: [l_partkey#X, l_quantity#X] + +(21) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(20) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(21) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(27) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(28) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(30) RegularHashAggregateExecTransformer +(33) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(37) Exchange +(40) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(40) Project +(43) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(41) Exchange +(44) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(43) Project +(46) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(44) Scan parquet +(47) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(45) Filter +(48) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(46) HashAggregate +(49) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(47) Exchange +(50) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(51) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(49) Filter +(52) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(50) ShuffledHashJoin +(53) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(51) Project +(54) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) HashAggregate +(55) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(53) HashAggregate +(56) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(54) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt index 7a6e11258914..6045020cb49a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt @@ -1,93 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (97) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- TakeOrderedAndProjectExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ ProjectExecTransformer (56) - +- ^ ShuffledHashJoinExecTransformer Inner (55) - :- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ ShuffleQueryStage (37) - : +- ColumnarExchange (36) - : +- ^ ProjectExecTransformer (34) - : +- ^ ShuffledHashJoinExecTransformer Inner (33) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ ShuffleQueryStage (30) - : +- ColumnarExchange (29) - : +- ^ ProjectExecTransformer (27) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (26) - : :- ^ InputIteratorTransformer (14) - : : +- ^ InputAdapter (13) - : : +- ^ ShuffleQueryStage (12) - : : +- ColumnarExchange (11) - : : +- ^ ProjectExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ ProjectExecTransformer (25) - : +- ^ FilterExecTransformer (24) - : +- ^ RegularHashAggregateExecTransformer (23) - : +- ^ InputIteratorTransformer (22) - : +- ^ InputAdapter (21) - : +- ^ ShuffleQueryStage (20) - : +- ColumnarExchange (19) - : +- ^ ProjectExecTransformer (17) - : +- ^ FlushableHashAggregateExecTransformer (16) - : +- ^ Scan parquet (15) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (54) - :- ^ InputIteratorTransformer (46) - : +- ^ InputAdapter (45) - : +- ^ ShuffleQueryStage (44) - : +- ColumnarExchange (43) - : +- ^ ProjectExecTransformer (41) - : +- ^ Scan parquet (40) - +- ^ ProjectExecTransformer (53) - +- ^ FilterExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48) - +- ReusedExchange (47) + VeloxColumnarToRowExec (64) + +- TakeOrderedAndProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner (58) + :- ^ InputIteratorTransformer (41) + : +- ^ InputAdapter (40) + : +- ^ ShuffleQueryStage (39) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ^ InputAdapter (33) + : +- ^ ShuffleQueryStage (32) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : :- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ^ InputAdapter (23) + : +- ^ ShuffleQueryStage (22) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + :- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ ShuffleQueryStage (47) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (93) - +- HashAggregate (92) - +- HashAggregate (91) - +- Project (90) - +- ShuffledHashJoin Inner BuildRight (89) - :- Exchange (78) - : +- Project (77) - : +- ShuffledHashJoin Inner BuildLeft (76) - : :- Exchange (64) - : : +- Filter (63) - : : +- Scan parquet (62) - : +- Exchange (75) - : +- ShuffledHashJoin LeftSemi BuildRight (74) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Project (73) - : +- Filter (72) - : +- HashAggregate (71) - : +- Exchange (70) - : +- HashAggregate (69) - : +- Scan parquet (68) - +- ShuffledHashJoin LeftSemi BuildRight (88) - :- Exchange (81) - : +- Filter (80) - : +- Scan parquet (79) - +- Project (87) - +- Filter (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Scan parquet (82) + TakeOrderedAndProject (96) + +- HashAggregate (95) + +- HashAggregate (94) + +- Project (93) + +- ShuffledHashJoin Inner BuildRight (92) + :- Exchange (81) + : +- Project (80) + : +- ShuffledHashJoin Inner BuildLeft (79) + : :- Exchange (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Exchange (78) + : +- ShuffledHashJoin LeftSemi BuildRight (77) + : :- Exchange (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- Project (76) + : +- Filter (75) + : +- HashAggregate (74) + : +- Exchange (73) + : +- HashAggregate (72) + : +- Scan parquet (71) + +- ShuffledHashJoin LeftSemi BuildRight (91) + :- Exchange (84) + : +- Filter (83) + : +- Scan parquet (82) + +- Project (90) + +- Filter (89) + +- HashAggregate (88) + +- Exchange (87) + +- HashAggregate (86) + +- Scan parquet (85) (1) Scan parquet @@ -97,420 +100,432 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X, c_name#X] + +(3) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X] Input [2]: [c_custkey#X, c_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(16) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(19) ColumnarExchange +(21) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(20) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(21) InputAdapter +(23) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(22) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(23) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(24) FilterExecTransformer +(26) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(25) ProjectExecTransformer +(27) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(26) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(29) ColumnarExchange +(31) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(32) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(31) InputAdapter +(33) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(32) InputIteratorTransformer +(34) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(33) ShuffledHashJoinExecTransformer +(35) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(34) ProjectExecTransformer +(36) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(36) ColumnarExchange +(38) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(39) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(38) InputAdapter +(40) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(39) InputIteratorTransformer +(41) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(40) Scan parquet +(42) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(41) ProjectExecTransformer +(43) NoopFilter +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X, l_quantity#X] + +(44) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(42) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(43) ColumnarExchange +(46) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(47) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(45) InputAdapter +(48) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(46) InputIteratorTransformer +(49) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(47) ReusedExchange [Reuses operator id: 19] +(50) ReusedExchange [Reuses operator id: 21] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(48) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(49) InputAdapter +(52) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(50) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(52) FilterExecTransformer +(55) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(53) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(54) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) ShuffledHashJoinExecTransformer +(58) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(59) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(57) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(58) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(59) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(60) TakeOrderedAndProjectExecTransformer +(63) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(61) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(62) Scan parquet +(65) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(63) Filter +(66) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(64) Exchange +(67) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(68) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(66) Filter +(69) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(67) Exchange +(70) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(69) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(70) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(72) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(73) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(74) ShuffledHashJoin +(77) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(75) Exchange +(78) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) ShuffledHashJoin +(79) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(77) Project +(80) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(78) Exchange +(81) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Scan parquet +(82) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(80) Filter +(83) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(81) Exchange +(84) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(85) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(83) HashAggregate +(86) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(84) Exchange +(87) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(88) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(86) Filter +(89) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(87) Project +(90) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(88) ShuffledHashJoin +(91) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(89) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(90) Project +(93) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(91) HashAggregate +(94) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(92) HashAggregate +(95) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(93) TakeOrderedAndProject +(96) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(94) AdaptiveSparkPlan +(97) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt index 260269c9eaef..7084e61f61d5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (32) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (31) - +- HashAggregate (30) - +- Project (29) - +- ShuffledHashJoin Inner BuildRight (28) - :- Exchange (24) - : +- Project (23) - : +- Filter (22) - : +- Scan parquet (21) - +- Exchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- HashAggregate (32) + +- Project (31) + +- ShuffledHashJoin Inner BuildRight (30) + :- Exchange (26) + : +- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- Exchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -39,144 +41,152 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(3) ProjectExecTransformer Output [5]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(20) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(21) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(22) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(23) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(24) Exchange +(26) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) Scan parquet +(27) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(28) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) Exchange +(29) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) ShuffledHashJoin +(30) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(31) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(32) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(32) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt index 2071bbf73f12..3f6d17512477 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt @@ -1,116 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (121) +AdaptiveSparkPlan (126) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ SortExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ ShuffledHashJoinExecTransformer Inner (72) - :- ^ InputIteratorTransformer (64) - : +- ^ InputAdapter (63) - : +- ^ ShuffleQueryStage (62) - : +- ColumnarExchange (61) - : +- ^ ProjectExecTransformer (59) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (58) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (57) - : +- ^ InputAdapter (56) - : +- ^ ShuffleQueryStage (55) - : +- ColumnarExchange (54) - : +- ^ ProjectExecTransformer (52) - : +- ^ ShuffledHashJoinExecTransformer Inner (51) - : :- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ ShuffleQueryStage (26) - : : +- ColumnarExchange (25) - : : +- ^ ProjectExecTransformer (23) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (22) - : : :- ^ InputIteratorTransformer (14) - : : : +- ^ InputAdapter (13) - : : : +- ^ ShuffleQueryStage (12) - : : : +- ColumnarExchange (11) - : : : +- ^ ProjectExecTransformer (9) - : : : +- ^ Scan parquet (8) - : : +- ^ InputIteratorTransformer (21) - : : +- ^ InputAdapter (20) - : : +- ^ ShuffleQueryStage (19) - : : +- ColumnarExchange (18) - : : +- ^ ProjectExecTransformer (16) - : : +- ^ Scan parquet (15) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ ShuffleQueryStage (48) - : +- ColumnarExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ FilterExecTransformer (44) - : +- ^ ProjectExecTransformer (43) - : +- ^ RegularHashAggregateExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (40) - : :- ^ InputIteratorTransformer (35) - : : +- ^ InputAdapter (34) - : : +- ^ ShuffleQueryStage (33) - : : +- ColumnarExchange (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ Scan parquet (29) - : +- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ ShuffleQueryStage (37) - : +- ReusedExchange (36) - +- ^ InputIteratorTransformer (71) - +- ^ InputAdapter (70) - +- ^ ShuffleQueryStage (69) - +- ColumnarExchange (68) - +- ^ ProjectExecTransformer (66) - +- ^ Scan parquet (65) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ^ InputAdapter (82) + +- ^ ShuffleQueryStage (81) + +- ColumnarExchange (80) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (31) + : : +- ^ InputAdapter (30) + : : +- ^ ShuffleQueryStage (29) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ ShuffleQueryStage (14) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ NoopFilter (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ^ InputAdapter (23) + : : +- ^ ShuffleQueryStage (22) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ NoopFilter (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) + : :- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ NoopFilter (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ^ InputAdapter (42) + : +- ^ ShuffleQueryStage (41) + : +- ReusedExchange (40) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (120) - +- Exchange (119) - +- Project (118) - +- ShuffledHashJoin Inner BuildRight (117) - :- Exchange (112) - : +- Project (111) - : +- ShuffledHashJoin LeftSemi BuildRight (110) - : :- Exchange (84) - : : +- Filter (83) - : : +- Scan parquet (82) - : +- Exchange (109) - : +- Project (108) - : +- ShuffledHashJoin Inner BuildLeft (107) - : :- Exchange (93) - : : +- ShuffledHashJoin LeftSemi BuildRight (92) - : : :- Exchange (87) - : : : +- Filter (86) - : : : +- Scan parquet (85) - : : +- Exchange (91) - : : +- Project (90) - : : +- Filter (89) - : : +- Scan parquet (88) - : +- Exchange (106) - : +- Filter (105) - : +- HashAggregate (104) - : +- HashAggregate (103) - : +- ShuffledHashJoin LeftSemi BuildRight (102) - : :- Exchange (97) - : : +- Project (96) - : : +- Filter (95) - : : +- Scan parquet (94) - : +- Exchange (101) - : +- Project (100) - : +- Filter (99) - : +- Scan parquet (98) - +- Exchange (116) - +- Project (115) - +- Filter (114) - +- Scan parquet (113) + Sort (125) + +- Exchange (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin LeftSemi BuildRight (115) + : :- Exchange (89) + : : +- Filter (88) + : : +- Scan parquet (87) + : +- Exchange (114) + : +- Project (113) + : +- ShuffledHashJoin Inner BuildLeft (112) + : :- Exchange (98) + : : +- ShuffledHashJoin LeftSemi BuildRight (97) + : : :- Exchange (92) + : : : +- Filter (91) + : : : +- Scan parquet (90) + : : +- Exchange (96) + : : +- Project (95) + : : +- Filter (94) + : : +- Scan parquet (93) + : +- Exchange (111) + : +- Filter (110) + : +- HashAggregate (109) + : +- HashAggregate (108) + : +- ShuffledHashJoin LeftSemi BuildRight (107) + : :- Exchange (102) + : : +- Project (101) + : : +- Filter (100) + : : +- Scan parquet (99) + : +- Exchange (106) + : +- Project (105) + : +- Filter (104) + : +- Scan parquet (103) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -120,510 +125,530 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(11) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(16) ProjectExecTransformer +(18) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(19) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(17) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(18) ColumnarExchange +(21) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(22) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(20) InputAdapter +(23) InputAdapter Input [1]: [p_partkey#X] -(21) InputIteratorTransformer +(24) InputIteratorTransformer Input [1]: [p_partkey#X] -(22) ShuffledHashJoinExecTransformer +(25) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(23) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(29) Scan parquet +(32) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(30) ProjectExecTransformer +(33) NoopFilter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(34) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(31) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(32) ColumnarExchange +(36) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(37) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(34) InputAdapter +(38) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(35) InputIteratorTransformer +(39) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(36) ReusedExchange [Reuses operator id: 18] +(40) ReusedExchange [Reuses operator id: 21] Output [1]: [p_partkey#X] -(37) ShuffleQueryStage +(41) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(38) InputAdapter +(42) InputAdapter Input [1]: [p_partkey#X] -(39) InputIteratorTransformer +(43) InputIteratorTransformer Input [1]: [p_partkey#X] -(40) ShuffledHashJoinExecTransformer +(44) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(42) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) ProjectExecTransformer +(47) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(44) FilterExecTransformer +(48) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(47) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(49) InputAdapter +(53) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(51) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(52) ProjectExecTransformer +(56) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(53) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(54) ColumnarExchange +(58) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(56) InputAdapter +(60) InputAdapter Input [1]: [ps_suppkey#X] -(57) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(58) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(59) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(60) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(61) ColumnarExchange +(65) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(62) ShuffleQueryStage +(66) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(67) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(64) InputIteratorTransformer +(68) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(65) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(66) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(71) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(67) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(68) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(69) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(70) InputAdapter +(75) InputAdapter Input [1]: [n_nationkey#X] -(71) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [n_nationkey#X] -(72) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(73) ProjectExecTransformer +(78) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(74) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(75) ColumnarExchange +(80) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(77) InputAdapter +(82) InputAdapter Input [2]: [s_name#X, s_address#X] -(78) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(79) SortExecTransformer +(84) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(80) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(81) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(82) Scan parquet +(87) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(84) Exchange +(89) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(86) Filter +(91) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(87) Exchange +(92) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Scan parquet +(93) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(89) Filter +(94) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(90) Project +(95) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(91) Exchange +(96) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(93) Exchange +(98) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(99) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(95) Filter +(100) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(96) Project +(101) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(97) Exchange +(102) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(103) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(99) Filter +(104) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(100) Project +(105) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(101) Exchange +(106) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(103) HashAggregate +(108) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(104) HashAggregate +(109) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(105) Filter +(110) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(106) Exchange +(111) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(112) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(108) Project +(113) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(109) Exchange +(114) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(111) Project +(116) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(112) Exchange +(117) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(115) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(116) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(118) Project +(123) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(119) Exchange +(124) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Sort +(125) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(121) AdaptiveSparkPlan +(126) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt index c4f7c527a056..b7067f8b5e3a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt @@ -1,109 +1,114 @@ == Physical Plan == -AdaptiveSparkPlan (114) +AdaptiveSparkPlan (119) +- == Final Plan == - VeloxColumnarToRowExec (77) - +- TakeOrderedAndProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (74) - +- ^ InputIteratorTransformer (73) - +- ^ InputAdapter (72) - +- ^ ShuffleQueryStage (71) - +- ColumnarExchange (70) - +- ^ ProjectExecTransformer (68) - +- ^ FlushableHashAggregateExecTransformer (67) - +- ^ ProjectExecTransformer (66) - +- ^ ShuffledHashJoinExecTransformer Inner (65) - :- ^ InputIteratorTransformer (57) - : +- ^ InputAdapter (56) - : +- ^ ShuffleQueryStage (55) - : +- ColumnarExchange (54) - : +- ^ ProjectExecTransformer (52) - : +- ^ ShuffledHashJoinExecTransformer Inner (51) - : :- ^ InputIteratorTransformer (43) - : : +- ^ InputAdapter (42) - : : +- ^ ShuffleQueryStage (41) - : : +- ColumnarExchange (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer Inner (37) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ ShuffleQueryStage (5) - : : : +- ColumnarExchange (4) - : : : +- ^ ProjectExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (36) - : : +- ^ InputAdapter (35) - : : +- ^ ShuffleQueryStage (34) - : : +- ColumnarExchange (33) - : : +- ^ ProjectExecTransformer (31) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (30) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (22) - : : : :- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (21) - : : : +- ^ InputAdapter (20) - : : : +- ^ ShuffleQueryStage (19) - : : : +- ColumnarExchange (18) - : : : +- ^ ProjectExecTransformer (16) - : : : +- ^ Scan parquet (15) - : : +- ^ InputIteratorTransformer (29) - : : +- ^ InputAdapter (28) - : : +- ^ ShuffleQueryStage (27) - : : +- ColumnarExchange (26) - : : +- ^ ProjectExecTransformer (24) - : : +- ^ Scan parquet (23) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ ShuffleQueryStage (48) - : +- ColumnarExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ Scan parquet (44) - +- ^ InputIteratorTransformer (64) - +- ^ InputAdapter (63) - +- ^ ShuffleQueryStage (62) - +- ColumnarExchange (61) - +- ^ ProjectExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (82) + +- TakeOrderedAndProjectExecTransformer (81) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ^ InputAdapter (77) + +- ^ ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner (70) + :- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ^ InputAdapter (22) + : : : +- ^ ShuffleQueryStage (21) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ^ InputAdapter (31) + : : +- ^ ShuffleQueryStage (30) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ NoopFilter (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ^ InputAdapter (68) + +- ^ ShuffleQueryStage (67) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ NoopFilter (63) + +- ^ Scan parquet (62) +- == Initial Plan == - TakeOrderedAndProject (113) - +- HashAggregate (112) - +- Exchange (111) - +- HashAggregate (110) - +- Project (109) - +- ShuffledHashJoin Inner BuildRight (108) - :- Exchange (103) - : +- Project (102) - : +- ShuffledHashJoin Inner BuildRight (101) - : :- Exchange (96) - : : +- Project (95) - : : +- ShuffledHashJoin Inner BuildLeft (94) - : : :- Exchange (80) - : : : +- Filter (79) - : : : +- Scan parquet (78) - : : +- Exchange (93) - : : +- ShuffledHashJoin LeftAnti BuildRight (92) - : : :- ShuffledHashJoin LeftSemi BuildRight (87) - : : : :- Exchange (84) - : : : : +- Project (83) - : : : : +- Filter (82) - : : : : +- Scan parquet (81) - : : : +- Exchange (86) - : : : +- Scan parquet (85) - : : +- Exchange (91) - : : +- Project (90) - : : +- Filter (89) - : : +- Scan parquet (88) - : +- Exchange (100) - : +- Project (99) - : +- Filter (98) - : +- Scan parquet (97) - +- Exchange (107) - +- Project (106) - +- Filter (105) - +- Scan parquet (104) + TakeOrderedAndProject (118) + +- HashAggregate (117) + +- Exchange (116) + +- HashAggregate (115) + +- Project (114) + +- ShuffledHashJoin Inner BuildRight (113) + :- Exchange (108) + : +- Project (107) + : +- ShuffledHashJoin Inner BuildRight (106) + : :- Exchange (101) + : : +- Project (100) + : : +- ShuffledHashJoin Inner BuildLeft (99) + : : :- Exchange (85) + : : : +- Filter (84) + : : : +- Scan parquet (83) + : : +- Exchange (98) + : : +- ShuffledHashJoin LeftAnti BuildRight (97) + : : :- ShuffledHashJoin LeftSemi BuildRight (92) + : : : :- Exchange (89) + : : : : +- Project (88) + : : : : +- Filter (87) + : : : : +- Scan parquet (86) + : : : +- Exchange (91) + : : : +- Scan parquet (90) + : : +- Exchange (96) + : : +- Project (95) + : : +- Filter (94) + : : +- Scan parquet (93) + : +- Exchange (105) + : +- Project (104) + : +- Filter (103) + : +- Scan parquet (102) + +- Exchange (112) + +- Project (111) + +- Filter (110) + +- Scan parquet (109) (1) Scan parquet @@ -113,486 +118,506 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [4]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(22) ShuffledHashJoinExecTransformer +(24) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(23) Scan parquet +(25) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(24) ProjectExecTransformer +(26) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(27) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(25) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(26) ColumnarExchange +(29) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(28) InputAdapter +(31) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(29) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(30) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(31) ProjectExecTransformer +(34) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(33) ColumnarExchange +(36) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(37) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(35) InputAdapter +(38) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(36) InputIteratorTransformer +(39) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(37) ShuffledHashJoinExecTransformer +(40) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(38) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(40) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(41) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(42) InputAdapter +(45) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(43) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(44) Scan parquet +(47) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(45) ProjectExecTransformer +(48) NoopFilter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: [o_orderkey#X, o_orderstatus#X] + +(49) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(46) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(47) ColumnarExchange +(51) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(52) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(49) InputAdapter +(53) InputAdapter Input [1]: [o_orderkey#X] -(50) InputIteratorTransformer +(54) InputIteratorTransformer Input [1]: [o_orderkey#X] -(51) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(52) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(53) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(54) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(56) InputAdapter +(60) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(57) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(58) Scan parquet +(62) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(59) ProjectExecTransformer +(63) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(64) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(60) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(61) ColumnarExchange +(66) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(62) ShuffleQueryStage +(67) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(63) InputAdapter +(68) InputAdapter Input [1]: [n_nationkey#X] -(64) InputIteratorTransformer +(69) InputIteratorTransformer Input [1]: [n_nationkey#X] -(65) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(66) ProjectExecTransformer +(71) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(67) FlushableHashAggregateExecTransformer +(72) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(68) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(69) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(70) ColumnarExchange +(75) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(71) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(72) InputAdapter +(77) InputAdapter Input [2]: [s_name#X, count#X] -(73) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(74) RegularHashAggregateExecTransformer +(79) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(75) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(76) TakeOrderedAndProjectExecTransformer +(81) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(77) VeloxColumnarToRowExec +(82) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(78) Scan parquet +(83) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(79) Filter +(84) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(80) Exchange +(85) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) Scan parquet +(86) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(82) Filter +(87) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(83) Project +(88) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) Exchange +(91) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(88) Scan parquet +(93) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(89) Filter +(94) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(90) Project +(95) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(91) Exchange +(96) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(93) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(99) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(95) Project +(100) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(96) Exchange +(101) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Scan parquet +(102) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(98) Filter +(103) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(99) Project +(104) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(100) Exchange +(105) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(107) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(103) Exchange +(108) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(105) Filter +(110) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(106) Project +(111) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(107) Exchange +(112) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(109) Project +(114) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(110) HashAggregate +(115) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(111) Exchange +(116) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) HashAggregate +(117) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(113) TakeOrderedAndProject +(118) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(114) AdaptiveSparkPlan +(119) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt index 0f13ad3079c9..dc38dbb7675a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt @@ -1,46 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (33) + +- ^ SortExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ^ InputAdapter (29) + +- ^ ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- ShuffledHashJoin LeftAnti BuildRight (38) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (37) - +- Scan parquet (36) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- ShuffledHashJoin LeftAnti BuildRight (39) + :- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Exchange (38) + +- Scan parquet (37) (1) Scan parquet @@ -50,190 +51,194 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X, c_phone#X, c_acctbal#X] + +(3) ProjectExecTransformer Output [4]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) Scan parquet +(9) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(9) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(12) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(13) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(13) InputAdapter +(14) InputAdapter Input [1]: [o_custkey#X] -(14) InputIteratorTransformer +(15) InputIteratorTransformer Input [1]: [o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(17) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(20) ColumnarExchange +(21) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) ColumnarExchange +(27) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) SortExecTransformer +(31) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(32) VeloxColumnarToRowExec +(33) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(35) Exchange +(36) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(37) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Exchange +(38) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) ShuffledHashJoin +(39) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(39) Project +(40) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(40) HashAggregate +(41) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(41) Exchange +(42) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(43) Exchange +(44) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt index 318b9bdf6a46..d6c6f13e9063 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt @@ -1,57 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- TakeOrderedAndProjectExecTransformer (35) - +- ^ ProjectExecTransformer (33) - +- ^ RegularHashAggregateExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ ProjectExecTransformer (30) - +- ^ ShuffledHashJoinExecTransformer Inner (29) - :- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ ShuffleQueryStage (19) - : +- ColumnarExchange (18) - : +- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ Scan parquet (22) + VeloxColumnarToRowExec (39) + +- TakeOrderedAndProjectExecTransformer (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (55) - +- HashAggregate (54) - +- HashAggregate (53) - +- Project (52) - +- ShuffledHashJoin Inner BuildRight (51) - :- Exchange (46) - : +- Project (45) - : +- ShuffledHashJoin Inner BuildLeft (44) - : :- Exchange (40) - : : +- Project (39) - : : +- Filter (38) - : : +- Scan parquet (37) - : +- Exchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Exchange (50) - +- Project (49) - +- Filter (48) - +- Scan parquet (47) + TakeOrderedAndProject (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (49) + : +- Project (48) + : +- ShuffledHashJoin Inner BuildLeft (47) + : :- Exchange (43) + : : +- Project (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Exchange (53) + +- Project (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -61,244 +64,256 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: [c_custkey#X, c_mktsegment#X] + +(3) ProjectExecTransformer Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(11) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(32) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(33) ProjectExecTransformer +(36) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(35) TakeOrderedAndProjectExecTransformer +(38) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(37) Scan parquet +(40) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(38) Filter +(41) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(39) Project +(42) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(40) Exchange +(43) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(44) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(42) Filter +(45) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(43) Exchange +(46) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(45) Project +(48) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(46) Exchange +(49) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Scan parquet +(50) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(48) Filter +(51) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(49) Project +(52) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(50) Exchange +(53) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(52) Project +(55) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(53) HashAggregate +(56) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(54) HashAggregate +(57) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(55) TakeOrderedAndProject +(58) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(56) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt index 807c29568ead..4c33fb73f757 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt @@ -1,49 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (48) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (47) - +- Exchange (46) - +- HashAggregate (45) - +- Exchange (44) - +- HashAggregate (43) - +- Project (42) - +- ShuffledHashJoin LeftSemi BuildRight (41) - :- Exchange (36) - : +- Project (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftSemi BuildRight (43) + :- Exchange (38) + : +- Project (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -53,204 +55,212 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [1]: [l_orderkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [l_orderkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(26) ColumnarExchange +(28) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(28) InputAdapter +(30) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(29) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(30) SortExecTransformer +(32) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(32) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(33) Scan parquet +(35) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(34) Filter +(36) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(35) Project +(37) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(36) Exchange +(38) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(39) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(38) Filter +(40) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(39) Project +(41) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(40) Exchange +(42) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) Project +(44) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(43) HashAggregate +(45) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(44) Exchange +(46) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(46) Exchange +(48) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Sort +(49) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(48) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt index fc394df77026..8bc2587dbaee 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt @@ -1,121 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (134) +- == Final Plan == - VeloxColumnarToRowExec (88) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ^ InputAdapter (78) - +- ^ ShuffleQueryStage (77) - +- ColumnarExchange (76) - +- ^ ProjectExecTransformer (74) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ ShuffledHashJoinExecTransformer Inner (71) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ ShuffleQueryStage (68) - +- ColumnarExchange (67) - +- ^ ProjectExecTransformer (65) - +- ^ Scan parquet (64) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildRight (108) - : : :- Exchange (104) - : : : +- Project (103) - : : : +- ShuffledHashJoin Inner BuildRight (102) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- ShuffledHashJoin Inner BuildLeft (96) - : : : : :- Exchange (91) - : : : : : +- Filter (90) - : : : : : +- Scan parquet (89) - : : : : +- Exchange (95) - : : : : +- Project (94) - : : : : +- Filter (93) - : : : : +- Scan parquet (92) - : : : +- Exchange (101) - : : : +- Filter (100) - : : : +- Scan parquet (99) - : : +- Exchange (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Exchange (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (120) - +- Project (119) - +- Filter (118) - +- Scan parquet (117) + Sort (133) + +- Exchange (132) + +- HashAggregate (131) + +- Exchange (130) + +- HashAggregate (129) + +- Project (128) + +- ShuffledHashJoin Inner BuildRight (127) + :- Exchange (122) + : +- Project (121) + : +- ShuffledHashJoin Inner BuildRight (120) + : :- Exchange (116) + : : +- Project (115) + : : +- ShuffledHashJoin Inner BuildRight (114) + : : :- Exchange (110) + : : : +- Project (109) + : : : +- ShuffledHashJoin Inner BuildRight (108) + : : : :- Exchange (104) + : : : : +- Project (103) + : : : : +- ShuffledHashJoin Inner BuildLeft (102) + : : : : :- Exchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Exchange (101) + : : : : +- Project (100) + : : : : +- Filter (99) + : : : : +- Scan parquet (98) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Exchange (126) + +- Project (125) + +- Filter (124) + +- Scan parquet (123) (1) Scan parquet @@ -125,540 +131,564 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(26) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(41) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(50) Scan parquet +(54) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] + +(56) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(71) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [1]: [r_regionkey#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [r_regionkey#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(73) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(74) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(82) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(83) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(84) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(79) InputIteratorTransformer +(85) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(81) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(82) ColumnarExchange +(88) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(84) InputAdapter +(90) InputAdapter Input [2]: [n_name#X, revenue#X] -(85) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(86) SortExecTransformer +(92) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(89) Scan parquet +(95) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(90) Filter +(96) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(91) Exchange +(97) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Scan parquet +(98) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(93) Filter +(99) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(94) Project +(100) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(95) Exchange +(101) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(97) Project +(103) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(98) Exchange +(104) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(105) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(100) Filter +(106) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(101) Exchange +(107) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(103) Project +(109) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(110) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(111) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(112) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(113) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(109) Project +(115) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(116) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(117) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(113) Exchange +(119) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(115) Project +(121) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(116) Exchange +(122) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(123) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(118) Filter +(124) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(119) Project +(125) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(120) Exchange +(126) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(127) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(122) Project +(128) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(123) HashAggregate +(129) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(124) Exchange +(130) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(131) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(126) Exchange +(132) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(133) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(128) AdaptiveSparkPlan +(134) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt index cc87fe6d2bbf..8d1a71e9751e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (18) +AdaptiveSparkPlan (19) +- == Final Plan == - VeloxColumnarToRowExec (11) - +- ^ RegularHashAggregateExecTransformer (9) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ ShuffleQueryStage (6) - +- ColumnarExchange (5) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (17) - +- Exchange (16) - +- HashAggregate (15) - +- Project (14) - +- Filter (13) - +- Scan parquet (12) + HashAggregate (18) + +- Exchange (17) + +- HashAggregate (16) + +- Project (15) + +- Filter (14) + +- Scan parquet (13) (1) Scan parquet @@ -26,82 +27,86 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(5) ColumnarExchange +(6) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [sum#X, isEmpty#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(9) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(11) VeloxColumnarToRowExec +(12) VeloxColumnarToRowExec Input [1]: [revenue#X] -(12) Scan parquet +(13) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(13) Filter +(14) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(14) Project +(15) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) HashAggregate +(16) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(16) Exchange +(17) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(17) HashAggregate +(18) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(18) AdaptiveSparkPlan +(19) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt index b9b72a9de06a..5ac66a7e2708 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt @@ -1,117 +1,122 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (85) - +- ^ SortExecTransformer (83) - +- ^ InputIteratorTransformer (82) - +- ^ InputAdapter (81) - +- ^ ShuffleQueryStage (80) - +- ColumnarExchange (79) - +- ^ RegularHashAggregateExecTransformer (77) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FlushableHashAggregateExecTransformer (70) - +- ^ ProjectExecTransformer (69) - +- ^ ShuffledHashJoinExecTransformer Inner (68) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (67) - +- ^ InputAdapter (66) - +- ^ ShuffleQueryStage (65) - +- ReusedExchange (64) + VeloxColumnarToRowExec (90) + +- ^ SortExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ^ InputAdapter (86) + +- ^ ShuffleQueryStage (85) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ^ InputAdapter (80) + +- ^ ShuffleQueryStage (79) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner (73) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ^ InputAdapter (71) + +- ^ ShuffleQueryStage (70) + +- ReusedExchange (69) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- HashAggregate (120) - +- Exchange (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (112) - : +- Project (111) - : +- ShuffledHashJoin Inner BuildRight (110) - : :- Exchange (106) - : : +- Project (105) - : : +- ShuffledHashJoin Inner BuildRight (104) - : : :- Exchange (100) - : : : +- Project (99) - : : : +- ShuffledHashJoin Inner BuildRight (98) - : : : :- Exchange (94) - : : : : +- Project (93) - : : : : +- ShuffledHashJoin Inner BuildLeft (92) - : : : : :- Exchange (88) - : : : : : +- Filter (87) - : : : : : +- Scan parquet (86) - : : : : +- Exchange (91) - : : : : +- Filter (90) - : : : : +- Scan parquet (89) - : : : +- Exchange (97) - : : : +- Filter (96) - : : : +- Scan parquet (95) - : : +- Exchange (103) - : : +- Filter (102) - : : +- Scan parquet (101) - : +- Exchange (109) - : +- Filter (108) - : +- Scan parquet (107) - +- Exchange (115) - +- Filter (114) - +- Scan parquet (113) + Sort (127) + +- Exchange (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (111) + : : +- Project (110) + : : +- ShuffledHashJoin Inner BuildRight (109) + : : :- Exchange (105) + : : : +- Project (104) + : : : +- ShuffledHashJoin Inner BuildRight (103) + : : : :- Exchange (99) + : : : : +- Project (98) + : : : : +- ShuffledHashJoin Inner BuildLeft (97) + : : : : :- Exchange (93) + : : : : : +- Filter (92) + : : : : : +- Scan parquet (91) + : : : : +- Exchange (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (102) + : : : +- Filter (101) + : : : +- Scan parquet (100) + : : +- Exchange (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (114) + : +- Filter (113) + : +- Scan parquet (112) + +- Exchange (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -121,516 +126,536 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X, o_custkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(41) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(56) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(64) ReusedExchange [Reuses operator id: 53] +(69) ReusedExchange [Reuses operator id: 58] Output [2]: [n_nationkey#X, n_name#X] -(65) ShuffleQueryStage +(70) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(66) InputAdapter +(71) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(67) InputIteratorTransformer +(72) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(68) ShuffledHashJoinExecTransformer +(73) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(69) ProjectExecTransformer +(74) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(70) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(71) ProjectExecTransformer +(76) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(72) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(73) ColumnarExchange +(78) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(79) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(75) InputAdapter +(80) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) InputIteratorTransformer +(81) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(78) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(79) ColumnarExchange +(84) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(80) ShuffleQueryStage +(85) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(81) InputAdapter +(86) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(82) InputIteratorTransformer +(87) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(83) SortExecTransformer +(88) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(84) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(85) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(86) Scan parquet +(91) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(87) Filter +(92) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(88) Exchange +(93) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(94) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(95) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) Exchange +(96) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(93) Project +(98) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(94) Exchange +(99) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(100) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(96) Filter +(101) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(97) Exchange +(102) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(103) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(99) Project +(104) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(100) Exchange +(105) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(102) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(103) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(105) Project +(110) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(106) Exchange +(111) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) Scan parquet +(112) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(108) Filter +(113) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(109) Exchange +(114) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(111) Project +(116) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(112) Exchange +(117) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(114) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(115) Exchange +(120) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(117) Project +(122) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(118) HashAggregate +(123) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(119) Exchange +(124) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) HashAggregate +(125) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(121) Exchange +(126) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(127) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt index 38e775dcd2ef..7e7064f9fbae 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt @@ -1,158 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (169) +AdaptiveSparkPlan (177) +- == Final Plan == - VeloxColumnarToRowExec (117) - +- ^ SortExecTransformer (115) - +- ^ InputIteratorTransformer (114) - +- ^ InputAdapter (113) - +- ^ ShuffleQueryStage (112) - +- ColumnarExchange (111) - +- ^ ProjectExecTransformer (109) - +- ^ RegularHashAggregateExecTransformer (108) - +- ^ InputIteratorTransformer (107) - +- ^ InputAdapter (106) - +- ^ ShuffleQueryStage (105) - +- ColumnarExchange (104) - +- ^ ProjectExecTransformer (102) - +- ^ FlushableHashAggregateExecTransformer (101) - +- ^ ProjectExecTransformer (100) - +- ^ ShuffledHashJoinExecTransformer Inner (99) - :- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner (85) - : :- ^ InputIteratorTransformer (77) - : : +- ^ InputAdapter (76) - : : +- ^ ShuffleQueryStage (75) - : : +- ColumnarExchange (74) - : : +- ^ ProjectExecTransformer (72) - : : +- ^ ShuffledHashJoinExecTransformer Inner (71) - : : :- ^ InputIteratorTransformer (63) - : : : +- ^ InputAdapter (62) - : : : +- ^ ShuffleQueryStage (61) - : : : +- ColumnarExchange (60) - : : : +- ^ ProjectExecTransformer (58) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : : : :- ^ InputIteratorTransformer (49) - : : : : +- ^ InputAdapter (48) - : : : : +- ^ ShuffleQueryStage (47) - : : : : +- ColumnarExchange (46) - : : : : +- ^ ProjectExecTransformer (44) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : : : :- ^ InputIteratorTransformer (35) - : : : : : +- ^ InputAdapter (34) - : : : : : +- ^ ShuffleQueryStage (33) - : : : : : +- ColumnarExchange (32) - : : : : : +- ^ ProjectExecTransformer (30) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : : : :- ^ InputIteratorTransformer (21) - : : : : : : +- ^ InputAdapter (20) - : : : : : : +- ^ ShuffleQueryStage (19) - : : : : : : +- ColumnarExchange (18) - : : : : : : +- ^ ProjectExecTransformer (16) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : : : :- ^ InputIteratorTransformer (7) - : : : : : : : +- ^ InputAdapter (6) - : : : : : : : +- ^ ShuffleQueryStage (5) - : : : : : : : +- ColumnarExchange (4) - : : : : : : : +- ^ ProjectExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (14) - : : : : : : +- ^ InputAdapter (13) - : : : : : : +- ^ ShuffleQueryStage (12) - : : : : : : +- ColumnarExchange (11) - : : : : : : +- ^ ProjectExecTransformer (9) - : : : : : : +- ^ Scan parquet (8) - : : : : : +- ^ InputIteratorTransformer (28) - : : : : : +- ^ InputAdapter (27) - : : : : : +- ^ ShuffleQueryStage (26) - : : : : : +- ColumnarExchange (25) - : : : : : +- ^ ProjectExecTransformer (23) - : : : : : +- ^ Scan parquet (22) - : : : : +- ^ InputIteratorTransformer (42) - : : : : +- ^ InputAdapter (41) - : : : : +- ^ ShuffleQueryStage (40) - : : : : +- ColumnarExchange (39) - : : : : +- ^ ProjectExecTransformer (37) - : : : : +- ^ Scan parquet (36) - : : : +- ^ InputIteratorTransformer (56) - : : : +- ^ InputAdapter (55) - : : : +- ^ ShuffleQueryStage (54) - : : : +- ColumnarExchange (53) - : : : +- ^ ProjectExecTransformer (51) - : : : +- ^ Scan parquet (50) - : : +- ^ InputIteratorTransformer (70) - : : +- ^ InputAdapter (69) - : : +- ^ ShuffleQueryStage (68) - : : +- ColumnarExchange (67) - : : +- ^ ProjectExecTransformer (65) - : : +- ^ Scan parquet (64) - : +- ^ InputIteratorTransformer (84) - : +- ^ InputAdapter (83) - : +- ^ ShuffleQueryStage (82) - : +- ColumnarExchange (81) - : +- ^ ProjectExecTransformer (79) - : +- ^ Scan parquet (78) - +- ^ InputIteratorTransformer (98) - +- ^ InputAdapter (97) - +- ^ ShuffleQueryStage (96) - +- ColumnarExchange (95) - +- ^ ProjectExecTransformer (93) - +- ^ Scan parquet (92) + VeloxColumnarToRowExec (125) + +- ^ SortExecTransformer (123) + +- ^ InputIteratorTransformer (122) + +- ^ InputAdapter (121) + +- ^ ShuffleQueryStage (120) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ^ InputAdapter (114) + +- ^ ShuffleQueryStage (113) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner (107) + :- ^ InputIteratorTransformer (98) + : +- ^ InputAdapter (97) + : +- ^ ShuffleQueryStage (96) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner (92) + : :- ^ InputIteratorTransformer (83) + : : +- ^ InputAdapter (82) + : : +- ^ ShuffleQueryStage (81) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ^ InputAdapter (67) + : : : +- ^ ShuffleQueryStage (66) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ^ InputAdapter (52) + : : : : +- ^ ShuffleQueryStage (51) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ^ InputAdapter (37) + : : : : : +- ^ ShuffleQueryStage (36) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ^ InputAdapter (22) + : : : : : : +- ^ ShuffleQueryStage (21) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ^ InputAdapter (7) + : : : : : : : +- ^ ShuffleQueryStage (6) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ^ InputAdapter (15) + : : : : : : +- ^ ShuffleQueryStage (14) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ^ InputAdapter (30) + : : : : : +- ^ ShuffleQueryStage (29) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ NoopFilter (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ^ InputAdapter (45) + : : : : +- ^ ShuffleQueryStage (44) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ NoopFilter (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ^ InputAdapter (60) + : : : +- ^ ShuffleQueryStage (59) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ NoopFilter (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ^ InputAdapter (75) + : : +- ^ ShuffleQueryStage (74) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ NoopFilter (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ^ InputAdapter (90) + : +- ^ ShuffleQueryStage (89) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ NoopFilter (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ^ InputAdapter (105) + +- ^ ShuffleQueryStage (104) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ NoopFilter (100) + +- ^ Scan parquet (99) +- == Initial Plan == - Sort (168) - +- Exchange (167) - +- HashAggregate (166) - +- Exchange (165) - +- HashAggregate (164) - +- Project (163) - +- ShuffledHashJoin Inner BuildRight (162) - :- Exchange (157) - : +- Project (156) - : +- ShuffledHashJoin Inner BuildRight (155) - : :- Exchange (151) - : : +- Project (150) - : : +- ShuffledHashJoin Inner BuildRight (149) - : : :- Exchange (145) - : : : +- Project (144) - : : : +- ShuffledHashJoin Inner BuildRight (143) - : : : :- Exchange (139) - : : : : +- Project (138) - : : : : +- ShuffledHashJoin Inner BuildRight (137) - : : : : :- Exchange (133) - : : : : : +- Project (132) - : : : : : +- ShuffledHashJoin Inner BuildRight (131) - : : : : : :- Exchange (127) - : : : : : : +- Project (126) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (125) - : : : : : : :- Exchange (121) - : : : : : : : +- Project (120) - : : : : : : : +- Filter (119) - : : : : : : : +- Scan parquet (118) - : : : : : : +- Exchange (124) - : : : : : : +- Filter (123) - : : : : : : +- Scan parquet (122) - : : : : : +- Exchange (130) - : : : : : +- Filter (129) - : : : : : +- Scan parquet (128) - : : : : +- Exchange (136) - : : : : +- Filter (135) - : : : : +- Scan parquet (134) - : : : +- Exchange (142) - : : : +- Filter (141) - : : : +- Scan parquet (140) - : : +- Exchange (148) - : : +- Filter (147) - : : +- Scan parquet (146) - : +- Exchange (154) - : +- Filter (153) - : +- Scan parquet (152) - +- Exchange (161) - +- Project (160) - +- Filter (159) - +- Scan parquet (158) + Sort (176) + +- Exchange (175) + +- HashAggregate (174) + +- Exchange (173) + +- HashAggregate (172) + +- Project (171) + +- ShuffledHashJoin Inner BuildRight (170) + :- Exchange (165) + : +- Project (164) + : +- ShuffledHashJoin Inner BuildRight (163) + : :- Exchange (159) + : : +- Project (158) + : : +- ShuffledHashJoin Inner BuildRight (157) + : : :- Exchange (153) + : : : +- Project (152) + : : : +- ShuffledHashJoin Inner BuildRight (151) + : : : :- Exchange (147) + : : : : +- Project (146) + : : : : +- ShuffledHashJoin Inner BuildRight (145) + : : : : :- Exchange (141) + : : : : : +- Project (140) + : : : : : +- ShuffledHashJoin Inner BuildRight (139) + : : : : : :- Exchange (135) + : : : : : : +- Project (134) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) + : : : : : : :- Exchange (129) + : : : : : : : +- Project (128) + : : : : : : : +- Filter (127) + : : : : : : : +- Scan parquet (126) + : : : : : : +- Exchange (132) + : : : : : : +- Filter (131) + : : : : : : +- Scan parquet (130) + : : : : : +- Exchange (138) + : : : : : +- Filter (137) + : : : : : +- Scan parquet (136) + : : : : +- Exchange (144) + : : : : +- Filter (143) + : : : : +- Scan parquet (142) + : : : +- Exchange (150) + : : : +- Filter (149) + : : : +- Scan parquet (148) + : : +- Exchange (156) + : : +- Filter (155) + : : +- Scan parquet (154) + : +- Exchange (162) + : +- Filter (161) + : +- Scan parquet (160) + +- Exchange (169) + +- Project (168) + +- Filter (167) + +- Scan parquet (166) (1) Scan parquet @@ -162,712 +170,744 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(3) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(11) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(36) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(41) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(56) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_regionkey#X] + +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(73) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(74) ColumnarExchange +(80) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(75) ShuffleQueryStage +(81) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(76) InputAdapter +(82) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(77) InputIteratorTransformer +(83) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) ProjectExecTransformer +(85) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(86) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(80) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(81) ColumnarExchange +(88) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(82) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(83) InputAdapter +(90) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(84) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(85) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) ProjectExecTransformer +(93) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(95) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(92) Scan parquet +(99) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(93) ProjectExecTransformer +(100) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(101) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(94) WholeStageCodegenTransformer (X) +(102) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(95) ColumnarExchange +(103) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(104) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(97) InputAdapter +(105) InputAdapter Input [1]: [r_regionkey#X] -(98) InputIteratorTransformer +(106) InputIteratorTransformer Input [1]: [r_regionkey#X] -(99) ShuffledHashJoinExecTransformer +(107) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(100) ProjectExecTransformer +(108) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(101) FlushableHashAggregateExecTransformer +(109) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(102) ProjectExecTransformer +(110) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(103) WholeStageCodegenTransformer (X) +(111) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(104) ColumnarExchange +(112) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(105) ShuffleQueryStage +(113) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(106) InputAdapter +(114) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(107) InputIteratorTransformer +(115) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(108) RegularHashAggregateExecTransformer +(116) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(109) ProjectExecTransformer +(117) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(110) WholeStageCodegenTransformer (X) +(118) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(111) ColumnarExchange +(119) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(112) ShuffleQueryStage +(120) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(113) InputAdapter +(121) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(114) InputIteratorTransformer +(122) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(115) SortExecTransformer +(123) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(116) WholeStageCodegenTransformer (X) +(124) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(117) VeloxColumnarToRowExec +(125) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(118) Scan parquet +(126) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(119) Filter +(127) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(120) Project +(128) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(121) Exchange +(129) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(130) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(123) Filter +(131) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(124) Exchange +(132) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(133) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(126) Project +(134) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(127) Exchange +(135) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(136) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(129) Filter +(137) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(130) Exchange +(138) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(132) Project +(140) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(133) Exchange +(141) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(142) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(135) Filter +(143) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(136) Exchange +(144) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) ShuffledHashJoin +(145) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(138) Project +(146) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(139) Exchange +(147) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(140) Scan parquet +(148) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(141) Filter +(149) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(142) Exchange +(150) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) ShuffledHashJoin +(151) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(144) Project +(152) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(145) Exchange +(153) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Scan parquet +(154) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(147) Filter +(155) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(148) Exchange +(156) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(149) ShuffledHashJoin +(157) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(150) Project +(158) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(151) Exchange +(159) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Scan parquet +(160) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(153) Filter +(161) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(154) Exchange +(162) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(155) ShuffledHashJoin +(163) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(156) Project +(164) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(157) Exchange +(165) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(158) Scan parquet +(166) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(159) Filter +(167) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(160) Project +(168) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(161) Exchange +(169) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(162) ShuffledHashJoin +(170) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(163) Project +(171) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(164) HashAggregate +(172) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(165) Exchange +(173) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) HashAggregate +(174) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(167) Exchange +(175) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(168) Sort +(176) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(169) AdaptiveSparkPlan +(177) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt index 8ba93a20d0cd..5ba4a9f7ce0e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt @@ -1,120 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (127) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (88) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ^ InputAdapter (78) - +- ^ ShuffleQueryStage (77) - +- ColumnarExchange (76) - +- ^ ProjectExecTransformer (74) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ ShuffledHashJoinExecTransformer Inner (71) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ ShuffleQueryStage (68) - +- ColumnarExchange (67) - +- ^ ProjectExecTransformer (65) - +- ^ Scan parquet (64) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (126) - +- Exchange (125) - +- HashAggregate (124) - +- Exchange (123) - +- HashAggregate (122) - +- Project (121) - +- ShuffledHashJoin Inner BuildRight (120) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildRight (108) - : : :- Exchange (104) - : : : +- Project (103) - : : : +- ShuffledHashJoin Inner BuildRight (102) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- ShuffledHashJoin Inner BuildLeft (96) - : : : : :- Exchange (92) - : : : : : +- Project (91) - : : : : : +- Filter (90) - : : : : : +- Scan parquet (89) - : : : : +- Exchange (95) - : : : : +- Filter (94) - : : : : +- Scan parquet (93) - : : : +- Exchange (101) - : : : +- Filter (100) - : : : +- Scan parquet (99) - : : +- Exchange (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Exchange (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (119) - +- Filter (118) - +- Scan parquet (117) + Sort (132) + +- Exchange (131) + +- HashAggregate (130) + +- Exchange (129) + +- HashAggregate (128) + +- Project (127) + +- ShuffledHashJoin Inner BuildRight (126) + :- Exchange (122) + : +- Project (121) + : +- ShuffledHashJoin Inner BuildRight (120) + : :- Exchange (116) + : : +- Project (115) + : : +- ShuffledHashJoin Inner BuildRight (114) + : : :- Exchange (110) + : : : +- Project (109) + : : : +- ShuffledHashJoin Inner BuildRight (108) + : : : :- Exchange (104) + : : : : +- Project (103) + : : : : +- ShuffledHashJoin Inner BuildLeft (102) + : : : : :- Exchange (98) + : : : : : +- Project (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Exchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Exchange (125) + +- Filter (124) + +- Scan parquet (123) (1) Scan parquet @@ -124,536 +130,560 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(3) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(11) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(36) Scan parquet +(39) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(41) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_orderdate#X] + +(56) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(73) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(74) ProjectExecTransformer +(80) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(82) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(83) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(84) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(79) InputIteratorTransformer +(85) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(81) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(82) ColumnarExchange +(88) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(84) InputAdapter +(90) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(85) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(86) SortExecTransformer +(92) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(89) Scan parquet +(95) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(90) Filter +(96) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(91) Project +(97) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(92) Exchange +(98) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Scan parquet +(99) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(94) Filter +(100) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(95) Exchange +(101) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(97) Project +(103) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(98) Exchange +(104) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(105) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(100) Filter +(106) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(101) Exchange +(107) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(103) Project +(109) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(104) Exchange +(110) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(111) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(106) Filter +(112) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(107) Exchange +(113) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(109) Project +(115) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(110) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(117) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(113) Exchange +(119) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(115) Project +(121) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(116) Exchange +(122) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(118) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(119) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(127) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(122) HashAggregate +(128) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(123) Exchange +(129) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) HashAggregate +(130) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(125) Exchange +(131) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) Sort +(132) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(127) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt index c5d58658acd5..74f79bd3ee64 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (27) +AdaptiveSparkPlan (28) +- == Final Plan == - VeloxColumnarToRowExec (18) - +- ^ SortExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (19) + +- ^ SortExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (26) - +- Exchange (25) - +- HashAggregate (24) - +- Exchange (23) - +- HashAggregate (22) - +- Project (21) - +- Filter (20) - +- Scan parquet (19) + Sort (27) + +- Exchange (26) + +- HashAggregate (25) + +- Exchange (24) + +- HashAggregate (23) + +- Project (22) + +- Filter (21) + +- Scan parquet (20) (1) Scan parquet @@ -34,116 +35,120 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(4) ProjectExecTransformer +(5) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(6) ColumnarExchange +(7) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(12) ColumnarExchange +(13) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(14) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) InputAdapter +(15) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(15) InputIteratorTransformer +(16) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) SortExecTransformer +(17) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(17) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(18) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(19) Scan parquet +(20) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(20) Filter +(21) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(21) Project +(22) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(22) HashAggregate +(23) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(23) Exchange +(24) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(24) HashAggregate +(25) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(25) Exchange +(26) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Sort +(27) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(27) AdaptiveSparkPlan +(28) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt index af8f35bc4b0e..66b2ccdc77b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt @@ -1,81 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (83) +AdaptiveSparkPlan (87) +- == Final Plan == - VeloxColumnarToRowExec (56) - +- TakeOrderedAndProjectExecTransformer (55) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ ShuffledHashJoinExecTransformer Inner (43) - :- ^ InputIteratorTransformer (35) - : +- ^ InputAdapter (34) - : +- ^ ShuffleQueryStage (33), Statistics(X) - : +- ColumnarExchange (32) - : +- ^ ProjectExecTransformer (30) - : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : :- ^ InputIteratorTransformer (21) - : : +- ^ InputAdapter (20) - : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : +- ColumnarExchange (18) - : : +- ^ ProjectExecTransformer (16) - : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : +- ColumnarExchange (4) - : : : +- ^ ProjectExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (14) - : : +- ^ InputAdapter (13) - : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : +- ColumnarExchange (11) - : : +- ^ ProjectExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (28) - : +- ^ InputAdapter (27) - : +- ^ ShuffleQueryStage (26), Statistics(X) - : +- ColumnarExchange (25) - : +- ^ ProjectExecTransformer (23) - : +- ^ Scan parquet (22) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ ShuffleQueryStage (40), Statistics(X) - +- ColumnarExchange (39) - +- ^ ProjectExecTransformer (37) - +- ^ Scan parquet (36) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner (47) + :- ^ InputIteratorTransformer (38) + : +- ^ InputAdapter (37) + : +- ^ ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : :- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (82) - +- HashAggregate (81) - +- Exchange (80) - +- HashAggregate (79) - +- Project (78) - +- ShuffledHashJoin Inner BuildRight (77) - :- Exchange (73) - : +- Project (72) - : +- ShuffledHashJoin Inner BuildRight (71) - : :- Exchange (66) - : : +- Project (65) - : : +- ShuffledHashJoin Inner BuildRight (64) - : : :- Exchange (59) - : : : +- Filter (58) - : : : +- Scan parquet (57) - : : +- Exchange (63) - : : +- Project (62) - : : +- Filter (61) - : : +- Scan parquet (60) - : +- Exchange (70) - : +- Project (69) - : +- Filter (68) - : +- Scan parquet (67) - +- Exchange (76) - +- Filter (75) - +- Scan parquet (74) + TakeOrderedAndProject (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- ShuffledHashJoin Inner BuildRight (81) + :- Exchange (77) + : +- Project (76) + : +- ShuffledHashJoin Inner BuildRight (75) + : :- Exchange (70) + : : +- Project (69) + : : +- ShuffledHashJoin Inner BuildRight (68) + : : :- Exchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- Exchange (67) + : : +- Project (66) + : : +- Filter (65) + : : +- Scan parquet (64) + : +- Exchange (74) + : +- Project (73) + : +- Filter (72) + : +- Scan parquet (71) + +- Exchange (80) + +- Filter (79) + +- Scan parquet (78) (1) Scan parquet @@ -85,354 +89,370 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(3) ProjectExecTransformer Output [8]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(41) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(45) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(50) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(52) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(53) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(54) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(55) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(53) ProjectExecTransformer +(57) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(55) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(56) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(57) Scan parquet +(61) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(58) Filter +(62) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(59) Exchange +(63) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(64) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(61) Filter +(65) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(62) Project +(66) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(63) Exchange +(67) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(68) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(65) Project +(69) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(66) Exchange +(70) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(67) Scan parquet +(71) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(68) Filter +(72) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(69) Project +(73) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(70) Exchange +(74) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(72) Project +(76) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(73) Exchange +(77) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(75) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(76) Exchange +(80) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(79) HashAggregate +(83) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(80) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(85) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(82) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(83) AdaptiveSparkPlan +(87) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt index e40294ba7679..4bd0783da6fe 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt @@ -1,68 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (72) +- == Final Plan == - VeloxColumnarToRowExec (47) - +- ^ SortExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ^ InputAdapter (43) - +- ^ ShuffleQueryStage (42), Statistics(X) - +- ColumnarExchange (41) - +- ^ FilterExecTransformer (39) - +- ^ RegularHashAggregateExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ ShuffleQueryStage (35), Statistics(X) - +- ColumnarExchange (34) - +- ^ ProjectExecTransformer (32) - +- ^ FlushableHashAggregateExecTransformer (31) - +- ^ ProjectExecTransformer (30) - +- ^ ShuffledHashJoinExecTransformer Inner (29) - :- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ ShuffleQueryStage (19), Statistics(X) - : +- ColumnarExchange (18) - : +- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12), Statistics(X) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ Scan parquet (22) + VeloxColumnarToRowExec (50) + +- ^ SortExecTransformer (48) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ ShuffleQueryStage (38), Statistics(X) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - Sort (68) - +- Exchange (67) - +- Filter (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- ShuffledHashJoin Inner BuildRight (61) - :- Exchange (56) - : +- Project (55) - : +- ShuffledHashJoin Inner BuildRight (54) - : :- Exchange (50) - : : +- Filter (49) - : : +- Scan parquet (48) - : +- Exchange (53) - : +- Filter (52) - : +- Scan parquet (51) - +- Exchange (60) - +- Project (59) - +- Filter (58) - +- Scan parquet (57) + Sort (71) + +- Exchange (70) + +- Filter (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- ShuffledHashJoin Inner BuildRight (64) + :- Exchange (59) + : +- Project (58) + : +- ShuffledHashJoin Inner BuildRight (57) + : :- Exchange (53) + : : +- Filter (52) + : : +- Scan parquet (51) + : +- Exchange (56) + : +- Filter (55) + : +- Scan parquet (54) + +- Exchange (63) + +- Project (62) + +- Filter (61) + +- Scan parquet (60) (1) Scan parquet @@ -72,548 +75,565 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(3) ProjectExecTransformer Output [5]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) Scan parquet +(9) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(11) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(26) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [1]: [n_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [1]: [n_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(31) FlushableHashAggregateExecTransformer +(34) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(34) ColumnarExchange +(37) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(38) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(36) InputAdapter +(39) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(37) InputIteratorTransformer +(40) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(38) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(39) FilterExecTransformer +(42) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(40) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(41) ColumnarExchange +(44) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(45) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(43) InputAdapter +(46) InputAdapter Input [2]: [ps_partkey#X, value#X] -(44) InputIteratorTransformer +(47) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(45) SortExecTransformer +(48) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(46) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(47) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(48) Scan parquet +(51) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(49) Filter +(52) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(50) Exchange +(53) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(53) Exchange +(56) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(55) Project +(58) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(56) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Scan parquet +(60) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(58) Filter +(61) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(59) Project +(62) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(60) Exchange +(63) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) ShuffledHashJoin +(64) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(65) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(63) HashAggregate +(66) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(64) Exchange +(67) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(68) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(66) Filter +(69) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(67) Exchange +(70) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Sort +(71) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(69) AdaptiveSparkPlan +(72) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 39 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (116) +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (120) +- == Final Plan == - VeloxColumnarToRowExec (98) - +- ^ ProjectExecTransformer (96) - +- ^ RegularHashAggregateExecTransformer (95) - +- ^ RegularHashAggregateExecTransformer (94) - +- ^ ProjectExecTransformer (93) - +- ^ ShuffledHashJoinExecTransformer Inner (92) - :- ^ InputIteratorTransformer (87) - : +- ^ InputAdapter (86) - : +- ^ ShuffleQueryStage (85), Statistics(X) - : +- ColumnarExchange (84) - : +- ^ ProjectExecTransformer (82) - : +- ^ ShuffledHashJoinExecTransformer Inner (81) - : :- ^ InputIteratorTransformer (76) - : : +- ^ InputAdapter (75) - : : +- ^ ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ Scan parquet (70) - : +- ^ InputIteratorTransformer (80) - : +- ^ InputAdapter (79) - : +- ^ ShuffleQueryStage (78), Statistics(X) - : +- ReusedExchange (77) - +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ReusedExchange (88) + VeloxColumnarToRowExec (102) + +- ^ ProjectExecTransformer (100) + +- ^ RegularHashAggregateExecTransformer (99) + +- ^ RegularHashAggregateExecTransformer (98) + +- ^ ProjectExecTransformer (97) + +- ^ ShuffledHashJoinExecTransformer Inner (96) + :- ^ InputIteratorTransformer (91) + : +- ^ InputAdapter (90) + : +- ^ ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ ShuffledHashJoinExecTransformer Inner (85) + : :- ^ InputIteratorTransformer (80) + : : +- ^ InputAdapter (79) + : : +- ^ ShuffleQueryStage (78), Statistics(X) + : : +- ColumnarExchange (77) + : : +- ^ ProjectExecTransformer (75) + : : +- ^ NoopFilter (74) + : : +- ^ Scan parquet (73) + : +- ^ InputIteratorTransformer (84) + : +- ^ InputAdapter (83) + : +- ^ ShuffleQueryStage (82), Statistics(X) + : +- ReusedExchange (81) + +- ^ InputIteratorTransformer (95) + +- ^ InputAdapter (94) + +- ^ ShuffleQueryStage (93), Statistics(X) + +- ReusedExchange (92) +- == Initial Plan == - HashAggregate (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (101) - : : +- Filter (100) - : : +- Scan parquet (99) - : +- Exchange (104) - : +- Filter (103) - : +- Scan parquet (102) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) - - -(70) Scan parquet + HashAggregate (119) + +- HashAggregate (118) + +- Project (117) + +- ShuffledHashJoin Inner BuildRight (116) + :- Exchange (111) + : +- Project (110) + : +- ShuffledHashJoin Inner BuildRight (109) + : :- Exchange (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (108) + : +- Filter (107) + : +- Scan parquet (106) + +- Exchange (115) + +- Project (114) + +- Filter (113) + +- Scan parquet (112) + + +(73) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(71) ProjectExecTransformer +(74) NoopFilter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(75) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(72) WholeStageCodegenTransformer (X) +(76) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(73) ColumnarExchange +(77) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(78) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(75) InputAdapter +(79) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) InputIteratorTransformer +(80) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(77) ReusedExchange [Reuses operator id: 11] +(81) ReusedExchange [Reuses operator id: 13] Output [2]: [s_suppkey#X, s_nationkey#X] -(78) ShuffleQueryStage +(82) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(79) InputAdapter +(83) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(80) InputIteratorTransformer +(84) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(81) ShuffledHashJoinExecTransformer +(85) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(82) ProjectExecTransformer +(86) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(83) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(84) ColumnarExchange +(88) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(86) InputAdapter +(90) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(87) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(88) ReusedExchange [Reuses operator id: 25] +(92) ReusedExchange [Reuses operator id: 28] Output [1]: [n_nationkey#X] -(89) ShuffleQueryStage +(93) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(90) InputAdapter +(94) InputAdapter Input [1]: [n_nationkey#X] -(91) InputIteratorTransformer +(95) InputIteratorTransformer Input [1]: [n_nationkey#X] -(92) ShuffledHashJoinExecTransformer +(96) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(94) RegularHashAggregateExecTransformer +(98) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(95) RegularHashAggregateExecTransformer +(99) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(96) ProjectExecTransformer +(100) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(97) WholeStageCodegenTransformer (X) +(101) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(98) VeloxColumnarToRowExec +(102) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) Scan parquet +(103) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(100) Filter +(104) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(101) Exchange +(105) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) Scan parquet +(106) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(103) Filter +(107) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(104) Exchange +(108) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(106) Project +(110) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(107) Exchange +(111) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(112) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(113) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(110) Project +(114) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(115) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(113) Project +(117) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(118) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(115) HashAggregate +(119) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(116) AdaptiveSparkPlan +(120) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt index a255925b847c..ec56c2f99543 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt @@ -1,48 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (47) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (46) - +- Exchange (45) - +- HashAggregate (44) - +- Exchange (43) - +- HashAggregate (42) - +- Project (41) - +- ShuffledHashJoin Inner BuildLeft (40) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (39) - +- Project (38) - +- Filter (37) - +- Scan parquet (36) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin Inner BuildLeft (42) + :- Exchange (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Exchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -52,200 +54,208 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(26) ColumnarExchange +(28) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(28) InputAdapter +(30) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(30) SortExecTransformer +(32) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(32) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(33) Scan parquet +(35) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(34) Filter +(36) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(35) Exchange +(37) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(38) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(37) Filter +(39) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(38) Project +(40) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(39) Exchange +(41) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(41) Project +(43) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(42) HashAggregate +(44) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(43) Exchange +(45) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) HashAggregate +(46) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(45) Exchange +(47) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(48) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(47) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt index eb454d4f7f23..f7d8f3c3b72f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (51) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (15) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5), Statistics(X) : +- ColumnarExchange (4) : +- ^ ProjectExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (50) - +- Exchange (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin LeftOuter BuildRight (42) - :- Exchange (37) - : +- Scan parquet (36) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- HashAggregate (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftOuter BuildRight (43) + :- Exchange (38) + : +- Scan parquet (37) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -84,198 +85,202 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) ProjectExecTransformer +(9) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] + +(10) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(12) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(13) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(14) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(15) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(17) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(19) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(29) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(36) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Exchange +(38) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) HashAggregate +(46) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(46) HashAggregate +(47) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(47) Exchange +(48) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(49) Exchange +(50) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Sort +(51) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(51) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt index 0cd11b2c11f7..4f4492d881fd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (32) - +- HashAggregate (31) - +- Project (30) - +- ShuffledHashJoin Inner BuildRight (29) - :- Exchange (25) - : +- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- Exchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- HashAggregate (33) + +- Project (32) + +- ShuffledHashJoin Inner BuildRight (31) + :- Exchange (27) + : +- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- Exchange (30) + +- Filter (29) + +- Scan parquet (28) (1) Scan parquet @@ -40,148 +42,156 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) Scan parquet +(9) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(11) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(23) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(24) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(25) Exchange +(27) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Scan parquet +(28) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(29) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) Exchange +(30) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) ShuffledHashJoin +(31) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(32) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(33) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(33) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt index 96f8900c810c..98249fae7dd7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (40) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (25) - +- AQEShuffleRead (24) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner (19) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FlushableHashAggregateExecTransformer (10) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ ShuffledHashJoinExecTransformer Inner (21) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (39) - +- Exchange (38) - +- Project (37) - +- ShuffledHashJoin Inner BuildLeft (36) - :- Exchange (28) - : +- Filter (27) - : +- Scan parquet (26) - +- Filter (35) - +- HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- Filter (30) - +- Scan parquet (29) + Sort (41) + +- Exchange (40) + +- Project (39) + +- ShuffledHashJoin Inner BuildLeft (38) + :- Exchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Filter (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -47,328 +49,341 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(11) ProjectExecTransformer +(13) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(19) ShuffledHashJoinExecTransformer +(21) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(24) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(25) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(26) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(27) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(28) Exchange +(30) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) Scan parquet +(31) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(30) Filter +(32) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(31) Project +(33) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(32) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(33) Exchange +(35) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(35) Filter +(37) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(36) ShuffledHashJoin +(38) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(37) Project +(39) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(38) Exchange +(40) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Sort +(41) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(40) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 18 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (64) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ ProjectExecTransformer (51) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ Scan parquet (41) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ^ InputAdapter (51) + +- ^ ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ NoopFilter (44) + +- ^ Scan parquet (43) +- == Initial Plan == - HashAggregate (63) - +- HashAggregate (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- Filter (57) - +- Scan parquet (56) + HashAggregate (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(41) Scan parquet +(43) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(42) ProjectExecTransformer +(44) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(45) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(43) FlushableHashAggregateExecTransformer +(46) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(49) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(51) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(51) ProjectExecTransformer +(54) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(52) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(53) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(54) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(55) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(56) Scan parquet +(59) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(57) Filter +(60) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(58) Project +(61) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(59) HashAggregate +(62) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(60) Exchange +(63) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(63) HashAggregate +(66) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt index d8bc92e439c3..230e9c890d1d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt @@ -1,62 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (62) +AdaptiveSparkPlan (64) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ SortExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ ShuffleQueryStage (35), Statistics(X) - +- ColumnarExchange (34) - +- ^ RegularHashAggregateExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FlushableHashAggregateExecTransformer (25) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (42) + +- ^ SortExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ^ InputAdapter (38) + +- ^ ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (61) - +- Exchange (60) - +- HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- ShuffledHashJoin Inner BuildRight (52) - :- Exchange (48) - : +- BroadcastHashJoin LeftAnti BuildRight (47) - : :- Filter (42) - : : +- Scan parquet (41) - : +- BroadcastExchange (46) - : +- Project (45) - : +- Filter (44) - : +- Scan parquet (43) - +- Exchange (51) - +- Filter (50) - +- Scan parquet (49) + Sort (63) + +- Exchange (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- HashAggregate (58) + +- Exchange (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (50) + : +- BroadcastHashJoin LeftAnti BuildRight (49) + : :- Filter (44) + : : +- Scan parquet (43) + : +- BroadcastExchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Exchange (53) + +- Filter (52) + +- Scan parquet (51) (1) Scan parquet @@ -66,274 +68,282 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X, ps_suppkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Input [2]: [ps_partkey#X, ps_suppkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) FlushableHashAggregateExecTransformer +(27) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(28) ColumnarExchange +(30) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(30) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(32) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(33) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(34) ColumnarExchange +(36) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(37) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(36) InputAdapter +(38) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(37) InputIteratorTransformer +(39) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(38) SortExecTransformer +(40) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(39) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(40) VeloxColumnarToRowExec +(42) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(41) Scan parquet +(43) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(43) Scan parquet +(45) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(44) Filter +(46) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(45) Project +(47) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(46) BroadcastExchange +(48) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(47) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(48) Exchange +(50) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Scan parquet +(51) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(50) Filter +(52) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(51) Exchange +(53) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(53) Project +(55) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(54) HashAggregate +(56) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(55) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(58) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) HashAggregate +(59) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(58) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(60) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(63) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(62) AdaptiveSparkPlan +(64) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt index a76baac13557..3848cd252b45 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ ProjectExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ ShuffledHashJoinExecTransformer Inner (28) - :- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12), Statistics(X) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ FilterExecTransformer (27) - +- ^ ProjectExecTransformer (26) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ Scan parquet (17) + VeloxColumnarToRowExec (37) + +- ^ ProjectExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ RegularHashAggregateExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ ShuffledHashJoinExecTransformer Inner (31) + :- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ FilterExecTransformer (30) + +- ^ ProjectExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ NoopFilter (20) + +- ^ Scan parquet (19) +- == Initial Plan == - HashAggregate (53) - +- HashAggregate (52) - +- Project (51) - +- ShuffledHashJoin Inner BuildRight (50) - :- Project (43) - : +- ShuffledHashJoin Inner BuildRight (42) - : :- Exchange (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Exchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Filter (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- Filter (45) - +- Scan parquet (44) + HashAggregate (56) + +- HashAggregate (55) + +- Project (54) + +- ShuffledHashJoin Inner BuildRight (53) + :- Project (46) + : +- ShuffledHashJoin Inner BuildRight (45) + : :- Exchange (40) + : : +- Filter (39) + : : +- Scan parquet (38) + : +- Exchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- Filter (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -60,250 +63,262 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X] + +(3) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [p_partkey#X, p_brand#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_container#X] + +(11) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [1]: [p_partkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [p_partkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) NoopFilter +Input [2]: [l_partkey#X, l_quantity#X] +Arguments: [l_partkey#X, l_quantity#X] + +(21) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(20) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(21) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(27) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(28) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(30) RegularHashAggregateExecTransformer +(33) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(37) Exchange +(40) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(40) Project +(43) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(41) Exchange +(44) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(43) Project +(46) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(44) Scan parquet +(47) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(45) Filter +(48) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(46) HashAggregate +(49) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(47) Exchange +(50) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(51) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(49) Filter +(52) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(50) ShuffledHashJoin +(53) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(51) Project +(54) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) HashAggregate +(55) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(53) HashAggregate +(56) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(54) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt index 36c934dd0416..08b902eed5ed 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt @@ -1,93 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (97) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- TakeOrderedAndProjectExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ ProjectExecTransformer (56) - +- ^ ShuffledHashJoinExecTransformer Inner (55) - :- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ ShuffleQueryStage (37), Statistics(X) - : +- ColumnarExchange (36) - : +- ^ ProjectExecTransformer (34) - : +- ^ ShuffledHashJoinExecTransformer Inner (33) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ ShuffleQueryStage (30), Statistics(X) - : +- ColumnarExchange (29) - : +- ^ ProjectExecTransformer (27) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (26) - : :- ^ InputIteratorTransformer (14) - : : +- ^ InputAdapter (13) - : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : +- ColumnarExchange (11) - : : +- ^ ProjectExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ ProjectExecTransformer (25) - : +- ^ FilterExecTransformer (24) - : +- ^ RegularHashAggregateExecTransformer (23) - : +- ^ InputIteratorTransformer (22) - : +- ^ InputAdapter (21) - : +- ^ ShuffleQueryStage (20), Statistics(X) - : +- ColumnarExchange (19) - : +- ^ ProjectExecTransformer (17) - : +- ^ FlushableHashAggregateExecTransformer (16) - : +- ^ Scan parquet (15) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (54) - :- ^ InputIteratorTransformer (46) - : +- ^ InputAdapter (45) - : +- ^ ShuffleQueryStage (44), Statistics(X) - : +- ColumnarExchange (43) - : +- ^ ProjectExecTransformer (41) - : +- ^ Scan parquet (40) - +- ^ ProjectExecTransformer (53) - +- ^ FilterExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48), Statistics(X) - +- ReusedExchange (47) + VeloxColumnarToRowExec (64) + +- TakeOrderedAndProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner (58) + :- ^ InputIteratorTransformer (41) + : +- ^ InputAdapter (40) + : +- ^ ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ^ InputAdapter (33) + : +- ^ ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : :- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ^ InputAdapter (23) + : +- ^ ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + :- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (93) - +- HashAggregate (92) - +- HashAggregate (91) - +- Project (90) - +- ShuffledHashJoin Inner BuildRight (89) - :- Exchange (78) - : +- Project (77) - : +- ShuffledHashJoin Inner BuildLeft (76) - : :- Exchange (64) - : : +- Filter (63) - : : +- Scan parquet (62) - : +- Exchange (75) - : +- ShuffledHashJoin LeftSemi BuildRight (74) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Project (73) - : +- Filter (72) - : +- HashAggregate (71) - : +- Exchange (70) - : +- HashAggregate (69) - : +- Scan parquet (68) - +- ShuffledHashJoin LeftSemi BuildRight (88) - :- Exchange (81) - : +- Filter (80) - : +- Scan parquet (79) - +- Project (87) - +- Filter (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Scan parquet (82) + TakeOrderedAndProject (96) + +- HashAggregate (95) + +- HashAggregate (94) + +- Project (93) + +- ShuffledHashJoin Inner BuildRight (92) + :- Exchange (81) + : +- Project (80) + : +- ShuffledHashJoin Inner BuildLeft (79) + : :- Exchange (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Exchange (78) + : +- ShuffledHashJoin LeftSemi BuildRight (77) + : :- Exchange (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- Project (76) + : +- Filter (75) + : +- HashAggregate (74) + : +- Exchange (73) + : +- HashAggregate (72) + : +- Scan parquet (71) + +- ShuffledHashJoin LeftSemi BuildRight (91) + :- Exchange (84) + : +- Filter (83) + : +- Scan parquet (82) + +- Project (90) + +- Filter (89) + +- HashAggregate (88) + +- Exchange (87) + +- HashAggregate (86) + +- Scan parquet (85) (1) Scan parquet @@ -97,420 +100,432 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X, c_name#X] + +(3) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X] Input [2]: [c_custkey#X, c_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(16) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(19) ColumnarExchange +(21) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(20) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(21) InputAdapter +(23) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(22) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(23) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(24) FilterExecTransformer +(26) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(25) ProjectExecTransformer +(27) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(26) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(29) ColumnarExchange +(31) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(32) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(31) InputAdapter +(33) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(32) InputIteratorTransformer +(34) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(33) ShuffledHashJoinExecTransformer +(35) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(34) ProjectExecTransformer +(36) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(36) ColumnarExchange +(38) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(39) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(38) InputAdapter +(40) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(39) InputIteratorTransformer +(41) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(40) Scan parquet +(42) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(41) ProjectExecTransformer +(43) NoopFilter +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X, l_quantity#X] + +(44) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(42) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(43) ColumnarExchange +(46) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(47) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(45) InputAdapter +(48) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(46) InputIteratorTransformer +(49) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(47) ReusedExchange [Reuses operator id: 19] +(50) ReusedExchange [Reuses operator id: 21] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(48) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(49) InputAdapter +(52) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(50) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(52) FilterExecTransformer +(55) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(53) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(54) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) ShuffledHashJoinExecTransformer +(58) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(59) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(57) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(58) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(59) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(60) TakeOrderedAndProjectExecTransformer +(63) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(61) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(62) Scan parquet +(65) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(63) Filter +(66) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(64) Exchange +(67) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(68) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(66) Filter +(69) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(67) Exchange +(70) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(69) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(70) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(72) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(73) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(74) ShuffledHashJoin +(77) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(75) Exchange +(78) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) ShuffledHashJoin +(79) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(77) Project +(80) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(78) Exchange +(81) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Scan parquet +(82) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(80) Filter +(83) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(81) Exchange +(84) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(85) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(83) HashAggregate +(86) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(84) Exchange +(87) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(88) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(86) Filter +(89) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(87) Project +(90) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(88) ShuffledHashJoin +(91) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(89) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(90) Project +(93) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(91) HashAggregate +(94) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(92) HashAggregate +(95) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(93) TakeOrderedAndProject +(96) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(94) AdaptiveSparkPlan +(97) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt index abcddb7ca47a..29b5d69fa0e8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (32) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (31) - +- HashAggregate (30) - +- Project (29) - +- ShuffledHashJoin Inner BuildRight (28) - :- Exchange (24) - : +- Project (23) - : +- Filter (22) - : +- Scan parquet (21) - +- Exchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- HashAggregate (32) + +- Project (31) + +- ShuffledHashJoin Inner BuildRight (30) + :- Exchange (26) + : +- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- Exchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -39,144 +41,152 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(3) ProjectExecTransformer Output [5]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(20) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(21) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(22) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(23) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(24) Exchange +(26) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) Scan parquet +(27) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(28) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) Exchange +(29) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) ShuffledHashJoin +(30) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(31) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(32) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(32) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt index e24568ef1f6c..c5dc33a694ee 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt @@ -1,114 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (123) +- == Final Plan == - VeloxColumnarToRowExec (78) - +- AQEShuffleRead (77) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ ShuffledHashJoinExecTransformer Inner (72) - :- ^ InputIteratorTransformer (64) - : +- ^ InputAdapter (63) - : +- ^ ShuffleQueryStage (62), Statistics(X) - : +- ColumnarExchange (61) - : +- ^ ProjectExecTransformer (59) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (58) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (57) - : +- ^ InputAdapter (56) - : +- ^ ShuffleQueryStage (55), Statistics(X) - : +- ColumnarExchange (54) - : +- ^ ProjectExecTransformer (52) - : +- ^ ShuffledHashJoinExecTransformer Inner (51) - : :- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : +- ColumnarExchange (25) - : : +- ^ ProjectExecTransformer (23) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (22) - : : :- ^ InputIteratorTransformer (14) - : : : +- ^ InputAdapter (13) - : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : +- ColumnarExchange (11) - : : : +- ^ ProjectExecTransformer (9) - : : : +- ^ Scan parquet (8) - : : +- ^ InputIteratorTransformer (21) - : : +- ^ InputAdapter (20) - : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : +- ColumnarExchange (18) - : : +- ^ ProjectExecTransformer (16) - : : +- ^ Scan parquet (15) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ ShuffleQueryStage (48), Statistics(X) - : +- ColumnarExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ FilterExecTransformer (44) - : +- ^ ProjectExecTransformer (43) - : +- ^ RegularHashAggregateExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (40) - : :- ^ InputIteratorTransformer (35) - : : +- ^ InputAdapter (34) - : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : +- ColumnarExchange (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ Scan parquet (29) - : +- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ ShuffleQueryStage (37), Statistics(X) - : +- ReusedExchange (36) - +- ^ InputIteratorTransformer (71) - +- ^ InputAdapter (70) - +- ^ ShuffleQueryStage (69), Statistics(X) - +- ColumnarExchange (68) - +- ^ ProjectExecTransformer (66) - +- ^ Scan parquet (65) + VeloxColumnarToRowExec (83) + +- AQEShuffleRead (82) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (31) + : : +- ^ InputAdapter (30) + : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ NoopFilter (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ^ InputAdapter (23) + : : +- ^ ShuffleQueryStage (22), Statistics(X) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ NoopFilter (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) + : :- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ NoopFilter (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ^ InputAdapter (42) + : +- ^ ShuffleQueryStage (41), Statistics(X) + : +- ReusedExchange (40) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (117) - +- Exchange (116) - +- Project (115) - +- ShuffledHashJoin Inner BuildRight (114) - :- Exchange (109) - : +- Project (108) - : +- ShuffledHashJoin LeftSemi BuildRight (107) - : :- Exchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- Exchange (106) - : +- Project (105) - : +- ShuffledHashJoin Inner BuildLeft (104) - : :- Exchange (90) - : : +- ShuffledHashJoin LeftSemi BuildRight (89) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (88) - : : +- Project (87) - : : +- Filter (86) - : : +- Scan parquet (85) - : +- Exchange (103) - : +- Filter (102) - : +- HashAggregate (101) - : +- HashAggregate (100) - : +- ShuffledHashJoin LeftSemi BuildRight (99) - : :- Exchange (94) - : : +- Project (93) - : : +- Filter (92) - : : +- Scan parquet (91) - : +- Exchange (98) - : +- Project (97) - : +- Filter (96) - : +- Scan parquet (95) - +- Exchange (113) - +- Project (112) - +- Filter (111) - +- Scan parquet (110) + Sort (122) + +- Exchange (121) + +- Project (120) + +- ShuffledHashJoin Inner BuildRight (119) + :- Exchange (114) + : +- Project (113) + : +- ShuffledHashJoin LeftSemi BuildRight (112) + : :- Exchange (86) + : : +- Filter (85) + : : +- Scan parquet (84) + : +- Exchange (111) + : +- Project (110) + : +- ShuffledHashJoin Inner BuildLeft (109) + : :- Exchange (95) + : : +- ShuffledHashJoin LeftSemi BuildRight (94) + : : :- Exchange (89) + : : : +- Filter (88) + : : : +- Scan parquet (87) + : : +- Exchange (93) + : : +- Project (92) + : : +- Filter (91) + : : +- Scan parquet (90) + : +- Exchange (108) + : +- Filter (107) + : +- HashAggregate (106) + : +- HashAggregate (105) + : +- ShuffledHashJoin LeftSemi BuildRight (104) + : :- Exchange (99) + : : +- Project (98) + : : +- Filter (97) + : : +- Scan parquet (96) + : +- Exchange (103) + : +- Project (102) + : +- Filter (101) + : +- Scan parquet (100) + +- Exchange (118) + +- Project (117) + +- Filter (116) + +- Scan parquet (115) (1) Scan parquet @@ -118,500 +123,520 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(11) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(16) ProjectExecTransformer +(18) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(19) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(17) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(18) ColumnarExchange +(21) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(22) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(20) InputAdapter +(23) InputAdapter Input [1]: [p_partkey#X] -(21) InputIteratorTransformer +(24) InputIteratorTransformer Input [1]: [p_partkey#X] -(22) ShuffledHashJoinExecTransformer +(25) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(23) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(29) Scan parquet +(32) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(30) ProjectExecTransformer +(33) NoopFilter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(34) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(31) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(32) ColumnarExchange +(36) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(37) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(34) InputAdapter +(38) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(35) InputIteratorTransformer +(39) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(36) ReusedExchange [Reuses operator id: 18] +(40) ReusedExchange [Reuses operator id: 21] Output [1]: [p_partkey#X] -(37) ShuffleQueryStage +(41) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(38) InputAdapter +(42) InputAdapter Input [1]: [p_partkey#X] -(39) InputIteratorTransformer +(43) InputIteratorTransformer Input [1]: [p_partkey#X] -(40) ShuffledHashJoinExecTransformer +(44) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(42) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) ProjectExecTransformer +(47) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(44) FilterExecTransformer +(48) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(47) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(49) InputAdapter +(53) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(51) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(52) ProjectExecTransformer +(56) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(53) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(54) ColumnarExchange +(58) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(56) InputAdapter +(60) InputAdapter Input [1]: [ps_suppkey#X] -(57) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(58) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(59) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(60) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(61) ColumnarExchange +(65) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(62) ShuffleQueryStage +(66) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(67) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(64) InputIteratorTransformer +(68) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(65) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(66) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(71) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(67) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(68) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(69) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(70) InputAdapter +(75) InputAdapter Input [1]: [n_nationkey#X] -(71) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [n_nationkey#X] -(72) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(73) ProjectExecTransformer +(78) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(74) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(75) ColumnarExchange +(80) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(77) AQEShuffleRead +(82) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(78) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(79) Scan parquet +(84) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(85) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(81) Exchange +(86) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(84) Exchange +(89) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(86) Filter +(91) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(87) Project +(92) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(88) Exchange +(93) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) ShuffledHashJoin +(94) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(90) Exchange +(95) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(96) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(92) Filter +(97) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(93) Project +(98) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(94) Exchange +(99) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(96) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(97) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(100) HashAggregate +(105) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(101) HashAggregate +(106) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(102) Filter +(107) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(103) Exchange +(108) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(105) Project +(110) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(106) Exchange +(111) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(112) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(108) Project +(113) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(109) Exchange +(114) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(111) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(112) Project +(117) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(113) Exchange +(118) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(115) Project +(120) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(116) Exchange +(121) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Sort +(122) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(118) AdaptiveSparkPlan +(123) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt index fec143f3c82c..91bb400aac04 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt @@ -1,108 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (113) +AdaptiveSparkPlan (118) +- == Final Plan == - VeloxColumnarToRowExec (76) - +- ^ RegularHashAggregateExecTransformer (74) - +- ^ InputIteratorTransformer (73) - +- ^ InputAdapter (72) - +- ^ ShuffleQueryStage (71), Statistics(X) - +- ColumnarExchange (70) - +- ^ ProjectExecTransformer (68) - +- ^ FlushableHashAggregateExecTransformer (67) - +- ^ ProjectExecTransformer (66) - +- ^ ShuffledHashJoinExecTransformer Inner (65) - :- ^ InputIteratorTransformer (57) - : +- ^ InputAdapter (56) - : +- ^ ShuffleQueryStage (55), Statistics(X) - : +- ColumnarExchange (54) - : +- ^ ProjectExecTransformer (52) - : +- ^ ShuffledHashJoinExecTransformer Inner (51) - : :- ^ InputIteratorTransformer (43) - : : +- ^ InputAdapter (42) - : : +- ^ ShuffleQueryStage (41), Statistics(X) - : : +- ColumnarExchange (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer Inner (37) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : +- ColumnarExchange (4) - : : : +- ^ ProjectExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (36) - : : +- ^ InputAdapter (35) - : : +- ^ ShuffleQueryStage (34), Statistics(X) - : : +- ColumnarExchange (33) - : : +- ^ ProjectExecTransformer (31) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (30) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (22) - : : : :- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (21) - : : : +- ^ InputAdapter (20) - : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : +- ColumnarExchange (18) - : : : +- ^ ProjectExecTransformer (16) - : : : +- ^ Scan parquet (15) - : : +- ^ InputIteratorTransformer (29) - : : +- ^ InputAdapter (28) - : : +- ^ ShuffleQueryStage (27), Statistics(X) - : : +- ColumnarExchange (26) - : : +- ^ ProjectExecTransformer (24) - : : +- ^ Scan parquet (23) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ ShuffleQueryStage (48), Statistics(X) - : +- ColumnarExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ Scan parquet (44) - +- ^ InputIteratorTransformer (64) - +- ^ InputAdapter (63) - +- ^ ShuffleQueryStage (62), Statistics(X) - +- ColumnarExchange (61) - +- ^ ProjectExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (81) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ^ InputAdapter (77) + +- ^ ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner (70) + :- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ^ InputAdapter (22) + : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ^ InputAdapter (31) + : : +- ^ ShuffleQueryStage (30), Statistics(X) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ NoopFilter (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ^ InputAdapter (68) + +- ^ ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ NoopFilter (63) + +- ^ Scan parquet (62) +- == Initial Plan == - TakeOrderedAndProject (112) - +- HashAggregate (111) - +- Exchange (110) - +- HashAggregate (109) - +- Project (108) - +- ShuffledHashJoin Inner BuildRight (107) - :- Exchange (102) - : +- Project (101) - : +- ShuffledHashJoin Inner BuildRight (100) - : :- Exchange (95) - : : +- Project (94) - : : +- ShuffledHashJoin Inner BuildLeft (93) - : : :- Exchange (79) - : : : +- Filter (78) - : : : +- Scan parquet (77) - : : +- Exchange (92) - : : +- ShuffledHashJoin LeftAnti BuildRight (91) - : : :- ShuffledHashJoin LeftSemi BuildRight (86) - : : : :- Exchange (83) - : : : : +- Project (82) - : : : : +- Filter (81) - : : : : +- Scan parquet (80) - : : : +- Exchange (85) - : : : +- Scan parquet (84) - : : +- Exchange (90) - : : +- Project (89) - : : +- Filter (88) - : : +- Scan parquet (87) - : +- Exchange (99) - : +- Project (98) - : +- Filter (97) - : +- Scan parquet (96) - +- Exchange (106) - +- Project (105) - +- Filter (104) - +- Scan parquet (103) + TakeOrderedAndProject (117) + +- HashAggregate (116) + +- Exchange (115) + +- HashAggregate (114) + +- Project (113) + +- ShuffledHashJoin Inner BuildRight (112) + :- Exchange (107) + : +- Project (106) + : +- ShuffledHashJoin Inner BuildRight (105) + : :- Exchange (100) + : : +- Project (99) + : : +- ShuffledHashJoin Inner BuildLeft (98) + : : :- Exchange (84) + : : : +- Filter (83) + : : : +- Scan parquet (82) + : : +- Exchange (97) + : : +- ShuffledHashJoin LeftAnti BuildRight (96) + : : :- ShuffledHashJoin LeftSemi BuildRight (91) + : : : :- Exchange (88) + : : : : +- Project (87) + : : : : +- Filter (86) + : : : : +- Scan parquet (85) + : : : +- Exchange (90) + : : : +- Scan parquet (89) + : : +- Exchange (95) + : : +- Project (94) + : : +- Filter (93) + : : +- Scan parquet (92) + : +- Exchange (104) + : +- Project (103) + : +- Filter (102) + : +- Scan parquet (101) + +- Exchange (111) + +- Project (110) + +- Filter (109) + +- Scan parquet (108) (1) Scan parquet @@ -112,482 +117,502 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [4]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(22) ShuffledHashJoinExecTransformer +(24) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(23) Scan parquet +(25) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(24) ProjectExecTransformer +(26) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(27) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(25) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(26) ColumnarExchange +(29) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(28) InputAdapter +(31) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(29) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(30) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(31) ProjectExecTransformer +(34) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(33) ColumnarExchange +(36) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(37) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(35) InputAdapter +(38) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(36) InputIteratorTransformer +(39) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(37) ShuffledHashJoinExecTransformer +(40) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(38) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(40) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(41) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(42) InputAdapter +(45) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(43) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(44) Scan parquet +(47) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(45) ProjectExecTransformer +(48) NoopFilter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: [o_orderkey#X, o_orderstatus#X] + +(49) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(46) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(47) ColumnarExchange +(51) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(52) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(49) InputAdapter +(53) InputAdapter Input [1]: [o_orderkey#X] -(50) InputIteratorTransformer +(54) InputIteratorTransformer Input [1]: [o_orderkey#X] -(51) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(52) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(53) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(54) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(56) InputAdapter +(60) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(57) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(58) Scan parquet +(62) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(59) ProjectExecTransformer +(63) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(64) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(60) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(61) ColumnarExchange +(66) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(62) ShuffleQueryStage +(67) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(63) InputAdapter +(68) InputAdapter Input [1]: [n_nationkey#X] -(64) InputIteratorTransformer +(69) InputIteratorTransformer Input [1]: [n_nationkey#X] -(65) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(66) ProjectExecTransformer +(71) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(67) FlushableHashAggregateExecTransformer +(72) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(68) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(69) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(70) ColumnarExchange +(75) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(71) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(72) InputAdapter +(77) InputAdapter Input [2]: [s_name#X, count#X] -(73) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(74) RegularHashAggregateExecTransformer +(79) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(75) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(76) VeloxColumnarToRowExec +(81) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(77) Scan parquet +(82) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(78) Filter +(83) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(79) Exchange +(84) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) Scan parquet +(85) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(81) Filter +(86) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(82) Project +(87) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(83) Exchange +(88) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Scan parquet +(89) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(85) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) ShuffledHashJoin +(91) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(87) Scan parquet +(92) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(88) Filter +(93) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(89) Project +(94) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(90) Exchange +(95) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(94) Project +(99) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(95) Exchange +(100) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(101) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(97) Filter +(102) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(98) Project +(103) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(99) Exchange +(104) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(101) Project +(106) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(102) Exchange +(107) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(108) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(104) Filter +(109) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(105) Project +(110) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(106) Exchange +(111) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(112) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(108) Project +(113) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(109) HashAggregate +(114) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(110) Exchange +(115) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) HashAggregate +(116) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(112) TakeOrderedAndProject +(117) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(113) AdaptiveSparkPlan +(118) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt index 2f5f9d303931..ba18df0d0119 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt @@ -1,46 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (33) + +- ^ SortExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ^ InputAdapter (29) + +- ^ ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- ShuffledHashJoin LeftAnti BuildRight (38) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (37) - +- Scan parquet (36) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- ShuffledHashJoin LeftAnti BuildRight (39) + :- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Exchange (38) + +- Scan parquet (37) (1) Scan parquet @@ -50,300 +51,309 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X, c_phone#X, c_acctbal#X] + +(3) ProjectExecTransformer Output [4]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) Scan parquet +(9) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(9) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(12) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(13) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(13) InputAdapter +(14) InputAdapter Input [1]: [o_custkey#X] -(14) InputIteratorTransformer +(15) InputIteratorTransformer Input [1]: [o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(17) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(20) ColumnarExchange +(21) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) ColumnarExchange +(27) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) SortExecTransformer +(31) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(32) VeloxColumnarToRowExec +(33) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(35) Exchange +(36) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(37) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Exchange +(38) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) ShuffledHashJoin +(39) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(39) Project +(40) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(40) HashAggregate +(41) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(41) Exchange +(42) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(43) Exchange +(44) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (65) +- == Final Plan == - VeloxColumnarToRowExec (56) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ColumnarExchange (50) - +- ^ FlushableHashAggregateExecTransformer (48) - +- ^ ProjectExecTransformer (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ NoopFilter (48) + +- ^ Scan parquet (47) +- == Initial Plan == - HashAggregate (62) - +- Exchange (61) - +- HashAggregate (60) - +- Project (59) - +- Filter (58) - +- Scan parquet (57) + HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(46) Scan parquet +(47) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(47) ProjectExecTransformer +(48) NoopFilter +Input [2]: [c_phone#X, c_acctbal#X] +Arguments: [c_phone#X, c_acctbal#X] + +(49) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(48) FlushableHashAggregateExecTransformer +(50) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(49) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(50) ColumnarExchange +(52) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(52) InputAdapter +(54) InputAdapter Input [2]: [sum#X, count#X] -(53) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [sum#X, count#X] -(54) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(55) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(56) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(57) Scan parquet +(59) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(58) Filter +(60) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(59) Project +(61) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(60) HashAggregate +(62) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(61) Exchange +(63) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(63) AdaptiveSparkPlan +(65) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt index 48b0e5f28558..08588f64d24f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt @@ -1,57 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- TakeOrderedAndProjectExecTransformer (35) - +- ^ ProjectExecTransformer (33) - +- ^ RegularHashAggregateExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ ProjectExecTransformer (30) - +- ^ ShuffledHashJoinExecTransformer Inner (29) - :- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ ShuffleQueryStage (19), Statistics(X) - : +- ColumnarExchange (18) - : +- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12), Statistics(X) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ Scan parquet (22) + VeloxColumnarToRowExec (39) + +- TakeOrderedAndProjectExecTransformer (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (55) - +- HashAggregate (54) - +- HashAggregate (53) - +- Project (52) - +- ShuffledHashJoin Inner BuildRight (51) - :- Exchange (46) - : +- Project (45) - : +- ShuffledHashJoin Inner BuildLeft (44) - : :- Exchange (40) - : : +- Project (39) - : : +- Filter (38) - : : +- Scan parquet (37) - : +- Exchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Exchange (50) - +- Project (49) - +- Filter (48) - +- Scan parquet (47) + TakeOrderedAndProject (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (49) + : +- Project (48) + : +- ShuffledHashJoin Inner BuildLeft (47) + : :- Exchange (43) + : : +- Project (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Exchange (53) + +- Project (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -61,244 +64,256 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: [c_custkey#X, c_mktsegment#X] + +(3) ProjectExecTransformer Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(11) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(32) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(33) ProjectExecTransformer +(36) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(35) TakeOrderedAndProjectExecTransformer +(38) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(37) Scan parquet +(40) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(38) Filter +(41) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(39) Project +(42) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(40) Exchange +(43) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(44) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(42) Filter +(45) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(43) Exchange +(46) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(45) Project +(48) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(46) Exchange +(49) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Scan parquet +(50) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(48) Filter +(51) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(49) Project +(52) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(50) Exchange +(53) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(52) Project +(55) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(53) HashAggregate +(56) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(54) HashAggregate +(57) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(55) TakeOrderedAndProject +(58) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(56) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt index 33b40f964185..421f6a412ec7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt @@ -1,49 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (48) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (47) - +- Exchange (46) - +- HashAggregate (45) - +- Exchange (44) - +- HashAggregate (43) - +- Project (42) - +- ShuffledHashJoin LeftSemi BuildRight (41) - :- Exchange (36) - : +- Project (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftSemi BuildRight (43) + :- Exchange (38) + : +- Project (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -53,204 +55,212 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [1]: [l_orderkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [l_orderkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(26) ColumnarExchange +(28) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(28) InputAdapter +(30) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(29) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(30) SortExecTransformer +(32) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(32) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(33) Scan parquet +(35) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(34) Filter +(36) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(35) Project +(37) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(36) Exchange +(38) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(39) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(38) Filter +(40) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(39) Project +(41) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(40) Exchange +(42) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) Project +(44) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(43) HashAggregate +(45) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(44) Exchange +(46) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(46) Exchange +(48) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Sort +(49) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(48) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt index 719d3611db67..50173820058c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt @@ -1,121 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (134) +- == Final Plan == - VeloxColumnarToRowExec (88) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ^ InputAdapter (78) - +- ^ ShuffleQueryStage (77), Statistics(X) - +- ColumnarExchange (76) - +- ^ ProjectExecTransformer (74) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ ShuffledHashJoinExecTransformer Inner (71) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61), Statistics(X) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54), Statistics(X) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ ShuffleQueryStage (68), Statistics(X) - +- ColumnarExchange (67) - +- ^ ProjectExecTransformer (65) - +- ^ Scan parquet (64) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildRight (108) - : : :- Exchange (104) - : : : +- Project (103) - : : : +- ShuffledHashJoin Inner BuildRight (102) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- ShuffledHashJoin Inner BuildLeft (96) - : : : : :- Exchange (91) - : : : : : +- Filter (90) - : : : : : +- Scan parquet (89) - : : : : +- Exchange (95) - : : : : +- Project (94) - : : : : +- Filter (93) - : : : : +- Scan parquet (92) - : : : +- Exchange (101) - : : : +- Filter (100) - : : : +- Scan parquet (99) - : : +- Exchange (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Exchange (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (120) - +- Project (119) - +- Filter (118) - +- Scan parquet (117) + Sort (133) + +- Exchange (132) + +- HashAggregate (131) + +- Exchange (130) + +- HashAggregate (129) + +- Project (128) + +- ShuffledHashJoin Inner BuildRight (127) + :- Exchange (122) + : +- Project (121) + : +- ShuffledHashJoin Inner BuildRight (120) + : :- Exchange (116) + : : +- Project (115) + : : +- ShuffledHashJoin Inner BuildRight (114) + : : :- Exchange (110) + : : : +- Project (109) + : : : +- ShuffledHashJoin Inner BuildRight (108) + : : : :- Exchange (104) + : : : : +- Project (103) + : : : : +- ShuffledHashJoin Inner BuildLeft (102) + : : : : :- Exchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Exchange (101) + : : : : +- Project (100) + : : : : +- Filter (99) + : : : : +- Scan parquet (98) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Exchange (126) + +- Project (125) + +- Filter (124) + +- Scan parquet (123) (1) Scan parquet @@ -125,540 +131,564 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(26) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(41) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(50) Scan parquet +(54) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] + +(56) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(71) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [1]: [r_regionkey#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [r_regionkey#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(73) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(74) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(82) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(83) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(84) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(79) InputIteratorTransformer +(85) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(81) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(82) ColumnarExchange +(88) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(84) InputAdapter +(90) InputAdapter Input [2]: [n_name#X, revenue#X] -(85) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(86) SortExecTransformer +(92) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(89) Scan parquet +(95) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(90) Filter +(96) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(91) Exchange +(97) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Scan parquet +(98) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(93) Filter +(99) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(94) Project +(100) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(95) Exchange +(101) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(97) Project +(103) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(98) Exchange +(104) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(105) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(100) Filter +(106) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(101) Exchange +(107) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(103) Project +(109) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(110) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(111) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(112) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(113) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(109) Project +(115) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(116) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(117) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(113) Exchange +(119) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(115) Project +(121) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(116) Exchange +(122) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(123) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(118) Filter +(124) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(119) Project +(125) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(120) Exchange +(126) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(127) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(122) Project +(128) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(123) HashAggregate +(129) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(124) Exchange +(130) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(131) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(126) Exchange +(132) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(133) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(128) AdaptiveSparkPlan +(134) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt index 2e51f22a6b0e..51c5836bdd11 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (18) +AdaptiveSparkPlan (19) +- == Final Plan == - VeloxColumnarToRowExec (11) - +- ^ RegularHashAggregateExecTransformer (9) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ ShuffleQueryStage (6), Statistics(X) - +- ColumnarExchange (5) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (17) - +- Exchange (16) - +- HashAggregate (15) - +- Project (14) - +- Filter (13) - +- Scan parquet (12) + HashAggregate (18) + +- Exchange (17) + +- HashAggregate (16) + +- Project (15) + +- Filter (14) + +- Scan parquet (13) (1) Scan parquet @@ -26,82 +27,86 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(5) ColumnarExchange +(6) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [sum#X, isEmpty#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(9) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(11) VeloxColumnarToRowExec +(12) VeloxColumnarToRowExec Input [1]: [revenue#X] -(12) Scan parquet +(13) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(13) Filter +(14) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(14) Project +(15) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) HashAggregate +(16) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(16) Exchange +(17) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(17) HashAggregate +(18) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(18) AdaptiveSparkPlan +(19) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt index eca3692340aa..67f4274b3aa3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt @@ -1,117 +1,122 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (85) - +- ^ SortExecTransformer (83) - +- ^ InputIteratorTransformer (82) - +- ^ InputAdapter (81) - +- ^ ShuffleQueryStage (80), Statistics(X) - +- ColumnarExchange (79) - +- ^ RegularHashAggregateExecTransformer (77) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FlushableHashAggregateExecTransformer (70) - +- ^ ProjectExecTransformer (69) - +- ^ ShuffledHashJoinExecTransformer Inner (68) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61), Statistics(X) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54), Statistics(X) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (67) - +- ^ InputAdapter (66) - +- ^ ShuffleQueryStage (65), Statistics(X) - +- ReusedExchange (64) + VeloxColumnarToRowExec (90) + +- ^ SortExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ^ InputAdapter (86) + +- ^ ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ^ InputAdapter (80) + +- ^ ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner (73) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ^ InputAdapter (71) + +- ^ ShuffleQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- HashAggregate (120) - +- Exchange (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (112) - : +- Project (111) - : +- ShuffledHashJoin Inner BuildRight (110) - : :- Exchange (106) - : : +- Project (105) - : : +- ShuffledHashJoin Inner BuildRight (104) - : : :- Exchange (100) - : : : +- Project (99) - : : : +- ShuffledHashJoin Inner BuildRight (98) - : : : :- Exchange (94) - : : : : +- Project (93) - : : : : +- ShuffledHashJoin Inner BuildLeft (92) - : : : : :- Exchange (88) - : : : : : +- Filter (87) - : : : : : +- Scan parquet (86) - : : : : +- Exchange (91) - : : : : +- Filter (90) - : : : : +- Scan parquet (89) - : : : +- Exchange (97) - : : : +- Filter (96) - : : : +- Scan parquet (95) - : : +- Exchange (103) - : : +- Filter (102) - : : +- Scan parquet (101) - : +- Exchange (109) - : +- Filter (108) - : +- Scan parquet (107) - +- Exchange (115) - +- Filter (114) - +- Scan parquet (113) + Sort (127) + +- Exchange (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (111) + : : +- Project (110) + : : +- ShuffledHashJoin Inner BuildRight (109) + : : :- Exchange (105) + : : : +- Project (104) + : : : +- ShuffledHashJoin Inner BuildRight (103) + : : : :- Exchange (99) + : : : : +- Project (98) + : : : : +- ShuffledHashJoin Inner BuildLeft (97) + : : : : :- Exchange (93) + : : : : : +- Filter (92) + : : : : : +- Scan parquet (91) + : : : : +- Exchange (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (102) + : : : +- Filter (101) + : : : +- Scan parquet (100) + : : +- Exchange (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (114) + : +- Filter (113) + : +- Scan parquet (112) + +- Exchange (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -121,516 +126,536 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X, o_custkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(41) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(56) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(64) ReusedExchange [Reuses operator id: 53] +(69) ReusedExchange [Reuses operator id: 58] Output [2]: [n_nationkey#X, n_name#X] -(65) ShuffleQueryStage +(70) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(66) InputAdapter +(71) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(67) InputIteratorTransformer +(72) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(68) ShuffledHashJoinExecTransformer +(73) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(69) ProjectExecTransformer +(74) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(70) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(71) ProjectExecTransformer +(76) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(72) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(73) ColumnarExchange +(78) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(79) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(75) InputAdapter +(80) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) InputIteratorTransformer +(81) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(78) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(79) ColumnarExchange +(84) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(80) ShuffleQueryStage +(85) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(81) InputAdapter +(86) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(82) InputIteratorTransformer +(87) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(83) SortExecTransformer +(88) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(84) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(85) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(86) Scan parquet +(91) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(87) Filter +(92) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(88) Exchange +(93) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(94) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(95) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) Exchange +(96) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(93) Project +(98) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(94) Exchange +(99) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(100) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(96) Filter +(101) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(97) Exchange +(102) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(103) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(99) Project +(104) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(100) Exchange +(105) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(102) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(103) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(105) Project +(110) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(106) Exchange +(111) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) Scan parquet +(112) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(108) Filter +(113) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(109) Exchange +(114) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(111) Project +(116) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(112) Exchange +(117) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(114) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(115) Exchange +(120) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(117) Project +(122) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(118) HashAggregate +(123) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(119) Exchange +(124) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) HashAggregate +(125) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(121) Exchange +(126) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(127) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt index 1883d0df585f..16838a0513b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt @@ -1,158 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (169) +AdaptiveSparkPlan (177) +- == Final Plan == - VeloxColumnarToRowExec (117) - +- ^ SortExecTransformer (115) - +- ^ InputIteratorTransformer (114) - +- ^ InputAdapter (113) - +- ^ ShuffleQueryStage (112), Statistics(X) - +- ColumnarExchange (111) - +- ^ ProjectExecTransformer (109) - +- ^ RegularHashAggregateExecTransformer (108) - +- ^ InputIteratorTransformer (107) - +- ^ InputAdapter (106) - +- ^ ShuffleQueryStage (105), Statistics(X) - +- ColumnarExchange (104) - +- ^ ProjectExecTransformer (102) - +- ^ FlushableHashAggregateExecTransformer (101) - +- ^ ProjectExecTransformer (100) - +- ^ ShuffledHashJoinExecTransformer Inner (99) - :- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner (85) - : :- ^ InputIteratorTransformer (77) - : : +- ^ InputAdapter (76) - : : +- ^ ShuffleQueryStage (75), Statistics(X) - : : +- ColumnarExchange (74) - : : +- ^ ProjectExecTransformer (72) - : : +- ^ ShuffledHashJoinExecTransformer Inner (71) - : : :- ^ InputIteratorTransformer (63) - : : : +- ^ InputAdapter (62) - : : : +- ^ ShuffleQueryStage (61), Statistics(X) - : : : +- ColumnarExchange (60) - : : : +- ^ ProjectExecTransformer (58) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : : : :- ^ InputIteratorTransformer (49) - : : : : +- ^ InputAdapter (48) - : : : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : : : +- ColumnarExchange (46) - : : : : +- ^ ProjectExecTransformer (44) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : : : :- ^ InputIteratorTransformer (35) - : : : : : +- ^ InputAdapter (34) - : : : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : : : +- ColumnarExchange (32) - : : : : : +- ^ ProjectExecTransformer (30) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : : : :- ^ InputIteratorTransformer (21) - : : : : : : +- ^ InputAdapter (20) - : : : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : : : +- ColumnarExchange (18) - : : : : : : +- ^ ProjectExecTransformer (16) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : : : :- ^ InputIteratorTransformer (7) - : : : : : : : +- ^ InputAdapter (6) - : : : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : : : +- ColumnarExchange (4) - : : : : : : : +- ^ ProjectExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (14) - : : : : : : +- ^ InputAdapter (13) - : : : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : : : +- ColumnarExchange (11) - : : : : : : +- ^ ProjectExecTransformer (9) - : : : : : : +- ^ Scan parquet (8) - : : : : : +- ^ InputIteratorTransformer (28) - : : : : : +- ^ InputAdapter (27) - : : : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : : : +- ColumnarExchange (25) - : : : : : +- ^ ProjectExecTransformer (23) - : : : : : +- ^ Scan parquet (22) - : : : : +- ^ InputIteratorTransformer (42) - : : : : +- ^ InputAdapter (41) - : : : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : : : +- ColumnarExchange (39) - : : : : +- ^ ProjectExecTransformer (37) - : : : : +- ^ Scan parquet (36) - : : : +- ^ InputIteratorTransformer (56) - : : : +- ^ InputAdapter (55) - : : : +- ^ ShuffleQueryStage (54), Statistics(X) - : : : +- ColumnarExchange (53) - : : : +- ^ ProjectExecTransformer (51) - : : : +- ^ Scan parquet (50) - : : +- ^ InputIteratorTransformer (70) - : : +- ^ InputAdapter (69) - : : +- ^ ShuffleQueryStage (68), Statistics(X) - : : +- ColumnarExchange (67) - : : +- ^ ProjectExecTransformer (65) - : : +- ^ Scan parquet (64) - : +- ^ InputIteratorTransformer (84) - : +- ^ InputAdapter (83) - : +- ^ ShuffleQueryStage (82), Statistics(X) - : +- ColumnarExchange (81) - : +- ^ ProjectExecTransformer (79) - : +- ^ Scan parquet (78) - +- ^ InputIteratorTransformer (98) - +- ^ InputAdapter (97) - +- ^ ShuffleQueryStage (96), Statistics(X) - +- ColumnarExchange (95) - +- ^ ProjectExecTransformer (93) - +- ^ Scan parquet (92) + VeloxColumnarToRowExec (125) + +- ^ SortExecTransformer (123) + +- ^ InputIteratorTransformer (122) + +- ^ InputAdapter (121) + +- ^ ShuffleQueryStage (120), Statistics(X) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ^ InputAdapter (114) + +- ^ ShuffleQueryStage (113), Statistics(X) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner (107) + :- ^ InputIteratorTransformer (98) + : +- ^ InputAdapter (97) + : +- ^ ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner (92) + : :- ^ InputIteratorTransformer (83) + : : +- ^ InputAdapter (82) + : : +- ^ ShuffleQueryStage (81), Statistics(X) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ^ InputAdapter (67) + : : : +- ^ ShuffleQueryStage (66), Statistics(X) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ^ InputAdapter (52) + : : : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ^ InputAdapter (37) + : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ^ InputAdapter (22) + : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ^ InputAdapter (7) + : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ^ InputAdapter (15) + : : : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ^ InputAdapter (30) + : : : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ NoopFilter (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ^ InputAdapter (45) + : : : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ NoopFilter (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ^ InputAdapter (60) + : : : +- ^ ShuffleQueryStage (59), Statistics(X) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ NoopFilter (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ^ InputAdapter (75) + : : +- ^ ShuffleQueryStage (74), Statistics(X) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ NoopFilter (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ^ InputAdapter (90) + : +- ^ ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ NoopFilter (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ^ InputAdapter (105) + +- ^ ShuffleQueryStage (104), Statistics(X) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ NoopFilter (100) + +- ^ Scan parquet (99) +- == Initial Plan == - Sort (168) - +- Exchange (167) - +- HashAggregate (166) - +- Exchange (165) - +- HashAggregate (164) - +- Project (163) - +- ShuffledHashJoin Inner BuildRight (162) - :- Exchange (157) - : +- Project (156) - : +- ShuffledHashJoin Inner BuildRight (155) - : :- Exchange (151) - : : +- Project (150) - : : +- ShuffledHashJoin Inner BuildRight (149) - : : :- Exchange (145) - : : : +- Project (144) - : : : +- ShuffledHashJoin Inner BuildRight (143) - : : : :- Exchange (139) - : : : : +- Project (138) - : : : : +- ShuffledHashJoin Inner BuildRight (137) - : : : : :- Exchange (133) - : : : : : +- Project (132) - : : : : : +- ShuffledHashJoin Inner BuildRight (131) - : : : : : :- Exchange (127) - : : : : : : +- Project (126) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (125) - : : : : : : :- Exchange (121) - : : : : : : : +- Project (120) - : : : : : : : +- Filter (119) - : : : : : : : +- Scan parquet (118) - : : : : : : +- Exchange (124) - : : : : : : +- Filter (123) - : : : : : : +- Scan parquet (122) - : : : : : +- Exchange (130) - : : : : : +- Filter (129) - : : : : : +- Scan parquet (128) - : : : : +- Exchange (136) - : : : : +- Filter (135) - : : : : +- Scan parquet (134) - : : : +- Exchange (142) - : : : +- Filter (141) - : : : +- Scan parquet (140) - : : +- Exchange (148) - : : +- Filter (147) - : : +- Scan parquet (146) - : +- Exchange (154) - : +- Filter (153) - : +- Scan parquet (152) - +- Exchange (161) - +- Project (160) - +- Filter (159) - +- Scan parquet (158) + Sort (176) + +- Exchange (175) + +- HashAggregate (174) + +- Exchange (173) + +- HashAggregate (172) + +- Project (171) + +- ShuffledHashJoin Inner BuildRight (170) + :- Exchange (165) + : +- Project (164) + : +- ShuffledHashJoin Inner BuildRight (163) + : :- Exchange (159) + : : +- Project (158) + : : +- ShuffledHashJoin Inner BuildRight (157) + : : :- Exchange (153) + : : : +- Project (152) + : : : +- ShuffledHashJoin Inner BuildRight (151) + : : : :- Exchange (147) + : : : : +- Project (146) + : : : : +- ShuffledHashJoin Inner BuildRight (145) + : : : : :- Exchange (141) + : : : : : +- Project (140) + : : : : : +- ShuffledHashJoin Inner BuildRight (139) + : : : : : :- Exchange (135) + : : : : : : +- Project (134) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) + : : : : : : :- Exchange (129) + : : : : : : : +- Project (128) + : : : : : : : +- Filter (127) + : : : : : : : +- Scan parquet (126) + : : : : : : +- Exchange (132) + : : : : : : +- Filter (131) + : : : : : : +- Scan parquet (130) + : : : : : +- Exchange (138) + : : : : : +- Filter (137) + : : : : : +- Scan parquet (136) + : : : : +- Exchange (144) + : : : : +- Filter (143) + : : : : +- Scan parquet (142) + : : : +- Exchange (150) + : : : +- Filter (149) + : : : +- Scan parquet (148) + : : +- Exchange (156) + : : +- Filter (155) + : : +- Scan parquet (154) + : +- Exchange (162) + : +- Filter (161) + : +- Scan parquet (160) + +- Exchange (169) + +- Project (168) + +- Filter (167) + +- Scan parquet (166) (1) Scan parquet @@ -162,712 +170,744 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(3) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(11) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(36) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(41) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(56) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_regionkey#X] + +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(73) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(74) ColumnarExchange +(80) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(75) ShuffleQueryStage +(81) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(76) InputAdapter +(82) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(77) InputIteratorTransformer +(83) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) ProjectExecTransformer +(85) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(86) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(80) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(81) ColumnarExchange +(88) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(82) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(83) InputAdapter +(90) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(84) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(85) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) ProjectExecTransformer +(93) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(95) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(92) Scan parquet +(99) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(93) ProjectExecTransformer +(100) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(101) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(94) WholeStageCodegenTransformer (X) +(102) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(95) ColumnarExchange +(103) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(104) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(97) InputAdapter +(105) InputAdapter Input [1]: [r_regionkey#X] -(98) InputIteratorTransformer +(106) InputIteratorTransformer Input [1]: [r_regionkey#X] -(99) ShuffledHashJoinExecTransformer +(107) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(100) ProjectExecTransformer +(108) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(101) FlushableHashAggregateExecTransformer +(109) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(102) ProjectExecTransformer +(110) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(103) WholeStageCodegenTransformer (X) +(111) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(104) ColumnarExchange +(112) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(105) ShuffleQueryStage +(113) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(106) InputAdapter +(114) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(107) InputIteratorTransformer +(115) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(108) RegularHashAggregateExecTransformer +(116) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(109) ProjectExecTransformer +(117) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(110) WholeStageCodegenTransformer (X) +(118) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(111) ColumnarExchange +(119) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(112) ShuffleQueryStage +(120) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(113) InputAdapter +(121) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(114) InputIteratorTransformer +(122) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(115) SortExecTransformer +(123) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(116) WholeStageCodegenTransformer (X) +(124) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(117) VeloxColumnarToRowExec +(125) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(118) Scan parquet +(126) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(119) Filter +(127) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(120) Project +(128) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(121) Exchange +(129) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(130) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(123) Filter +(131) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(124) Exchange +(132) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(133) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(126) Project +(134) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(127) Exchange +(135) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(136) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(129) Filter +(137) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(130) Exchange +(138) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(132) Project +(140) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(133) Exchange +(141) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(142) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(135) Filter +(143) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(136) Exchange +(144) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) ShuffledHashJoin +(145) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(138) Project +(146) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(139) Exchange +(147) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(140) Scan parquet +(148) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(141) Filter +(149) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(142) Exchange +(150) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) ShuffledHashJoin +(151) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(144) Project +(152) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(145) Exchange +(153) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Scan parquet +(154) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(147) Filter +(155) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(148) Exchange +(156) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(149) ShuffledHashJoin +(157) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(150) Project +(158) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(151) Exchange +(159) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Scan parquet +(160) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(153) Filter +(161) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(154) Exchange +(162) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(155) ShuffledHashJoin +(163) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(156) Project +(164) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(157) Exchange +(165) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(158) Scan parquet +(166) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(159) Filter +(167) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(160) Project +(168) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(161) Exchange +(169) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(162) ShuffledHashJoin +(170) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(163) Project +(171) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(164) HashAggregate +(172) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(165) Exchange +(173) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) HashAggregate +(174) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(167) Exchange +(175) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(168) Sort +(176) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(169) AdaptiveSparkPlan +(177) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt index feffe6710113..a24b228c5f77 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt @@ -1,120 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (127) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (88) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ^ InputAdapter (78) - +- ^ ShuffleQueryStage (77), Statistics(X) - +- ColumnarExchange (76) - +- ^ ProjectExecTransformer (74) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ ShuffledHashJoinExecTransformer Inner (71) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61), Statistics(X) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54), Statistics(X) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ ShuffleQueryStage (68), Statistics(X) - +- ColumnarExchange (67) - +- ^ ProjectExecTransformer (65) - +- ^ Scan parquet (64) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (126) - +- Exchange (125) - +- HashAggregate (124) - +- Exchange (123) - +- HashAggregate (122) - +- Project (121) - +- ShuffledHashJoin Inner BuildRight (120) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildRight (108) - : : :- Exchange (104) - : : : +- Project (103) - : : : +- ShuffledHashJoin Inner BuildRight (102) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- ShuffledHashJoin Inner BuildLeft (96) - : : : : :- Exchange (92) - : : : : : +- Project (91) - : : : : : +- Filter (90) - : : : : : +- Scan parquet (89) - : : : : +- Exchange (95) - : : : : +- Filter (94) - : : : : +- Scan parquet (93) - : : : +- Exchange (101) - : : : +- Filter (100) - : : : +- Scan parquet (99) - : : +- Exchange (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Exchange (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (119) - +- Filter (118) - +- Scan parquet (117) + Sort (132) + +- Exchange (131) + +- HashAggregate (130) + +- Exchange (129) + +- HashAggregate (128) + +- Project (127) + +- ShuffledHashJoin Inner BuildRight (126) + :- Exchange (122) + : +- Project (121) + : +- ShuffledHashJoin Inner BuildRight (120) + : :- Exchange (116) + : : +- Project (115) + : : +- ShuffledHashJoin Inner BuildRight (114) + : : :- Exchange (110) + : : : +- Project (109) + : : : +- ShuffledHashJoin Inner BuildRight (108) + : : : :- Exchange (104) + : : : : +- Project (103) + : : : : +- ShuffledHashJoin Inner BuildLeft (102) + : : : : :- Exchange (98) + : : : : : +- Project (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Exchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Exchange (125) + +- Filter (124) + +- Scan parquet (123) (1) Scan parquet @@ -124,536 +130,560 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(3) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(11) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(36) Scan parquet +(39) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(41) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_orderdate#X] + +(56) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(73) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(74) ProjectExecTransformer +(80) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(82) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(83) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(84) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(79) InputIteratorTransformer +(85) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(81) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(82) ColumnarExchange +(88) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(84) InputAdapter +(90) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(85) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(86) SortExecTransformer +(92) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(89) Scan parquet +(95) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(90) Filter +(96) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(91) Project +(97) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(92) Exchange +(98) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Scan parquet +(99) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(94) Filter +(100) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(95) Exchange +(101) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(97) Project +(103) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(98) Exchange +(104) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(105) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(100) Filter +(106) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(101) Exchange +(107) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(103) Project +(109) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(104) Exchange +(110) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(111) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(106) Filter +(112) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(107) Exchange +(113) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(109) Project +(115) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(110) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(117) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(113) Exchange +(119) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(115) Project +(121) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(116) Exchange +(122) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(118) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(119) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(127) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(122) HashAggregate +(128) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(123) Exchange +(129) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) HashAggregate +(130) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(125) Exchange +(131) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) Sort +(132) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(127) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt index 0217849e5de2..41613c36f7bd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (27) +AdaptiveSparkPlan (28) +- == Final Plan == - VeloxColumnarToRowExec (18) - +- ^ SortExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (19) + +- ^ SortExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (26) - +- Exchange (25) - +- HashAggregate (24) - +- Exchange (23) - +- HashAggregate (22) - +- Project (21) - +- Filter (20) - +- Scan parquet (19) + Sort (27) + +- Exchange (26) + +- HashAggregate (25) + +- Exchange (24) + +- HashAggregate (23) + +- Project (22) + +- Filter (21) + +- Scan parquet (20) (1) Scan parquet @@ -34,116 +35,120 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X, ((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(4) ProjectExecTransformer +(5) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(6) ColumnarExchange +(7) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(12) ColumnarExchange +(13) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(14) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) InputAdapter +(15) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(15) InputIteratorTransformer +(16) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) SortExecTransformer +(17) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(17) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(18) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(19) Scan parquet +(20) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(20) Filter +(21) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(21) Project +(22) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(22) HashAggregate +(23) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(23) Exchange +(24) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(24) HashAggregate +(25) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(25) Exchange +(26) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Sort +(27) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(27) AdaptiveSparkPlan +(28) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt index d88b2d72aa24..79ff600fa6b7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt @@ -1,81 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (83) +AdaptiveSparkPlan (87) +- == Final Plan == - VeloxColumnarToRowExec (56) - +- TakeOrderedAndProjectExecTransformer (55) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ ShuffledHashJoinExecTransformer Inner (43) - :- ^ InputIteratorTransformer (35) - : +- ^ InputAdapter (34) - : +- ^ ShuffleQueryStage (33), Statistics(X) - : +- ColumnarExchange (32) - : +- ^ ProjectExecTransformer (30) - : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : :- ^ InputIteratorTransformer (21) - : : +- ^ InputAdapter (20) - : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : +- ColumnarExchange (18) - : : +- ^ ProjectExecTransformer (16) - : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : +- ColumnarExchange (4) - : : : +- ^ ProjectExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (14) - : : +- ^ InputAdapter (13) - : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : +- ColumnarExchange (11) - : : +- ^ ProjectExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (28) - : +- ^ InputAdapter (27) - : +- ^ ShuffleQueryStage (26), Statistics(X) - : +- ColumnarExchange (25) - : +- ^ ProjectExecTransformer (23) - : +- ^ Scan parquet (22) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ ShuffleQueryStage (40), Statistics(X) - +- ColumnarExchange (39) - +- ^ ProjectExecTransformer (37) - +- ^ Scan parquet (36) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner (47) + :- ^ InputIteratorTransformer (38) + : +- ^ InputAdapter (37) + : +- ^ ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : :- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (82) - +- HashAggregate (81) - +- Exchange (80) - +- HashAggregate (79) - +- Project (78) - +- ShuffledHashJoin Inner BuildRight (77) - :- Exchange (73) - : +- Project (72) - : +- ShuffledHashJoin Inner BuildRight (71) - : :- Exchange (66) - : : +- Project (65) - : : +- ShuffledHashJoin Inner BuildRight (64) - : : :- Exchange (59) - : : : +- Filter (58) - : : : +- Scan parquet (57) - : : +- Exchange (63) - : : +- Project (62) - : : +- Filter (61) - : : +- Scan parquet (60) - : +- Exchange (70) - : +- Project (69) - : +- Filter (68) - : +- Scan parquet (67) - +- Exchange (76) - +- Filter (75) - +- Scan parquet (74) + TakeOrderedAndProject (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- ShuffledHashJoin Inner BuildRight (81) + :- Exchange (77) + : +- Project (76) + : +- ShuffledHashJoin Inner BuildRight (75) + : :- Exchange (70) + : : +- Project (69) + : : +- ShuffledHashJoin Inner BuildRight (68) + : : :- Exchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- Exchange (67) + : : +- Project (66) + : : +- Filter (65) + : : +- Scan parquet (64) + : +- Exchange (74) + : +- Project (73) + : +- Filter (72) + : +- Scan parquet (71) + +- Exchange (80) + +- Filter (79) + +- Scan parquet (78) (1) Scan parquet @@ -85,360 +89,376 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(3) ProjectExecTransformer Output [8]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(41) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(45) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(50) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(52) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(53) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(54) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(55) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(53) ProjectExecTransformer +(57) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(55) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(56) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(57) Scan parquet +(61) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(58) Filter +(62) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(59) Exchange +(63) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(64) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(61) Filter +(65) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(62) Project +(66) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(63) Exchange +(67) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(68) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(65) Project +(69) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(66) Exchange +(70) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(67) Scan parquet +(71) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(68) Filter +(72) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(69) Project +(73) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(70) Exchange +(74) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(72) Project +(76) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(73) Exchange +(77) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(75) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(76) Exchange +(80) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(79) HashAggregate +(83) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(80) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(85) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(82) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(83) AdaptiveSparkPlan +(87) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt index 486fe217ad6f..1b5a8743db5a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt @@ -1,68 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (72) +- == Final Plan == - VeloxColumnarToRowExec (47) - +- ^ SortExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ^ InputAdapter (43) - +- ^ ShuffleQueryStage (42), Statistics(X) - +- ColumnarExchange (41) - +- ^ FilterExecTransformer (39) - +- ^ RegularHashAggregateExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ ShuffleQueryStage (35), Statistics(X) - +- ColumnarExchange (34) - +- ^ ProjectExecTransformer (32) - +- ^ FlushableHashAggregateExecTransformer (31) - +- ^ ProjectExecTransformer (30) - +- ^ ShuffledHashJoinExecTransformer Inner (29) - :- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ ShuffleQueryStage (19), Statistics(X) - : +- ColumnarExchange (18) - : +- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12), Statistics(X) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ Scan parquet (22) + VeloxColumnarToRowExec (50) + +- ^ SortExecTransformer (48) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ ShuffleQueryStage (38), Statistics(X) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - Sort (68) - +- Exchange (67) - +- Filter (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- ShuffledHashJoin Inner BuildRight (61) - :- Exchange (56) - : +- Project (55) - : +- ShuffledHashJoin Inner BuildRight (54) - : :- Exchange (50) - : : +- Filter (49) - : : +- Scan parquet (48) - : +- Exchange (53) - : +- Filter (52) - : +- Scan parquet (51) - +- Exchange (60) - +- Project (59) - +- Filter (58) - +- Scan parquet (57) + Sort (71) + +- Exchange (70) + +- Filter (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- ShuffledHashJoin Inner BuildRight (64) + :- Exchange (59) + : +- Project (58) + : +- ShuffledHashJoin Inner BuildRight (57) + : :- Exchange (53) + : : +- Filter (52) + : : +- Scan parquet (51) + : +- Exchange (56) + : +- Filter (55) + : +- Scan parquet (54) + +- Exchange (63) + +- Project (62) + +- Filter (61) + +- Scan parquet (60) (1) Scan parquet @@ -72,556 +75,573 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(3) ProjectExecTransformer Output [5]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) Scan parquet +(9) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(11) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(26) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [1]: [n_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [1]: [n_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(31) FlushableHashAggregateExecTransformer +(34) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(34) ColumnarExchange +(37) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(38) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(36) InputAdapter +(39) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(37) InputIteratorTransformer +(40) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(38) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(39) FilterExecTransformer +(42) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(40) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(41) ColumnarExchange +(44) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(45) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(43) InputAdapter +(46) InputAdapter Input [2]: [ps_partkey#X, value#X] -(44) InputIteratorTransformer +(47) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(45) SortExecTransformer +(48) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(46) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(47) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(48) Scan parquet +(51) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(49) Filter +(52) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(50) Exchange +(53) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(53) Exchange +(56) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(55) Project +(58) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(56) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Scan parquet +(60) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(58) Filter +(61) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(59) Project +(62) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(60) Exchange +(63) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) ShuffledHashJoin +(64) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(62) Project +(65) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(63) HashAggregate +(66) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(64) Exchange +(67) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(68) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(66) Filter +(69) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(67) Exchange +(70) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Sort +(71) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(69) AdaptiveSparkPlan +(72) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 39 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (116) +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (120) +- == Final Plan == - VeloxColumnarToRowExec (98) - +- ^ ProjectExecTransformer (96) - +- ^ RegularHashAggregateExecTransformer (95) - +- ^ RegularHashAggregateExecTransformer (94) - +- ^ ProjectExecTransformer (93) - +- ^ ShuffledHashJoinExecTransformer Inner (92) - :- ^ InputIteratorTransformer (87) - : +- ^ InputAdapter (86) - : +- ^ ShuffleQueryStage (85), Statistics(X) - : +- ColumnarExchange (84) - : +- ^ ProjectExecTransformer (82) - : +- ^ ShuffledHashJoinExecTransformer Inner (81) - : :- ^ InputIteratorTransformer (76) - : : +- ^ InputAdapter (75) - : : +- ^ ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ Scan parquet (70) - : +- ^ InputIteratorTransformer (80) - : +- ^ InputAdapter (79) - : +- ^ ShuffleQueryStage (78), Statistics(X) - : +- ReusedExchange (77) - +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ReusedExchange (88) + VeloxColumnarToRowExec (102) + +- ^ ProjectExecTransformer (100) + +- ^ RegularHashAggregateExecTransformer (99) + +- ^ RegularHashAggregateExecTransformer (98) + +- ^ ProjectExecTransformer (97) + +- ^ ShuffledHashJoinExecTransformer Inner (96) + :- ^ InputIteratorTransformer (91) + : +- ^ InputAdapter (90) + : +- ^ ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ ShuffledHashJoinExecTransformer Inner (85) + : :- ^ InputIteratorTransformer (80) + : : +- ^ InputAdapter (79) + : : +- ^ ShuffleQueryStage (78), Statistics(X) + : : +- ColumnarExchange (77) + : : +- ^ ProjectExecTransformer (75) + : : +- ^ NoopFilter (74) + : : +- ^ Scan parquet (73) + : +- ^ InputIteratorTransformer (84) + : +- ^ InputAdapter (83) + : +- ^ ShuffleQueryStage (82), Statistics(X) + : +- ReusedExchange (81) + +- ^ InputIteratorTransformer (95) + +- ^ InputAdapter (94) + +- ^ ShuffleQueryStage (93), Statistics(X) + +- ReusedExchange (92) +- == Initial Plan == - HashAggregate (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (101) - : : +- Filter (100) - : : +- Scan parquet (99) - : +- Exchange (104) - : +- Filter (103) - : +- Scan parquet (102) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) - - -(70) Scan parquet + HashAggregate (119) + +- HashAggregate (118) + +- Project (117) + +- ShuffledHashJoin Inner BuildRight (116) + :- Exchange (111) + : +- Project (110) + : +- ShuffledHashJoin Inner BuildRight (109) + : :- Exchange (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (108) + : +- Filter (107) + : +- Scan parquet (106) + +- Exchange (115) + +- Project (114) + +- Filter (113) + +- Scan parquet (112) + + +(73) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(71) ProjectExecTransformer +(74) NoopFilter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(75) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(72) WholeStageCodegenTransformer (X) +(76) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(73) ColumnarExchange +(77) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(78) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(75) InputAdapter +(79) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) InputIteratorTransformer +(80) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(77) ReusedExchange [Reuses operator id: 11] +(81) ReusedExchange [Reuses operator id: 13] Output [2]: [s_suppkey#X, s_nationkey#X] -(78) ShuffleQueryStage +(82) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(79) InputAdapter +(83) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(80) InputIteratorTransformer +(84) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(81) ShuffledHashJoinExecTransformer +(85) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(82) ProjectExecTransformer +(86) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(83) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(84) ColumnarExchange +(88) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(86) InputAdapter +(90) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(87) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(88) ReusedExchange [Reuses operator id: 25] +(92) ReusedExchange [Reuses operator id: 28] Output [1]: [n_nationkey#X] -(89) ShuffleQueryStage +(93) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(90) InputAdapter +(94) InputAdapter Input [1]: [n_nationkey#X] -(91) InputIteratorTransformer +(95) InputIteratorTransformer Input [1]: [n_nationkey#X] -(92) ShuffledHashJoinExecTransformer +(96) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(94) RegularHashAggregateExecTransformer +(98) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(95) RegularHashAggregateExecTransformer +(99) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(96) ProjectExecTransformer +(100) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(97) WholeStageCodegenTransformer (X) +(101) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(98) VeloxColumnarToRowExec +(102) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) Scan parquet +(103) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(100) Filter +(104) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(101) Exchange +(105) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) Scan parquet +(106) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(103) Filter +(107) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(104) Exchange +(108) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(106) Project +(110) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(107) Exchange +(111) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(112) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(113) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(110) Project +(114) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(115) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(113) Project +(117) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(118) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(115) HashAggregate +(119) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(116) AdaptiveSparkPlan +(120) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt index d77bf02f980a..595d52af5256 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt @@ -1,48 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (47) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (46) - +- Exchange (45) - +- HashAggregate (44) - +- Exchange (43) - +- HashAggregate (42) - +- Project (41) - +- ShuffledHashJoin Inner BuildLeft (40) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (39) - +- Project (38) - +- Filter (37) - +- Scan parquet (36) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin Inner BuildLeft (42) + :- Exchange (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Exchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -52,202 +54,210 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(26) ColumnarExchange +(28) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(28) InputAdapter +(30) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(30) SortExecTransformer +(32) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(32) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(33) Scan parquet +(35) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(34) Filter +(36) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(35) Exchange +(37) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(38) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(37) Filter +(39) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(38) Project +(40) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(39) Exchange +(41) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(41) Project +(43) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(42) HashAggregate +(44) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(43) Exchange +(45) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) HashAggregate +(46) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(45) Exchange +(47) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(48) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(47) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt index 796c40d94d57..10a7818f6c4d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (51) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (15) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5), Statistics(X) : +- ColumnarExchange (4) : +- ^ ProjectExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (50) - +- Exchange (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin LeftOuter BuildRight (42) - :- Exchange (37) - : +- Scan parquet (36) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- HashAggregate (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftOuter BuildRight (43) + :- Exchange (38) + : +- Scan parquet (37) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -84,200 +85,204 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) ProjectExecTransformer +(9) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] + +(10) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(12) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(13) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(14) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(15) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(16) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(17) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(19) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(29) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(36) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Exchange +(38) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) HashAggregate +(46) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(46) HashAggregate +(47) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(47) Exchange +(48) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(49) Exchange +(50) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Sort +(51) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(51) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt index 180411f0b335..4a4159446f40 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (32) - +- HashAggregate (31) - +- Project (30) - +- ShuffledHashJoin Inner BuildRight (29) - :- Exchange (25) - : +- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- Exchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- HashAggregate (33) + +- Project (32) + +- ShuffledHashJoin Inner BuildRight (31) + :- Exchange (27) + : +- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- Exchange (30) + +- Filter (29) + +- Scan parquet (28) (1) Scan parquet @@ -40,150 +42,158 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) Scan parquet +(9) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(11) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(23) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(24) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(25) Exchange +(27) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Scan parquet +(28) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(29) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) Exchange +(30) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) ShuffledHashJoin +(31) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(30) Project +(32) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(33) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(33) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt index 84df2af7dd53..90558947bb35 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (40) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (25) - +- AQEShuffleRead (24) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner (19) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FlushableHashAggregateExecTransformer (10) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ ShuffledHashJoinExecTransformer Inner (21) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (39) - +- Exchange (38) - +- Project (37) - +- ShuffledHashJoin Inner BuildLeft (36) - :- Exchange (28) - : +- Filter (27) - : +- Scan parquet (26) - +- Filter (35) - +- HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- Filter (30) - +- Scan parquet (29) + Sort (41) + +- Exchange (40) + +- Project (39) + +- ShuffledHashJoin Inner BuildLeft (38) + :- Exchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Filter (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -47,330 +49,343 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(11) ProjectExecTransformer +(13) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(19) ShuffledHashJoinExecTransformer +(21) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(24) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(25) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(26) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(27) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(28) Exchange +(30) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) Scan parquet +(31) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(30) Filter +(32) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(31) Project +(33) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(32) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(33) Exchange +(35) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(35) Filter +(37) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(36) ShuffledHashJoin +(38) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(37) Project +(39) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(38) Exchange +(40) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Sort +(41) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(40) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 18 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (64) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ ProjectExecTransformer (51) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ Scan parquet (41) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ^ InputAdapter (51) + +- ^ ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ NoopFilter (44) + +- ^ Scan parquet (43) +- == Initial Plan == - HashAggregate (63) - +- HashAggregate (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- Filter (57) - +- Scan parquet (56) + HashAggregate (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(41) Scan parquet +(43) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(42) ProjectExecTransformer +(44) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(45) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(43) FlushableHashAggregateExecTransformer +(46) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(49) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(51) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(51) ProjectExecTransformer +(54) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(52) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(53) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(54) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(55) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(56) Scan parquet +(59) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(57) Filter +(60) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(58) Project +(61) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(59) HashAggregate +(62) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(60) Exchange +(63) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(63) HashAggregate +(66) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt index 456656af6264..98c7da22a549 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt @@ -1,62 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (62) +AdaptiveSparkPlan (64) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ SortExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ ShuffleQueryStage (35), Statistics(X) - +- ColumnarExchange (34) - +- ^ RegularHashAggregateExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FlushableHashAggregateExecTransformer (25) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (42) + +- ^ SortExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ^ InputAdapter (38) + +- ^ ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (61) - +- Exchange (60) - +- HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- ShuffledHashJoin Inner BuildRight (52) - :- Exchange (48) - : +- BroadcastHashJoin LeftAnti BuildRight (47) - : :- Filter (42) - : : +- Scan parquet (41) - : +- BroadcastExchange (46) - : +- Project (45) - : +- Filter (44) - : +- Scan parquet (43) - +- Exchange (51) - +- Filter (50) - +- Scan parquet (49) + Sort (63) + +- Exchange (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- HashAggregate (58) + +- Exchange (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (50) + : +- BroadcastHashJoin LeftAnti BuildRight (49) + : :- Filter (44) + : : +- Scan parquet (43) + : +- BroadcastExchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Exchange (53) + +- Filter (52) + +- Scan parquet (51) (1) Scan parquet @@ -66,277 +68,285 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X, ps_suppkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Input [2]: [ps_partkey#X, ps_suppkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) FlushableHashAggregateExecTransformer +(27) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(28) ColumnarExchange +(30) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(30) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(32) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(33) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(34) ColumnarExchange +(36) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(37) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(36) InputAdapter +(38) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(37) InputIteratorTransformer +(39) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(38) SortExecTransformer +(40) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(39) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(40) VeloxColumnarToRowExec +(42) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(41) Scan parquet +(43) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(43) Scan parquet +(45) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(44) Filter +(46) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(45) Project +(47) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(46) BroadcastExchange +(48) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(47) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(48) Exchange +(50) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Scan parquet +(51) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(50) Filter +(52) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(51) Exchange +(53) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(53) Project +(55) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(54) HashAggregate +(56) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(55) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(58) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) HashAggregate +(59) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(58) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(60) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(63) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(62) AdaptiveSparkPlan +(64) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt index 276e3e66628e..97c56c7dae8d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ ProjectExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ ShuffledHashJoinExecTransformer Inner (28) - :- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12), Statistics(X) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ FilterExecTransformer (27) - +- ^ ProjectExecTransformer (26) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ Scan parquet (17) + VeloxColumnarToRowExec (37) + +- ^ ProjectExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ RegularHashAggregateExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ ShuffledHashJoinExecTransformer Inner (31) + :- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ FilterExecTransformer (30) + +- ^ ProjectExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ NoopFilter (20) + +- ^ Scan parquet (19) +- == Initial Plan == - HashAggregate (53) - +- HashAggregate (52) - +- Project (51) - +- ShuffledHashJoin Inner BuildRight (50) - :- Project (43) - : +- ShuffledHashJoin Inner BuildRight (42) - : :- Exchange (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Exchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Filter (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- Filter (45) - +- Scan parquet (44) + HashAggregate (56) + +- HashAggregate (55) + +- Project (54) + +- ShuffledHashJoin Inner BuildRight (53) + :- Project (46) + : +- ShuffledHashJoin Inner BuildRight (45) + : :- Exchange (40) + : : +- Filter (39) + : : +- Scan parquet (38) + : +- Exchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- Filter (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -60,254 +63,266 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X] + +(3) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [p_partkey#X, p_brand#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_container#X] + +(11) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [1]: [p_partkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [p_partkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) NoopFilter +Input [2]: [l_partkey#X, l_quantity#X] +Arguments: [l_partkey#X, l_quantity#X] + +(21) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(20) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(21) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(27) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(28) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(30) RegularHashAggregateExecTransformer +(33) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(37) Exchange +(40) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(40) Project +(43) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(41) Exchange +(44) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(43) Project +(46) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(44) Scan parquet +(47) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(45) Filter +(48) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(46) HashAggregate +(49) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(47) Exchange +(50) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(51) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(49) Filter +(52) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(50) ShuffledHashJoin +(53) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(51) Project +(54) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) HashAggregate +(55) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(53) HashAggregate +(56) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(54) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt index 2bda86546324..f0537d1fcf07 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt @@ -1,93 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (97) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- TakeOrderedAndProjectExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ ProjectExecTransformer (56) - +- ^ ShuffledHashJoinExecTransformer Inner (55) - :- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ ShuffleQueryStage (37), Statistics(X) - : +- ColumnarExchange (36) - : +- ^ ProjectExecTransformer (34) - : +- ^ ShuffledHashJoinExecTransformer Inner (33) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ ShuffleQueryStage (30), Statistics(X) - : +- ColumnarExchange (29) - : +- ^ ProjectExecTransformer (27) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (26) - : :- ^ InputIteratorTransformer (14) - : : +- ^ InputAdapter (13) - : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : +- ColumnarExchange (11) - : : +- ^ ProjectExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ ProjectExecTransformer (25) - : +- ^ FilterExecTransformer (24) - : +- ^ RegularHashAggregateExecTransformer (23) - : +- ^ InputIteratorTransformer (22) - : +- ^ InputAdapter (21) - : +- ^ ShuffleQueryStage (20), Statistics(X) - : +- ColumnarExchange (19) - : +- ^ ProjectExecTransformer (17) - : +- ^ FlushableHashAggregateExecTransformer (16) - : +- ^ Scan parquet (15) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (54) - :- ^ InputIteratorTransformer (46) - : +- ^ InputAdapter (45) - : +- ^ ShuffleQueryStage (44), Statistics(X) - : +- ColumnarExchange (43) - : +- ^ ProjectExecTransformer (41) - : +- ^ Scan parquet (40) - +- ^ ProjectExecTransformer (53) - +- ^ FilterExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48), Statistics(X) - +- ReusedExchange (47) + VeloxColumnarToRowExec (64) + +- TakeOrderedAndProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner (58) + :- ^ InputIteratorTransformer (41) + : +- ^ InputAdapter (40) + : +- ^ ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ^ InputAdapter (33) + : +- ^ ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : :- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ^ InputAdapter (23) + : +- ^ ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + :- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (93) - +- HashAggregate (92) - +- HashAggregate (91) - +- Project (90) - +- ShuffledHashJoin Inner BuildRight (89) - :- Exchange (78) - : +- Project (77) - : +- ShuffledHashJoin Inner BuildLeft (76) - : :- Exchange (64) - : : +- Filter (63) - : : +- Scan parquet (62) - : +- Exchange (75) - : +- ShuffledHashJoin LeftSemi BuildRight (74) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Project (73) - : +- Filter (72) - : +- HashAggregate (71) - : +- Exchange (70) - : +- HashAggregate (69) - : +- Scan parquet (68) - +- ShuffledHashJoin LeftSemi BuildRight (88) - :- Exchange (81) - : +- Filter (80) - : +- Scan parquet (79) - +- Project (87) - +- Filter (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Scan parquet (82) + TakeOrderedAndProject (96) + +- HashAggregate (95) + +- HashAggregate (94) + +- Project (93) + +- ShuffledHashJoin Inner BuildRight (92) + :- Exchange (81) + : +- Project (80) + : +- ShuffledHashJoin Inner BuildLeft (79) + : :- Exchange (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Exchange (78) + : +- ShuffledHashJoin LeftSemi BuildRight (77) + : :- Exchange (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- Project (76) + : +- Filter (75) + : +- HashAggregate (74) + : +- Exchange (73) + : +- HashAggregate (72) + : +- Scan parquet (71) + +- ShuffledHashJoin LeftSemi BuildRight (91) + :- Exchange (84) + : +- Filter (83) + : +- Scan parquet (82) + +- Project (90) + +- Filter (89) + +- HashAggregate (88) + +- Exchange (87) + +- HashAggregate (86) + +- Scan parquet (85) (1) Scan parquet @@ -97,428 +100,440 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X, c_name#X] + +(3) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X] Input [2]: [c_custkey#X, c_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(16) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(19) ColumnarExchange +(21) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(20) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(21) InputAdapter +(23) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(22) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(23) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(24) FilterExecTransformer +(26) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(25) ProjectExecTransformer +(27) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(26) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(29) ColumnarExchange +(31) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(32) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(31) InputAdapter +(33) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(32) InputIteratorTransformer +(34) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(33) ShuffledHashJoinExecTransformer +(35) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(34) ProjectExecTransformer +(36) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(36) ColumnarExchange +(38) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(39) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(38) InputAdapter +(40) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(39) InputIteratorTransformer +(41) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(40) Scan parquet +(42) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(41) ProjectExecTransformer +(43) NoopFilter +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X, l_quantity#X] + +(44) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(42) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(43) ColumnarExchange +(46) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(47) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(45) InputAdapter +(48) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(46) InputIteratorTransformer +(49) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(47) ReusedExchange [Reuses operator id: 19] +(50) ReusedExchange [Reuses operator id: 21] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(48) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(49) InputAdapter +(52) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(50) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(52) FilterExecTransformer +(55) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(53) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(54) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(55) ShuffledHashJoinExecTransformer +(58) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(56) ProjectExecTransformer +(59) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(57) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(58) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(59) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(60) TakeOrderedAndProjectExecTransformer +(63) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(61) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(62) Scan parquet +(65) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(63) Filter +(66) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(64) Exchange +(67) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(68) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(66) Filter +(69) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(67) Exchange +(70) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(69) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(70) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(72) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(73) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(74) ShuffledHashJoin +(77) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(75) Exchange +(78) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) ShuffledHashJoin +(79) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(77) Project +(80) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(78) Exchange +(81) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Scan parquet +(82) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(80) Filter +(83) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(81) Exchange +(84) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(85) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(83) HashAggregate +(86) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(84) Exchange +(87) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(88) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(86) Filter +(89) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(87) Project +(90) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(88) ShuffledHashJoin +(91) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(89) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(90) Project +(93) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(91) HashAggregate +(94) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(92) HashAggregate +(95) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(93) TakeOrderedAndProject +(96) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(94) AdaptiveSparkPlan +(97) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt index cff613274b00..7d03f6eca963 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (32) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (31) - +- HashAggregate (30) - +- Project (29) - +- ShuffledHashJoin Inner BuildRight (28) - :- Exchange (24) - : +- Project (23) - : +- Filter (22) - : +- Scan parquet (21) - +- Exchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- HashAggregate (32) + +- Project (31) + +- ShuffledHashJoin Inner BuildRight (30) + :- Exchange (26) + : +- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- Exchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -39,146 +41,154 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(3) ProjectExecTransformer Output [5]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(20) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(21) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(22) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(23) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(24) Exchange +(26) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) Scan parquet +(27) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(28) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) Exchange +(29) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) ShuffledHashJoin +(30) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(31) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(32) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(32) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt index 541b0f0aebab..4e67d57112e7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt @@ -1,114 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (123) +- == Final Plan == - VeloxColumnarToRowExec (78) - +- AQEShuffleRead (77) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ ShuffledHashJoinExecTransformer Inner (72) - :- ^ InputIteratorTransformer (64) - : +- ^ InputAdapter (63) - : +- ^ ShuffleQueryStage (62), Statistics(X) - : +- ColumnarExchange (61) - : +- ^ ProjectExecTransformer (59) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (58) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (57) - : +- ^ InputAdapter (56) - : +- ^ ShuffleQueryStage (55), Statistics(X) - : +- ColumnarExchange (54) - : +- ^ ProjectExecTransformer (52) - : +- ^ ShuffledHashJoinExecTransformer Inner (51) - : :- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : +- ColumnarExchange (25) - : : +- ^ ProjectExecTransformer (23) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (22) - : : :- ^ InputIteratorTransformer (14) - : : : +- ^ InputAdapter (13) - : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : +- ColumnarExchange (11) - : : : +- ^ ProjectExecTransformer (9) - : : : +- ^ Scan parquet (8) - : : +- ^ InputIteratorTransformer (21) - : : +- ^ InputAdapter (20) - : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : +- ColumnarExchange (18) - : : +- ^ ProjectExecTransformer (16) - : : +- ^ Scan parquet (15) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ ShuffleQueryStage (48), Statistics(X) - : +- ColumnarExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ FilterExecTransformer (44) - : +- ^ ProjectExecTransformer (43) - : +- ^ RegularHashAggregateExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (40) - : :- ^ InputIteratorTransformer (35) - : : +- ^ InputAdapter (34) - : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : +- ColumnarExchange (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ Scan parquet (29) - : +- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ ShuffleQueryStage (37), Statistics(X) - : +- ReusedExchange (36) - +- ^ InputIteratorTransformer (71) - +- ^ InputAdapter (70) - +- ^ ShuffleQueryStage (69), Statistics(X) - +- ColumnarExchange (68) - +- ^ ProjectExecTransformer (66) - +- ^ Scan parquet (65) + VeloxColumnarToRowExec (83) + +- AQEShuffleRead (82) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (31) + : : +- ^ InputAdapter (30) + : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ NoopFilter (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ^ InputAdapter (23) + : : +- ^ ShuffleQueryStage (22), Statistics(X) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ NoopFilter (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) + : :- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ NoopFilter (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ^ InputAdapter (42) + : +- ^ ShuffleQueryStage (41), Statistics(X) + : +- ReusedExchange (40) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (117) - +- Exchange (116) - +- Project (115) - +- ShuffledHashJoin Inner BuildRight (114) - :- Exchange (109) - : +- Project (108) - : +- ShuffledHashJoin LeftSemi BuildRight (107) - : :- Exchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- Exchange (106) - : +- Project (105) - : +- ShuffledHashJoin Inner BuildLeft (104) - : :- Exchange (90) - : : +- ShuffledHashJoin LeftSemi BuildRight (89) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (88) - : : +- Project (87) - : : +- Filter (86) - : : +- Scan parquet (85) - : +- Exchange (103) - : +- Filter (102) - : +- HashAggregate (101) - : +- HashAggregate (100) - : +- ShuffledHashJoin LeftSemi BuildRight (99) - : :- Exchange (94) - : : +- Project (93) - : : +- Filter (92) - : : +- Scan parquet (91) - : +- Exchange (98) - : +- Project (97) - : +- Filter (96) - : +- Scan parquet (95) - +- Exchange (113) - +- Project (112) - +- Filter (111) - +- Scan parquet (110) + Sort (122) + +- Exchange (121) + +- Project (120) + +- ShuffledHashJoin Inner BuildRight (119) + :- Exchange (114) + : +- Project (113) + : +- ShuffledHashJoin LeftSemi BuildRight (112) + : :- Exchange (86) + : : +- Filter (85) + : : +- Scan parquet (84) + : +- Exchange (111) + : +- Project (110) + : +- ShuffledHashJoin Inner BuildLeft (109) + : :- Exchange (95) + : : +- ShuffledHashJoin LeftSemi BuildRight (94) + : : :- Exchange (89) + : : : +- Filter (88) + : : : +- Scan parquet (87) + : : +- Exchange (93) + : : +- Project (92) + : : +- Filter (91) + : : +- Scan parquet (90) + : +- Exchange (108) + : +- Filter (107) + : +- HashAggregate (106) + : +- HashAggregate (105) + : +- ShuffledHashJoin LeftSemi BuildRight (104) + : :- Exchange (99) + : : +- Project (98) + : : +- Filter (97) + : : +- Scan parquet (96) + : +- Exchange (103) + : +- Project (102) + : +- Filter (101) + : +- Scan parquet (100) + +- Exchange (118) + +- Project (117) + +- Filter (116) + +- Scan parquet (115) (1) Scan parquet @@ -118,510 +123,530 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(11) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(16) ProjectExecTransformer +(18) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(19) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(17) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(18) ColumnarExchange +(21) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(22) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(20) InputAdapter +(23) InputAdapter Input [1]: [p_partkey#X] -(21) InputIteratorTransformer +(24) InputIteratorTransformer Input [1]: [p_partkey#X] -(22) ShuffledHashJoinExecTransformer +(25) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(23) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(29) Scan parquet +(32) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(30) ProjectExecTransformer +(33) NoopFilter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(34) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(31) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(32) ColumnarExchange +(36) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(37) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(34) InputAdapter +(38) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(35) InputIteratorTransformer +(39) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(36) ReusedExchange [Reuses operator id: 18] +(40) ReusedExchange [Reuses operator id: 21] Output [1]: [p_partkey#X] -(37) ShuffleQueryStage +(41) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(38) InputAdapter +(42) InputAdapter Input [1]: [p_partkey#X] -(39) InputIteratorTransformer +(43) InputIteratorTransformer Input [1]: [p_partkey#X] -(40) ShuffledHashJoinExecTransformer +(44) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(42) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) ProjectExecTransformer +(47) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(44) FilterExecTransformer +(48) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(47) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(49) InputAdapter +(53) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(51) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(52) ProjectExecTransformer +(56) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(53) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(54) ColumnarExchange +(58) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(56) InputAdapter +(60) InputAdapter Input [1]: [ps_suppkey#X] -(57) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(58) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(59) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(60) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(61) ColumnarExchange +(65) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(62) ShuffleQueryStage +(66) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(67) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(64) InputIteratorTransformer +(68) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(65) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(66) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(71) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(67) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(68) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(69) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(70) InputAdapter +(75) InputAdapter Input [1]: [n_nationkey#X] -(71) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [n_nationkey#X] -(72) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(73) ProjectExecTransformer +(78) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(74) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(75) ColumnarExchange +(80) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(77) AQEShuffleRead +(82) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(78) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(79) Scan parquet +(84) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(85) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(81) Exchange +(86) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(84) Exchange +(89) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(86) Filter +(91) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(87) Project +(92) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(88) Exchange +(93) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) ShuffledHashJoin +(94) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(90) Exchange +(95) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(96) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(92) Filter +(97) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(93) Project +(98) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(94) Exchange +(99) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(96) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(97) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(100) HashAggregate +(105) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(101) HashAggregate +(106) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(102) Filter +(107) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(103) Exchange +(108) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(105) Project +(110) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(106) Exchange +(111) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(112) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(108) Project +(113) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(109) Exchange +(114) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(111) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(112) Project +(117) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(113) Exchange +(118) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(115) Project +(120) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(116) Exchange +(121) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Sort +(122) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(118) AdaptiveSparkPlan +(123) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt index 58be95c2bc00..658a499a30ce 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt @@ -1,108 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (113) +AdaptiveSparkPlan (118) +- == Final Plan == - VeloxColumnarToRowExec (76) - +- ^ RegularHashAggregateExecTransformer (74) - +- ^ InputIteratorTransformer (73) - +- ^ InputAdapter (72) - +- ^ ShuffleQueryStage (71), Statistics(X) - +- ColumnarExchange (70) - +- ^ ProjectExecTransformer (68) - +- ^ FlushableHashAggregateExecTransformer (67) - +- ^ ProjectExecTransformer (66) - +- ^ ShuffledHashJoinExecTransformer Inner (65) - :- ^ InputIteratorTransformer (57) - : +- ^ InputAdapter (56) - : +- ^ ShuffleQueryStage (55), Statistics(X) - : +- ColumnarExchange (54) - : +- ^ ProjectExecTransformer (52) - : +- ^ ShuffledHashJoinExecTransformer Inner (51) - : :- ^ InputIteratorTransformer (43) - : : +- ^ InputAdapter (42) - : : +- ^ ShuffleQueryStage (41), Statistics(X) - : : +- ColumnarExchange (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer Inner (37) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : +- ColumnarExchange (4) - : : : +- ^ ProjectExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (36) - : : +- ^ InputAdapter (35) - : : +- ^ ShuffleQueryStage (34), Statistics(X) - : : +- ColumnarExchange (33) - : : +- ^ ProjectExecTransformer (31) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (30) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (22) - : : : :- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (21) - : : : +- ^ InputAdapter (20) - : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : +- ColumnarExchange (18) - : : : +- ^ ProjectExecTransformer (16) - : : : +- ^ Scan parquet (15) - : : +- ^ InputIteratorTransformer (29) - : : +- ^ InputAdapter (28) - : : +- ^ ShuffleQueryStage (27), Statistics(X) - : : +- ColumnarExchange (26) - : : +- ^ ProjectExecTransformer (24) - : : +- ^ Scan parquet (23) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ ShuffleQueryStage (48), Statistics(X) - : +- ColumnarExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ Scan parquet (44) - +- ^ InputIteratorTransformer (64) - +- ^ InputAdapter (63) - +- ^ ShuffleQueryStage (62), Statistics(X) - +- ColumnarExchange (61) - +- ^ ProjectExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (81) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ^ InputAdapter (77) + +- ^ ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner (70) + :- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ^ InputAdapter (22) + : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ^ InputAdapter (31) + : : +- ^ ShuffleQueryStage (30), Statistics(X) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ NoopFilter (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ^ InputAdapter (68) + +- ^ ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ NoopFilter (63) + +- ^ Scan parquet (62) +- == Initial Plan == - TakeOrderedAndProject (112) - +- HashAggregate (111) - +- Exchange (110) - +- HashAggregate (109) - +- Project (108) - +- ShuffledHashJoin Inner BuildRight (107) - :- Exchange (102) - : +- Project (101) - : +- ShuffledHashJoin Inner BuildRight (100) - : :- Exchange (95) - : : +- Project (94) - : : +- ShuffledHashJoin Inner BuildLeft (93) - : : :- Exchange (79) - : : : +- Filter (78) - : : : +- Scan parquet (77) - : : +- Exchange (92) - : : +- ShuffledHashJoin LeftAnti BuildRight (91) - : : :- ShuffledHashJoin LeftSemi BuildRight (86) - : : : :- Exchange (83) - : : : : +- Project (82) - : : : : +- Filter (81) - : : : : +- Scan parquet (80) - : : : +- Exchange (85) - : : : +- Scan parquet (84) - : : +- Exchange (90) - : : +- Project (89) - : : +- Filter (88) - : : +- Scan parquet (87) - : +- Exchange (99) - : +- Project (98) - : +- Filter (97) - : +- Scan parquet (96) - +- Exchange (106) - +- Project (105) - +- Filter (104) - +- Scan parquet (103) + TakeOrderedAndProject (117) + +- HashAggregate (116) + +- Exchange (115) + +- HashAggregate (114) + +- Project (113) + +- ShuffledHashJoin Inner BuildRight (112) + :- Exchange (107) + : +- Project (106) + : +- ShuffledHashJoin Inner BuildRight (105) + : :- Exchange (100) + : : +- Project (99) + : : +- ShuffledHashJoin Inner BuildLeft (98) + : : :- Exchange (84) + : : : +- Filter (83) + : : : +- Scan parquet (82) + : : +- Exchange (97) + : : +- ShuffledHashJoin LeftAnti BuildRight (96) + : : :- ShuffledHashJoin LeftSemi BuildRight (91) + : : : :- Exchange (88) + : : : : +- Project (87) + : : : : +- Filter (86) + : : : : +- Scan parquet (85) + : : : +- Exchange (90) + : : : +- Scan parquet (89) + : : +- Exchange (95) + : : +- Project (94) + : : +- Filter (93) + : : +- Scan parquet (92) + : +- Exchange (104) + : +- Project (103) + : +- Filter (102) + : +- Scan parquet (101) + +- Exchange (111) + +- Project (110) + +- Filter (109) + +- Scan parquet (108) (1) Scan parquet @@ -112,492 +117,512 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [4]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(22) ShuffledHashJoinExecTransformer +(24) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(23) Scan parquet +(25) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(24) ProjectExecTransformer +(26) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(27) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(25) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(26) ColumnarExchange +(29) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(28) InputAdapter +(31) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(29) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(30) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(31) ProjectExecTransformer +(34) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(33) ColumnarExchange +(36) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(37) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(35) InputAdapter +(38) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(36) InputIteratorTransformer +(39) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(37) ShuffledHashJoinExecTransformer +(40) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(38) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(40) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(41) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(42) InputAdapter +(45) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(43) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(44) Scan parquet +(47) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(45) ProjectExecTransformer +(48) NoopFilter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: [o_orderkey#X, o_orderstatus#X] + +(49) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(46) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(47) ColumnarExchange +(51) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(52) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(49) InputAdapter +(53) InputAdapter Input [1]: [o_orderkey#X] -(50) InputIteratorTransformer +(54) InputIteratorTransformer Input [1]: [o_orderkey#X] -(51) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(52) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(53) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(54) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(56) InputAdapter +(60) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(57) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(58) Scan parquet +(62) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(59) ProjectExecTransformer +(63) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(64) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(60) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(61) ColumnarExchange +(66) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(62) ShuffleQueryStage +(67) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(63) InputAdapter +(68) InputAdapter Input [1]: [n_nationkey#X] -(64) InputIteratorTransformer +(69) InputIteratorTransformer Input [1]: [n_nationkey#X] -(65) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(66) ProjectExecTransformer +(71) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(67) FlushableHashAggregateExecTransformer +(72) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(68) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(69) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(70) ColumnarExchange +(75) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(71) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(72) InputAdapter +(77) InputAdapter Input [2]: [s_name#X, count#X] -(73) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(74) RegularHashAggregateExecTransformer +(79) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(75) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(76) VeloxColumnarToRowExec +(81) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(77) Scan parquet +(82) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(78) Filter +(83) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(79) Exchange +(84) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) Scan parquet +(85) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(81) Filter +(86) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(82) Project +(87) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(83) Exchange +(88) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Scan parquet +(89) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(85) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) ShuffledHashJoin +(91) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(87) Scan parquet +(92) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(88) Filter +(93) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(89) Project +(94) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(90) Exchange +(95) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(94) Project +(99) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(95) Exchange +(100) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(101) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(97) Filter +(102) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(98) Project +(103) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(99) Exchange +(104) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(101) Project +(106) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(102) Exchange +(107) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(108) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(104) Filter +(109) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(105) Project +(110) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(106) Exchange +(111) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(112) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(108) Project +(113) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(109) HashAggregate +(114) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(110) Exchange +(115) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) HashAggregate +(116) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(112) TakeOrderedAndProject +(117) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(113) AdaptiveSparkPlan +(118) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt index 5a5cb9c96758..e94b7b01715f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt @@ -1,46 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (33) + +- ^ SortExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ^ InputAdapter (29) + +- ^ ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- ShuffledHashJoin LeftAnti BuildRight (38) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (37) - +- Scan parquet (36) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- ShuffledHashJoin LeftAnti BuildRight (39) + :- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Exchange (38) + +- Scan parquet (37) (1) Scan parquet @@ -50,302 +51,311 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X, c_phone#X, c_acctbal#X] + +(3) ProjectExecTransformer Output [4]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) Scan parquet +(9) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(9) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(12) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(13) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(13) InputAdapter +(14) InputAdapter Input [1]: [o_custkey#X] -(14) InputIteratorTransformer +(15) InputIteratorTransformer Input [1]: [o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(16) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(17) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(20) ColumnarExchange +(21) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) ColumnarExchange +(27) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) SortExecTransformer +(31) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(32) VeloxColumnarToRowExec +(33) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(35) Exchange +(36) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(37) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Exchange +(38) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) ShuffledHashJoin +(39) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(39) Project +(40) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(40) HashAggregate +(41) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(41) Exchange +(42) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(43) Exchange +(44) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (65) +- == Final Plan == - VeloxColumnarToRowExec (56) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ColumnarExchange (50) - +- ^ FlushableHashAggregateExecTransformer (48) - +- ^ ProjectExecTransformer (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ NoopFilter (48) + +- ^ Scan parquet (47) +- == Initial Plan == - HashAggregate (62) - +- Exchange (61) - +- HashAggregate (60) - +- Project (59) - +- Filter (58) - +- Scan parquet (57) + HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(46) Scan parquet +(47) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(47) ProjectExecTransformer +(48) NoopFilter +Input [2]: [c_phone#X, c_acctbal#X] +Arguments: [c_phone#X, c_acctbal#X] + +(49) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(48) FlushableHashAggregateExecTransformer +(50) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(49) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(50) ColumnarExchange +(52) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(52) InputAdapter +(54) InputAdapter Input [2]: [sum#X, count#X] -(53) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [sum#X, count#X] -(54) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(55) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(56) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(57) Scan parquet +(59) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(58) Filter +(60) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(59) Project +(61) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(60) HashAggregate +(62) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(61) Exchange +(63) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(63) AdaptiveSparkPlan +(65) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt index 25e019dc37ba..16615ac0598f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt @@ -1,57 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- TakeOrderedAndProjectExecTransformer (35) - +- ^ ProjectExecTransformer (33) - +- ^ RegularHashAggregateExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ ProjectExecTransformer (30) - +- ^ ShuffledHashJoinExecTransformer Inner (29) - :- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ ShuffleQueryStage (19), Statistics(X) - : +- ColumnarExchange (18) - : +- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12), Statistics(X) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ Scan parquet (22) + VeloxColumnarToRowExec (39) + +- TakeOrderedAndProjectExecTransformer (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (55) - +- HashAggregate (54) - +- HashAggregate (53) - +- Project (52) - +- ShuffledHashJoin Inner BuildRight (51) - :- Exchange (46) - : +- Project (45) - : +- ShuffledHashJoin Inner BuildLeft (44) - : :- Exchange (40) - : : +- Project (39) - : : +- Filter (38) - : : +- Scan parquet (37) - : +- Exchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Exchange (50) - +- Project (49) - +- Filter (48) - +- Scan parquet (47) + TakeOrderedAndProject (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (49) + : +- Project (48) + : +- ShuffledHashJoin Inner BuildLeft (47) + : :- Exchange (43) + : : +- Project (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Exchange (53) + +- Project (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -61,248 +64,260 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: [c_custkey#X, c_mktsegment#X] + +(3) ProjectExecTransformer Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(11) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(32) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(33) ProjectExecTransformer +(36) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(35) TakeOrderedAndProjectExecTransformer +(38) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(37) Scan parquet +(40) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(38) Filter +(41) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(39) Project +(42) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(40) Exchange +(43) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(44) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(42) Filter +(45) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(43) Exchange +(46) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(45) Project +(48) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(46) Exchange +(49) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Scan parquet +(50) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(48) Filter +(51) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(49) Project +(52) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(50) Exchange +(53) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(52) Project +(55) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(53) HashAggregate +(56) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(54) HashAggregate +(57) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(55) TakeOrderedAndProject +(58) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(56) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt index f57e74a85df5..1d3f8903f89f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt @@ -1,49 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (48) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (47) - +- Exchange (46) - +- HashAggregate (45) - +- Exchange (44) - +- HashAggregate (43) - +- Project (42) - +- ShuffledHashJoin LeftSemi BuildRight (41) - :- Exchange (36) - : +- Project (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftSemi BuildRight (43) + :- Exchange (38) + : +- Project (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -53,206 +55,214 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [1]: [l_orderkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [l_orderkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(26) ColumnarExchange +(28) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(28) InputAdapter +(30) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(29) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(30) SortExecTransformer +(32) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(32) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(33) Scan parquet +(35) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(34) Filter +(36) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(35) Project +(37) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(36) Exchange +(38) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(39) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(38) Filter +(40) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(39) Project +(41) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(40) Exchange +(42) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(42) Project +(44) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(43) HashAggregate +(45) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(44) Exchange +(46) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(46) Exchange +(48) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Sort +(49) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(48) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt index 9de96b917fad..12c4c6add240 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt @@ -1,121 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (134) +- == Final Plan == - VeloxColumnarToRowExec (88) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ^ InputAdapter (78) - +- ^ ShuffleQueryStage (77), Statistics(X) - +- ColumnarExchange (76) - +- ^ ProjectExecTransformer (74) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ ShuffledHashJoinExecTransformer Inner (71) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61), Statistics(X) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54), Statistics(X) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ ShuffleQueryStage (68), Statistics(X) - +- ColumnarExchange (67) - +- ^ ProjectExecTransformer (65) - +- ^ Scan parquet (64) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildRight (108) - : : :- Exchange (104) - : : : +- Project (103) - : : : +- ShuffledHashJoin Inner BuildRight (102) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- ShuffledHashJoin Inner BuildLeft (96) - : : : : :- Exchange (91) - : : : : : +- Filter (90) - : : : : : +- Scan parquet (89) - : : : : +- Exchange (95) - : : : : +- Project (94) - : : : : +- Filter (93) - : : : : +- Scan parquet (92) - : : : +- Exchange (101) - : : : +- Filter (100) - : : : +- Scan parquet (99) - : : +- Exchange (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Exchange (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (120) - +- Project (119) - +- Filter (118) - +- Scan parquet (117) + Sort (133) + +- Exchange (132) + +- HashAggregate (131) + +- Exchange (130) + +- HashAggregate (129) + +- Project (128) + +- ShuffledHashJoin Inner BuildRight (127) + :- Exchange (122) + : +- Project (121) + : +- ShuffledHashJoin Inner BuildRight (120) + : :- Exchange (116) + : : +- Project (115) + : : +- ShuffledHashJoin Inner BuildRight (114) + : : :- Exchange (110) + : : : +- Project (109) + : : : +- ShuffledHashJoin Inner BuildRight (108) + : : : :- Exchange (104) + : : : : +- Project (103) + : : : : +- ShuffledHashJoin Inner BuildLeft (102) + : : : : :- Exchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Exchange (101) + : : : : +- Project (100) + : : : : +- Filter (99) + : : : : +- Scan parquet (98) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Exchange (126) + +- Project (125) + +- Filter (124) + +- Scan parquet (123) (1) Scan parquet @@ -125,550 +131,574 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(26) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(41) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(50) Scan parquet +(54) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] + +(56) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(71) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [1]: [r_regionkey#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [r_regionkey#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(73) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(74) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(82) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(83) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(84) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(79) InputIteratorTransformer +(85) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(81) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(82) ColumnarExchange +(88) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(84) InputAdapter +(90) InputAdapter Input [2]: [n_name#X, revenue#X] -(85) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(86) SortExecTransformer +(92) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(89) Scan parquet +(95) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(90) Filter +(96) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(91) Exchange +(97) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Scan parquet +(98) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(93) Filter +(99) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(94) Project +(100) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(95) Exchange +(101) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(97) Project +(103) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(98) Exchange +(104) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(105) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(100) Filter +(106) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(101) Exchange +(107) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(103) Project +(109) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(110) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(111) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(112) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(113) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(109) Project +(115) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(116) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(117) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(113) Exchange +(119) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(115) Project +(121) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(116) Exchange +(122) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(123) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(118) Filter +(124) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(119) Project +(125) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(120) Exchange +(126) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(127) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(122) Project +(128) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(123) HashAggregate +(129) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(124) Exchange +(130) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(131) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(126) Exchange +(132) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(133) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(128) AdaptiveSparkPlan +(134) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt index 733adf0d0b4a..9333e2a8ad59 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (18) +AdaptiveSparkPlan (19) +- == Final Plan == - VeloxColumnarToRowExec (11) - +- ^ RegularHashAggregateExecTransformer (9) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ ShuffleQueryStage (6), Statistics(X) - +- ColumnarExchange (5) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (17) - +- Exchange (16) - +- HashAggregate (15) - +- Project (14) - +- Filter (13) - +- Scan parquet (12) + HashAggregate (18) + +- Exchange (17) + +- HashAggregate (16) + +- Project (15) + +- Filter (14) + +- Scan parquet (13) (1) Scan parquet @@ -26,82 +27,86 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(5) ColumnarExchange +(6) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [sum#X, isEmpty#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(9) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(11) VeloxColumnarToRowExec +(12) VeloxColumnarToRowExec Input [1]: [revenue#X] -(12) Scan parquet +(13) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(13) Filter +(14) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(14) Project +(15) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) HashAggregate +(16) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(16) Exchange +(17) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(17) HashAggregate +(18) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(18) AdaptiveSparkPlan +(19) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt index 50c9b4a231f1..03305d572de7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt @@ -1,117 +1,122 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (85) - +- ^ SortExecTransformer (83) - +- ^ InputIteratorTransformer (82) - +- ^ InputAdapter (81) - +- ^ ShuffleQueryStage (80), Statistics(X) - +- ColumnarExchange (79) - +- ^ RegularHashAggregateExecTransformer (77) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FlushableHashAggregateExecTransformer (70) - +- ^ ProjectExecTransformer (69) - +- ^ ShuffledHashJoinExecTransformer Inner (68) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61), Statistics(X) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54), Statistics(X) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (67) - +- ^ InputAdapter (66) - +- ^ ShuffleQueryStage (65), Statistics(X) - +- ReusedExchange (64) + VeloxColumnarToRowExec (90) + +- ^ SortExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ^ InputAdapter (86) + +- ^ ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ^ InputAdapter (80) + +- ^ ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner (73) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ^ InputAdapter (71) + +- ^ ShuffleQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- HashAggregate (120) - +- Exchange (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (112) - : +- Project (111) - : +- ShuffledHashJoin Inner BuildRight (110) - : :- Exchange (106) - : : +- Project (105) - : : +- ShuffledHashJoin Inner BuildRight (104) - : : :- Exchange (100) - : : : +- Project (99) - : : : +- ShuffledHashJoin Inner BuildRight (98) - : : : :- Exchange (94) - : : : : +- Project (93) - : : : : +- ShuffledHashJoin Inner BuildLeft (92) - : : : : :- Exchange (88) - : : : : : +- Filter (87) - : : : : : +- Scan parquet (86) - : : : : +- Exchange (91) - : : : : +- Filter (90) - : : : : +- Scan parquet (89) - : : : +- Exchange (97) - : : : +- Filter (96) - : : : +- Scan parquet (95) - : : +- Exchange (103) - : : +- Filter (102) - : : +- Scan parquet (101) - : +- Exchange (109) - : +- Filter (108) - : +- Scan parquet (107) - +- Exchange (115) - +- Filter (114) - +- Scan parquet (113) + Sort (127) + +- Exchange (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (111) + : : +- Project (110) + : : +- ShuffledHashJoin Inner BuildRight (109) + : : :- Exchange (105) + : : : +- Project (104) + : : : +- ShuffledHashJoin Inner BuildRight (103) + : : : :- Exchange (99) + : : : : +- Project (98) + : : : : +- ShuffledHashJoin Inner BuildLeft (97) + : : : : :- Exchange (93) + : : : : : +- Filter (92) + : : : : : +- Scan parquet (91) + : : : : +- Exchange (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (102) + : : : +- Filter (101) + : : : +- Scan parquet (100) + : : +- Exchange (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (114) + : +- Filter (113) + : +- Scan parquet (112) + +- Exchange (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -121,526 +126,546 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X, o_custkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(41) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(56) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(64) ReusedExchange [Reuses operator id: 53] +(69) ReusedExchange [Reuses operator id: 58] Output [2]: [n_nationkey#X, n_name#X] -(65) ShuffleQueryStage +(70) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(66) InputAdapter +(71) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(67) InputIteratorTransformer +(72) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(68) ShuffledHashJoinExecTransformer +(73) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(69) ProjectExecTransformer +(74) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(70) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(71) ProjectExecTransformer +(76) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(72) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(73) ColumnarExchange +(78) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(79) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(75) InputAdapter +(80) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) InputIteratorTransformer +(81) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(78) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(79) ColumnarExchange +(84) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(80) ShuffleQueryStage +(85) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(81) InputAdapter +(86) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(82) InputIteratorTransformer +(87) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(83) SortExecTransformer +(88) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(84) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(85) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(86) Scan parquet +(91) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(87) Filter +(92) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(88) Exchange +(93) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(94) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(95) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) Exchange +(96) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(93) Project +(98) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(94) Exchange +(99) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(100) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(96) Filter +(101) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(97) Exchange +(102) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(103) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(99) Project +(104) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(100) Exchange +(105) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(102) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(103) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(105) Project +(110) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(106) Exchange +(111) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) Scan parquet +(112) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(108) Filter +(113) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(109) Exchange +(114) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(111) Project +(116) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(112) Exchange +(117) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(114) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(115) Exchange +(120) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(117) Project +(122) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(118) HashAggregate +(123) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(119) Exchange +(124) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) HashAggregate +(125) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(121) Exchange +(126) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(127) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt index 5ed8ee8b6b4f..f0176bc5e011 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt @@ -1,158 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (169) +AdaptiveSparkPlan (177) +- == Final Plan == - VeloxColumnarToRowExec (117) - +- ^ SortExecTransformer (115) - +- ^ InputIteratorTransformer (114) - +- ^ InputAdapter (113) - +- ^ ShuffleQueryStage (112), Statistics(X) - +- ColumnarExchange (111) - +- ^ ProjectExecTransformer (109) - +- ^ RegularHashAggregateExecTransformer (108) - +- ^ InputIteratorTransformer (107) - +- ^ InputAdapter (106) - +- ^ ShuffleQueryStage (105), Statistics(X) - +- ColumnarExchange (104) - +- ^ ProjectExecTransformer (102) - +- ^ FlushableHashAggregateExecTransformer (101) - +- ^ ProjectExecTransformer (100) - +- ^ ShuffledHashJoinExecTransformer Inner (99) - :- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner (85) - : :- ^ InputIteratorTransformer (77) - : : +- ^ InputAdapter (76) - : : +- ^ ShuffleQueryStage (75), Statistics(X) - : : +- ColumnarExchange (74) - : : +- ^ ProjectExecTransformer (72) - : : +- ^ ShuffledHashJoinExecTransformer Inner (71) - : : :- ^ InputIteratorTransformer (63) - : : : +- ^ InputAdapter (62) - : : : +- ^ ShuffleQueryStage (61), Statistics(X) - : : : +- ColumnarExchange (60) - : : : +- ^ ProjectExecTransformer (58) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : : : :- ^ InputIteratorTransformer (49) - : : : : +- ^ InputAdapter (48) - : : : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : : : +- ColumnarExchange (46) - : : : : +- ^ ProjectExecTransformer (44) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : : : :- ^ InputIteratorTransformer (35) - : : : : : +- ^ InputAdapter (34) - : : : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : : : +- ColumnarExchange (32) - : : : : : +- ^ ProjectExecTransformer (30) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : : : :- ^ InputIteratorTransformer (21) - : : : : : : +- ^ InputAdapter (20) - : : : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : : : +- ColumnarExchange (18) - : : : : : : +- ^ ProjectExecTransformer (16) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : : : :- ^ InputIteratorTransformer (7) - : : : : : : : +- ^ InputAdapter (6) - : : : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : : : +- ColumnarExchange (4) - : : : : : : : +- ^ ProjectExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (14) - : : : : : : +- ^ InputAdapter (13) - : : : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : : : +- ColumnarExchange (11) - : : : : : : +- ^ ProjectExecTransformer (9) - : : : : : : +- ^ Scan parquet (8) - : : : : : +- ^ InputIteratorTransformer (28) - : : : : : +- ^ InputAdapter (27) - : : : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : : : +- ColumnarExchange (25) - : : : : : +- ^ ProjectExecTransformer (23) - : : : : : +- ^ Scan parquet (22) - : : : : +- ^ InputIteratorTransformer (42) - : : : : +- ^ InputAdapter (41) - : : : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : : : +- ColumnarExchange (39) - : : : : +- ^ ProjectExecTransformer (37) - : : : : +- ^ Scan parquet (36) - : : : +- ^ InputIteratorTransformer (56) - : : : +- ^ InputAdapter (55) - : : : +- ^ ShuffleQueryStage (54), Statistics(X) - : : : +- ColumnarExchange (53) - : : : +- ^ ProjectExecTransformer (51) - : : : +- ^ Scan parquet (50) - : : +- ^ InputIteratorTransformer (70) - : : +- ^ InputAdapter (69) - : : +- ^ ShuffleQueryStage (68), Statistics(X) - : : +- ColumnarExchange (67) - : : +- ^ ProjectExecTransformer (65) - : : +- ^ Scan parquet (64) - : +- ^ InputIteratorTransformer (84) - : +- ^ InputAdapter (83) - : +- ^ ShuffleQueryStage (82), Statistics(X) - : +- ColumnarExchange (81) - : +- ^ ProjectExecTransformer (79) - : +- ^ Scan parquet (78) - +- ^ InputIteratorTransformer (98) - +- ^ InputAdapter (97) - +- ^ ShuffleQueryStage (96), Statistics(X) - +- ColumnarExchange (95) - +- ^ ProjectExecTransformer (93) - +- ^ Scan parquet (92) + VeloxColumnarToRowExec (125) + +- ^ SortExecTransformer (123) + +- ^ InputIteratorTransformer (122) + +- ^ InputAdapter (121) + +- ^ ShuffleQueryStage (120), Statistics(X) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ^ InputAdapter (114) + +- ^ ShuffleQueryStage (113), Statistics(X) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner (107) + :- ^ InputIteratorTransformer (98) + : +- ^ InputAdapter (97) + : +- ^ ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner (92) + : :- ^ InputIteratorTransformer (83) + : : +- ^ InputAdapter (82) + : : +- ^ ShuffleQueryStage (81), Statistics(X) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ^ InputAdapter (67) + : : : +- ^ ShuffleQueryStage (66), Statistics(X) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ^ InputAdapter (52) + : : : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ^ InputAdapter (37) + : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ^ InputAdapter (22) + : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ^ InputAdapter (7) + : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ^ InputAdapter (15) + : : : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ^ InputAdapter (30) + : : : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ NoopFilter (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ^ InputAdapter (45) + : : : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ NoopFilter (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ^ InputAdapter (60) + : : : +- ^ ShuffleQueryStage (59), Statistics(X) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ NoopFilter (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ^ InputAdapter (75) + : : +- ^ ShuffleQueryStage (74), Statistics(X) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ NoopFilter (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ^ InputAdapter (90) + : +- ^ ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ NoopFilter (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ^ InputAdapter (105) + +- ^ ShuffleQueryStage (104), Statistics(X) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ NoopFilter (100) + +- ^ Scan parquet (99) +- == Initial Plan == - Sort (168) - +- Exchange (167) - +- HashAggregate (166) - +- Exchange (165) - +- HashAggregate (164) - +- Project (163) - +- ShuffledHashJoin Inner BuildRight (162) - :- Exchange (157) - : +- Project (156) - : +- ShuffledHashJoin Inner BuildRight (155) - : :- Exchange (151) - : : +- Project (150) - : : +- ShuffledHashJoin Inner BuildRight (149) - : : :- Exchange (145) - : : : +- Project (144) - : : : +- ShuffledHashJoin Inner BuildRight (143) - : : : :- Exchange (139) - : : : : +- Project (138) - : : : : +- ShuffledHashJoin Inner BuildRight (137) - : : : : :- Exchange (133) - : : : : : +- Project (132) - : : : : : +- ShuffledHashJoin Inner BuildRight (131) - : : : : : :- Exchange (127) - : : : : : : +- Project (126) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (125) - : : : : : : :- Exchange (121) - : : : : : : : +- Project (120) - : : : : : : : +- Filter (119) - : : : : : : : +- Scan parquet (118) - : : : : : : +- Exchange (124) - : : : : : : +- Filter (123) - : : : : : : +- Scan parquet (122) - : : : : : +- Exchange (130) - : : : : : +- Filter (129) - : : : : : +- Scan parquet (128) - : : : : +- Exchange (136) - : : : : +- Filter (135) - : : : : +- Scan parquet (134) - : : : +- Exchange (142) - : : : +- Filter (141) - : : : +- Scan parquet (140) - : : +- Exchange (148) - : : +- Filter (147) - : : +- Scan parquet (146) - : +- Exchange (154) - : +- Filter (153) - : +- Scan parquet (152) - +- Exchange (161) - +- Project (160) - +- Filter (159) - +- Scan parquet (158) + Sort (176) + +- Exchange (175) + +- HashAggregate (174) + +- Exchange (173) + +- HashAggregate (172) + +- Project (171) + +- ShuffledHashJoin Inner BuildRight (170) + :- Exchange (165) + : +- Project (164) + : +- ShuffledHashJoin Inner BuildRight (163) + : :- Exchange (159) + : : +- Project (158) + : : +- ShuffledHashJoin Inner BuildRight (157) + : : :- Exchange (153) + : : : +- Project (152) + : : : +- ShuffledHashJoin Inner BuildRight (151) + : : : :- Exchange (147) + : : : : +- Project (146) + : : : : +- ShuffledHashJoin Inner BuildRight (145) + : : : : :- Exchange (141) + : : : : : +- Project (140) + : : : : : +- ShuffledHashJoin Inner BuildRight (139) + : : : : : :- Exchange (135) + : : : : : : +- Project (134) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) + : : : : : : :- Exchange (129) + : : : : : : : +- Project (128) + : : : : : : : +- Filter (127) + : : : : : : : +- Scan parquet (126) + : : : : : : +- Exchange (132) + : : : : : : +- Filter (131) + : : : : : : +- Scan parquet (130) + : : : : : +- Exchange (138) + : : : : : +- Filter (137) + : : : : : +- Scan parquet (136) + : : : : +- Exchange (144) + : : : : +- Filter (143) + : : : : +- Scan parquet (142) + : : : +- Exchange (150) + : : : +- Filter (149) + : : : +- Scan parquet (148) + : : +- Exchange (156) + : : +- Filter (155) + : : +- Scan parquet (154) + : +- Exchange (162) + : +- Filter (161) + : +- Scan parquet (160) + +- Exchange (169) + +- Project (168) + +- Filter (167) + +- Scan parquet (166) (1) Scan parquet @@ -162,726 +170,758 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(3) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(11) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(36) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(41) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(56) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_regionkey#X] + +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(73) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(74) ColumnarExchange +(80) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(75) ShuffleQueryStage +(81) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(76) InputAdapter +(82) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(77) InputIteratorTransformer +(83) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) ProjectExecTransformer +(85) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(86) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(80) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(81) ColumnarExchange +(88) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(82) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(83) InputAdapter +(90) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(84) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(85) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(86) ProjectExecTransformer +(93) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(95) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(92) Scan parquet +(99) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(93) ProjectExecTransformer +(100) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(101) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(94) WholeStageCodegenTransformer (X) +(102) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(95) ColumnarExchange +(103) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(104) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(97) InputAdapter +(105) InputAdapter Input [1]: [r_regionkey#X] -(98) InputIteratorTransformer +(106) InputIteratorTransformer Input [1]: [r_regionkey#X] -(99) ShuffledHashJoinExecTransformer +(107) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(100) ProjectExecTransformer +(108) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(101) FlushableHashAggregateExecTransformer +(109) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(102) ProjectExecTransformer +(110) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(103) WholeStageCodegenTransformer (X) +(111) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(104) ColumnarExchange +(112) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(105) ShuffleQueryStage +(113) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(106) InputAdapter +(114) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(107) InputIteratorTransformer +(115) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(108) RegularHashAggregateExecTransformer +(116) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(109) ProjectExecTransformer +(117) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(110) WholeStageCodegenTransformer (X) +(118) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(111) ColumnarExchange +(119) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(112) ShuffleQueryStage +(120) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(113) InputAdapter +(121) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(114) InputIteratorTransformer +(122) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(115) SortExecTransformer +(123) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(116) WholeStageCodegenTransformer (X) +(124) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(117) VeloxColumnarToRowExec +(125) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(118) Scan parquet +(126) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(119) Filter +(127) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(120) Project +(128) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(121) Exchange +(129) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(130) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(123) Filter +(131) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(124) Exchange +(132) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(133) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(126) Project +(134) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(127) Exchange +(135) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(136) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(129) Filter +(137) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(130) Exchange +(138) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(132) Project +(140) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(133) Exchange +(141) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(142) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(135) Filter +(143) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(136) Exchange +(144) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) ShuffledHashJoin +(145) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(138) Project +(146) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(139) Exchange +(147) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(140) Scan parquet +(148) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(141) Filter +(149) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(142) Exchange +(150) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) ShuffledHashJoin +(151) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(144) Project +(152) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(145) Exchange +(153) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Scan parquet +(154) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(147) Filter +(155) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(148) Exchange +(156) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(149) ShuffledHashJoin +(157) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(150) Project +(158) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(151) Exchange +(159) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Scan parquet +(160) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(153) Filter +(161) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(154) Exchange +(162) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(155) ShuffledHashJoin +(163) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(156) Project +(164) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(157) Exchange +(165) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(158) Scan parquet +(166) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(159) Filter +(167) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(160) Project +(168) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(161) Exchange +(169) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(162) ShuffledHashJoin +(170) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(163) Project +(171) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(164) HashAggregate +(172) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(165) Exchange +(173) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) HashAggregate +(174) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(167) Exchange +(175) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(168) Sort +(176) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(169) AdaptiveSparkPlan +(177) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt index 0d3c59505d07..3f5479af2b14 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt @@ -1,120 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (127) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (88) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ^ InputAdapter (78) - +- ^ ShuffleQueryStage (77), Statistics(X) - +- ColumnarExchange (76) - +- ^ ProjectExecTransformer (74) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ ShuffledHashJoinExecTransformer Inner (71) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61), Statistics(X) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54), Statistics(X) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ ShuffleQueryStage (68), Statistics(X) - +- ColumnarExchange (67) - +- ^ ProjectExecTransformer (65) - +- ^ Scan parquet (64) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (126) - +- Exchange (125) - +- HashAggregate (124) - +- Exchange (123) - +- HashAggregate (122) - +- Project (121) - +- ShuffledHashJoin Inner BuildRight (120) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildRight (108) - : : :- Exchange (104) - : : : +- Project (103) - : : : +- ShuffledHashJoin Inner BuildRight (102) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- ShuffledHashJoin Inner BuildLeft (96) - : : : : :- Exchange (92) - : : : : : +- Project (91) - : : : : : +- Filter (90) - : : : : : +- Scan parquet (89) - : : : : +- Exchange (95) - : : : : +- Filter (94) - : : : : +- Scan parquet (93) - : : : +- Exchange (101) - : : : +- Filter (100) - : : : +- Scan parquet (99) - : : +- Exchange (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Exchange (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (119) - +- Filter (118) - +- Scan parquet (117) + Sort (132) + +- Exchange (131) + +- HashAggregate (130) + +- Exchange (129) + +- HashAggregate (128) + +- Project (127) + +- ShuffledHashJoin Inner BuildRight (126) + :- Exchange (122) + : +- Project (121) + : +- ShuffledHashJoin Inner BuildRight (120) + : :- Exchange (116) + : : +- Project (115) + : : +- ShuffledHashJoin Inner BuildRight (114) + : : :- Exchange (110) + : : : +- Project (109) + : : : +- ShuffledHashJoin Inner BuildRight (108) + : : : :- Exchange (104) + : : : : +- Project (103) + : : : : +- ShuffledHashJoin Inner BuildLeft (102) + : : : : :- Exchange (98) + : : : : : +- Project (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Exchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Exchange (125) + +- Filter (124) + +- Scan parquet (123) (1) Scan parquet @@ -124,546 +130,570 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(3) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(11) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(36) Scan parquet +(39) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(41) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_orderdate#X] + +(56) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(73) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(74) ProjectExecTransformer +(80) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(82) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(83) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(84) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(79) InputIteratorTransformer +(85) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(81) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(82) ColumnarExchange +(88) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(84) InputAdapter +(90) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(85) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(86) SortExecTransformer +(92) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(89) Scan parquet +(95) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(90) Filter +(96) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(91) Project +(97) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(92) Exchange +(98) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Scan parquet +(99) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(94) Filter +(100) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(95) Exchange +(101) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(97) Project +(103) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(98) Exchange +(104) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(105) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(100) Filter +(106) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(101) Exchange +(107) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(103) Project +(109) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(104) Exchange +(110) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(111) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(106) Filter +(112) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(107) Exchange +(113) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(109) Project +(115) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(110) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(117) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(113) Exchange +(119) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(115) Project +(121) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(116) Exchange +(122) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(118) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(119) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(121) Project +(127) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(122) HashAggregate +(128) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(123) Exchange +(129) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) HashAggregate +(130) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(125) Exchange +(131) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) Sort +(132) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(127) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala index 17f27a407e52..792ab407836c 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala @@ -329,7 +329,7 @@ class VeloxTPCHV1RasSuite extends VeloxTPCHSuite { super.sparkConf .set("spark.sql.sources.useV1SourceList", "parquet") .set("spark.sql.autoBroadcastJoinThreshold", "-1") - .set("spark.gluten.sql.ras.enabled", "true") + .set("spark.gluten.ras.enabled", "true") } } @@ -340,7 +340,7 @@ class VeloxTPCHV1BhjRasSuite extends VeloxTPCHSuite { super.sparkConf .set("spark.sql.sources.useV1SourceList", "parquet") .set("spark.sql.autoBroadcastJoinThreshold", "30M") - .set("spark.gluten.sql.ras.enabled", "true") + .set("spark.gluten.ras.enabled", "true") } } diff --git a/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala index ae2cea0ba03d..1806eacfc677 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala @@ -16,10 +16,12 @@ */ package org.apache.gluten.planner +import org.apache.gluten.GlutenConfig import org.apache.gluten.extension.columnar.transition.ConventionReq +import org.apache.gluten.planner.cost.GlutenCostModel import org.apache.gluten.planner.property.Conv +import org.apache.gluten.ras.{Cost, CostModel, Ras} import org.apache.gluten.ras.Best.BestNotFoundException -import org.apache.gluten.ras.Ras import org.apache.gluten.ras.RasSuiteBase._ import org.apache.gluten.ras.path.RasPath import org.apache.gluten.ras.property.PropertySet @@ -117,15 +119,46 @@ class VeloxRasSuite extends SharedSparkSession { planner2.plan() } } + + test("User cost model") { + withSQLConf(GlutenConfig.RAS_COST_MODEL.key -> classOf[UserCostModel1].getName) { + val in = RowUnary(RowLeaf(TRIVIAL_SCHEMA)) + val planner = newRas(List(RowUnaryToColumnarUnary)).newPlanner(in) + val out = planner.plan() + assert(out == ColumnarUnary(RowToColumnarExec(RowLeaf(TRIVIAL_SCHEMA)))) + } + withSQLConf(GlutenConfig.RAS_COST_MODEL.key -> classOf[UserCostModel2].getName) { + val in = RowUnary(RowLeaf(TRIVIAL_SCHEMA)) + val planner = newRas(List(RowUnaryToColumnarUnary)).newPlanner(in) + val out = planner.plan() + assert(out == RowUnary(RowLeaf(TRIVIAL_SCHEMA))) + } + withSQLConf(GlutenConfig.RAS_COST_MODEL.key -> "user.dummy.CostModel") { + val in = RowUnary(RowLeaf(TRIVIAL_SCHEMA)) + assertThrows[ClassNotFoundException] { + newRas().newPlanner(in) + } + } + } } object VeloxRasSuite { def newRas(): Ras[SparkPlan] = { - GlutenOptimization(List()).asInstanceOf[Ras[SparkPlan]] + GlutenOptimization + .builder() + .costModel(GlutenCostModel.find()) + .addRules(List()) + .create() + .asInstanceOf[Ras[SparkPlan]] } def newRas(RasRules: Seq[RasRule[SparkPlan]]): Ras[SparkPlan] = { - GlutenOptimization(RasRules).asInstanceOf[Ras[SparkPlan]] + GlutenOptimization + .builder() + .costModel(GlutenCostModel.find()) + .addRules(RasRules) + .create() + .asInstanceOf[Ras[SparkPlan]] } val TRIVIAL_SCHEMA: Seq[AttributeReference] = List(AttributeReference("value", StringType)()) @@ -152,4 +185,32 @@ object VeloxRasSuite { override protected def withNewChildInternal(newChild: SparkPlan): ColumnarUnary = copy(child = newChild) } + + object RowUnaryToColumnarUnary extends RasRule[SparkPlan] { + override def shift(node: SparkPlan): Iterable[SparkPlan] = node match { + case RowUnary(child) => List(ColumnarUnary(child)) + case _ => List.empty + } + override def shape(): Shape[SparkPlan] = Shapes.fixedHeight(1) + } + + class UserCostModel1 extends CostModel[SparkPlan] { + private val base = GlutenCostModel.rough() + override def costOf(node: SparkPlan): Cost = node match { + case _: RowUnary => base.makeInfCost() + case other => base.costOf(other) + } + override def costComparator(): Ordering[Cost] = base.costComparator() + override def makeInfCost(): Cost = base.makeInfCost() + } + + class UserCostModel2 extends CostModel[SparkPlan] { + private val base = GlutenCostModel.rough() + override def costOf(node: SparkPlan): Cost = node match { + case _: ColumnarUnary => base.makeInfCost() + case other => base.costOf(other) + } + override def costComparator(): Ordering[Cost] = base.costComparator() + override def makeInfCost(): Cost = base.makeInfCost() + } } diff --git a/docs/Configuration.md b/docs/Configuration.md index f23ca3b66363..a148ec3aa4cb 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -22,7 +22,7 @@ You can add these configurations into spark-defaults.conf to enable or disable t | spark.plugins | To load Gluten's components by Spark's plug-in loader | org.apache.gluten.GlutenPlugin | | spark.shuffle.manager | To turn on Gluten Columnar Shuffle Plugin | org.apache.spark.shuffle.sort.ColumnarShuffleManager | | spark.gluten.enabled | Enable Gluten, default is true. Just an experimental property. Recommend to enable/disable Gluten through the setting for `spark.plugins`. | true | -| spark.gluten.sql.ras.enabled | Experimental: Enables RAS (relation algebra selector) during physical planning to generate more efficient query plan. Note, this feature is still in development and may not bring performance profits. | false | +| spark.gluten.ras.enabled | Experimental: Enables RAS (relation algebra selector) during physical planning to generate more efficient query plan. Note, this feature is still in development and may not bring performance profits. | false | | spark.gluten.sql.columnar.maxBatchSize | Number of rows to be processed in each batch. Default value is 4096. | 4096 | | spark.gluten.memory.isolation | (Experimental) Enable isolated memory mode. If true, Gluten controls the maximum off-heap memory can be used by each task to X, X = executor memory / max task slots. It's recommended to set true if Gluten serves concurrent queries within a single session, since not all memory Gluten allocated is guaranteed to be spillable. In the case, the feature should be enabled to avoid OOM. Note when true, setting spark.memory.storageFraction to a lower value is suggested since storage memory is considered non-usable by Gluten. | false | | spark.gluten.sql.columnar.scanOnly | When enabled, this config will overwrite all other operators' enabling, and only Scan and Filter pushdown will be offloaded to native. | false | diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala index 3f8ee870609e..a259641f5049 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala @@ -45,35 +45,23 @@ class EnumeratedApplier(session: SparkSession) extends ColumnarRuleApplier with Logging with LogLevelUtil { + // An empirical value. + private val aqeStackTraceIndex = 16 private lazy val transformPlanLogLevel = GlutenConfig.getConf.transformPlanLogLevel private lazy val planChangeLogger = new PlanChangeLogger[SparkPlan]() - private val adaptiveContext = AdaptiveContext(session) + private val adaptiveContext = AdaptiveContext(session, aqeStackTraceIndex) override def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan = - withTransformRules(transformRules(outputsColumnar)).apply(plan) - - // Visible for testing. - private def withTransformRules( - transformRules: List[SparkSession => Rule[SparkPlan]]): Rule[SparkPlan] = - plan => - PhysicalPlanSelector.maybe(session, plan) { - val finalPlan = prepareFallback(plan) { - p => - val suggestedPlan = transformPlan(transformRules, p, "transform") - transformPlan(fallbackPolicies(), suggestedPlan, "fallback") match { - case FallbackNode(fallbackPlan) => - // we should use vanilla c2r rather than native c2r, - // and there should be no `GlutenPlan` any more, - // so skip the `postRules()`. - fallbackPlan - case plan => - transformPlan(postRules(), plan, "post") - } - } - transformPlan(finalRules(), finalPlan, "final") + PhysicalPlanSelector.maybe(session, plan) { + val transformed = transformPlan(transformRules(outputsColumnar), plan, "transform") + val postPlan = maybeAqe { + transformPlan(postRules(), transformed, "post") } + val finalPlan = transformPlan(finalRules(), postPlan, "final") + finalPlan + } private def transformPlan( getRules: List[SparkSession => Rule[SparkPlan]], @@ -95,13 +83,11 @@ class EnumeratedApplier(session: SparkSession) overridden }(t => logOnLevel(transformPlanLogLevel, s"${step}Transform SparkPlan took: $t ms.")) - private def prepareFallback[T](plan: SparkPlan)(f: SparkPlan => T): T = { + private def maybeAqe[T](f: => T): T = { adaptiveContext.setAdaptiveContext() - adaptiveContext.setOriginalPlan(plan) try { - f(plan) + f } finally { - adaptiveContext.resetOriginalPlan() adaptiveContext.resetAdaptiveContext() } } @@ -114,7 +100,6 @@ class EnumeratedApplier(session: SparkSession) List( (_: SparkSession) => RemoveTransitions, (spark: SparkSession) => FallbackOnANSIMode(spark), - (spark: SparkSession) => FallbackMultiCodegens(spark), (spark: SparkSession) => PlanOneRowRelation(spark), (_: SparkSession) => FallbackEmptySchemaRelation() ) ::: @@ -136,16 +121,6 @@ class EnumeratedApplier(session: SparkSession) List((_: SparkSession) => InsertTransitions(outputsColumnar)) } - /** - * Rules to add wrapper `FallbackNode`s on top of the input plan, as hints to make planner fall - * back the whole input plan to the original vanilla Spark plan. - */ - private def fallbackPolicies(): List[SparkSession => Rule[SparkPlan]] = { - List( - (_: SparkSession) => - ExpandFallbackPolicy(adaptiveContext.isAdaptiveContext(), adaptiveContext.originalPlan())) - } - /** * Rules applying to non-fallen-back Gluten plans. To do some post cleanup works on the plan to * make sure it be able to run and be compatible with Spark's execution engine. diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala index c41c1ca2caa0..0b9dcc663246 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala @@ -19,6 +19,7 @@ package org.apache.gluten.extension.columnar.enumerated import org.apache.gluten.extension.columnar.{OffloadExchange, OffloadJoin, OffloadOthers} import org.apache.gluten.extension.columnar.transition.ConventionReq import org.apache.gluten.planner.GlutenOptimization +import org.apache.gluten.planner.cost.GlutenCostModel import org.apache.gluten.planner.property.Conv import org.apache.gluten.ras.property.PropertySet import org.apache.gluten.sql.shims.SparkShimLoader @@ -79,7 +80,13 @@ case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean) RasOffload.from[EvalPythonExec](OffloadOthers()).toRule ) - private val optimization = GlutenOptimization(rules ++ offloadRules) + private val optimization = { + GlutenOptimization + .builder() + .costModel(GlutenCostModel.find()) + .addRules(rules ++ offloadRules) + .create() + } private val reqConvention = Conv.any diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala index 46b3b7f9e088..a3b4831a622d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala @@ -16,13 +16,19 @@ */ package org.apache.gluten.extension.columnar.enumerated -import org.apache.gluten.execution.{BasicScanExecTransformer, FilterExecTransformerBase} +import org.apache.gluten.execution._ +import org.apache.gluten.metrics.{MetricsUpdater, NoopMetricsUpdater} import org.apache.gluten.ras.path.Pattern._ import org.apache.gluten.ras.path.Pattern.Matchers._ import org.apache.gluten.ras.rule.{RasRule, Shape} import org.apache.gluten.ras.rule.Shapes._ +import org.apache.gluten.substrait.SubstraitContext +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder} +import org.apache.spark.sql.catalyst.plans.physical.Partitioning import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.vectorized.ColumnarBatch // Removes Gluten filter operator if its no-op. Typically a Gluten filter is no-op when it // pushes all of its conditions into the child scan. @@ -35,7 +41,8 @@ object RemoveFilter extends RasRule[SparkPlan] { override def shift(node: SparkPlan): Iterable[SparkPlan] = { val filter = node.asInstanceOf[FilterExecTransformerBase] if (filter.isNoop()) { - return List(filter.child) + val out = NoopFilter(filter.child, filter.output) + return List(out) } List.empty } @@ -46,4 +53,30 @@ object RemoveFilter extends RasRule[SparkPlan] { clazz(classOf[FilterExecTransformerBase]), leaf(clazz(classOf[BasicScanExecTransformer])) ).build()) + + // A noop filter placeholder that indicates that all conditions are pushed into scan. + // + // This operator has zero cost in cost model to avoid planner from choosing the + // original filter-scan that doesn't have all conditions pushed down to scan. + // + // We cannot simplify remove the filter to let planner choose the scan since by vanilla + // Spark's definition the filter may have different output nullability than scan. So + // we have to keep this empty filter to let the optimized tree have the identical output schema + // with the original tree. If we simply remove the filter, possible UBs might be caused. For + // example, redundant broadcast exchanges may be added by EnsureRequirements because the + // broadcast join detects that its join keys' nullabilities have been changed. Then AQE + // re-optimization could be broken by ValidateSparkPlan so that AQE could completely + // have no effect as if it's off. This case can be observed by explicitly setting a higher + // AQE logger level to make sure the validation log doesn't get suppressed, e.g., + // spark.sql.adaptive.logLevel=ERROR. + case class NoopFilter(override val child: SparkPlan, override val output: Seq[Attribute]) + extends UnaryTransformSupport { + override def metricsUpdater(): MetricsUpdater = NoopMetricsUpdater + override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = copy(newChild) + override def outputPartitioning: Partitioning = child.outputPartitioning + override def outputOrdering: Seq[SortOrder] = child.outputOrdering + override def doTransform(context: SubstraitContext): TransformContext = + child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doExecuteColumnar(): RDD[ColumnarBatch] = child.executeColumnar() + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala index 2b5b18abb27a..941677a6b933 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala @@ -40,11 +40,13 @@ class HeuristicApplier(session: SparkSession) extends ColumnarRuleApplier with Logging with LogLevelUtil { + // This is an empirical value, may need to be changed for supporting other versions of spark. + private val aqeStackTraceIndex = 19 private lazy val transformPlanLogLevel = GlutenConfig.getConf.transformPlanLogLevel private lazy val planChangeLogger = new PlanChangeLogger[SparkPlan]() - private val adaptiveContext = AdaptiveContext(session) + private val adaptiveContext = AdaptiveContext(session, aqeStackTraceIndex) override def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan = withTransformRules(transformRules(outputsColumnar)).apply(plan) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala index 0592a3acafcc..4a9d69f8f0b1 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala @@ -33,20 +33,19 @@ sealed trait AdaptiveContext { } object AdaptiveContext { - def apply(session: SparkSession): AdaptiveContext = new AdaptiveContextImpl(session) + def apply(session: SparkSession, aqeStackTraceIndex: Int): AdaptiveContext = + new AdaptiveContextImpl(session, aqeStackTraceIndex) private val GLUTEN_IS_ADAPTIVE_CONTEXT = "gluten.isAdaptiveContext" - // This is an empirical value, may need to be changed for supporting other versions of spark. - private val aqeStackTraceIndex = 19 - // Holds the original plan for possible entire fallback. private val localOriginalPlans: ThreadLocal[ListBuffer[SparkPlan]] = ThreadLocal.withInitial(() => ListBuffer.empty[SparkPlan]) private val localIsAdaptiveContextFlags: ThreadLocal[ListBuffer[Boolean]] = ThreadLocal.withInitial(() => ListBuffer.empty[Boolean]) - private class AdaptiveContextImpl(session: SparkSession) extends AdaptiveContext { + private class AdaptiveContextImpl(session: SparkSession, aqeStackTraceIndex: Int) + extends AdaptiveContext { // Just for test use. override def enableAdaptiveContext(): Unit = { session.sparkContext.setLocalProperty(GLUTEN_IS_ADAPTIVE_CONTEXT, "true") diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/GlutenOptimization.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/GlutenOptimization.scala index 98c4ca37c370..555e7d6143bc 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/GlutenOptimization.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/GlutenOptimization.scala @@ -16,27 +16,52 @@ */ package org.apache.gluten.planner -import org.apache.gluten.planner.cost.GlutenCostModel import org.apache.gluten.planner.metadata.GlutenMetadataModel import org.apache.gluten.planner.plan.GlutenPlanModel import org.apache.gluten.planner.property.GlutenPropertyModel -import org.apache.gluten.ras.{Optimization, RasExplain} +import org.apache.gluten.ras.{CostModel, Optimization, RasExplain} import org.apache.gluten.ras.rule.RasRule import org.apache.spark.sql.execution.SparkPlan +import scala.collection.mutable + object GlutenOptimization { + def builder(): Builder = new BuilderImpl + private object GlutenExplain extends RasExplain[SparkPlan] { override def describeNode(node: SparkPlan): String = node.nodeName } - def apply(rules: Seq[RasRule[SparkPlan]]): Optimization[SparkPlan] = { - Optimization[SparkPlan]( - GlutenPlanModel(), - GlutenCostModel(), - GlutenMetadataModel(), - GlutenPropertyModel(), - GlutenExplain, - RasRule.Factory.reuse(rules)) + trait Builder { + def addRules(rules: Seq[RasRule[SparkPlan]]): Builder + def costModel(costModel: CostModel[SparkPlan]): Builder + def create(): Optimization[SparkPlan] + } + + private class BuilderImpl extends Builder { + private val rules: mutable.ListBuffer[RasRule[SparkPlan]] = mutable.ListBuffer() + private var costModel: Option[CostModel[SparkPlan]] = None + + override def addRules(rules: Seq[RasRule[SparkPlan]]): Builder = { + this.rules ++= rules + this + } + + override def costModel(costModel: CostModel[SparkPlan]): Builder = { + this.costModel = Some(costModel) + this + } + + override def create(): Optimization[SparkPlan] = { + assert(costModel.isDefined, "Cost model is required to initialize GlutenOptimization") + Optimization[SparkPlan]( + GlutenPlanModel(), + costModel.get, + GlutenMetadataModel(), + GlutenPropertyModel(), + GlutenExplain, + RasRule.Factory.reuse(rules)) + } } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala index fa69eedb5f23..4b5fa6803eec 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala @@ -16,23 +16,40 @@ */ package org.apache.gluten.planner.cost +import org.apache.gluten.GlutenConfig import org.apache.gluten.extension.columnar.OffloadJoin +import org.apache.gluten.extension.columnar.enumerated.RemoveFilter import org.apache.gluten.extension.columnar.transition.{ColumnarToRowLike, RowToColumnarLike} import org.apache.gluten.planner.plan.GlutenPlanModel.GroupLeafExec import org.apache.gluten.ras.{Cost, CostModel} import org.apache.gluten.utils.PlanUtil +import org.apache.spark.internal.Logging import org.apache.spark.sql.execution.{ColumnarToRowExec, RowToColumnarExec, SparkPlan} import org.apache.spark.sql.execution.joins.ShuffledHashJoinExec +import org.apache.spark.sql.utils.ReflectionUtil -class GlutenCostModel {} - -object GlutenCostModel { - def apply(): CostModel[SparkPlan] = { - RoughCostModel +object GlutenCostModel extends Logging { + def find(): CostModel[SparkPlan] = { + val aliases: Map[String, Class[_ <: CostModel[SparkPlan]]] = Map( + "rough" -> classOf[RoughCostModel]) + val aliasOrClass = GlutenConfig.getConf.rasCostModel + val clazz: Class[_ <: CostModel[SparkPlan]] = if (aliases.contains(aliasOrClass)) { + aliases(aliasOrClass) + } else { + val userModel = ReflectionUtil.classForName(aliasOrClass) + logInfo(s"Using user cost model: $aliasOrClass") + userModel + } + val ctor = clazz.getDeclaredConstructor() + ctor.setAccessible(true) + val model = ctor.newInstance() + model } - private object RoughCostModel extends CostModel[SparkPlan] { + def rough(): CostModel[SparkPlan] = new RoughCostModel() + + private class RoughCostModel extends CostModel[SparkPlan] { private val infLongCost = Long.MaxValue override def costOf(node: SparkPlan): GlutenCost = node match { @@ -62,6 +79,9 @@ object GlutenCostModel { // To exclude the rewritten intermediate plan that is not executable // by vanilla Spark and was generated by strategy "JoinSelectionOverrides" infLongCost + case _: RemoveFilter.NoopFilter => + // To make planner choose the tree that has applied rule PushFilterToScan. + 0L case ColumnarToRowExec(child) => 3L case RowToColumnarExec(child) => 3L case ColumnarToRowLike(child) => 3L diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/utils/ReflectionUtil.scala b/gluten-core/src/main/scala/org/apache/spark/sql/utils/ReflectionUtil.scala new file mode 100644 index 000000000000..f674e7ec2e70 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/spark/sql/utils/ReflectionUtil.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.utils + +import org.apache.spark.util.Utils + +object ReflectionUtil { + def classForName[C]( + className: String, + initialize: Boolean = true, + noSparkClassLoader: Boolean = false): Class[C] = { + Utils.classForName(className, initialize, noSparkClassLoader) + } +} diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 0fcaed8217ec..9db063a6623f 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -42,8 +42,6 @@ class GlutenConfig(conf: SQLConf) extends Logging { def enableGluten: Boolean = conf.getConf(GLUTEN_ENABLED) - def enableRas: Boolean = conf.getConf(RAS_ENABLED) - // FIXME the option currently controls both JVM and native validation against a Substrait plan. def enableNativeValidation: Boolean = conf.getConf(NATIVE_VALIDATION_ENABLED) @@ -251,6 +249,11 @@ class GlutenConfig(conf: SQLConf) extends Logging { def conservativeTaskOffHeapMemorySize: Long = conf.getConf(COLUMNAR_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES) + // Options used by RAS. + def enableRas: Boolean = conf.getConf(RAS_ENABLED) + + def rasCostModel: String = conf.getConf(RAS_COST_MODEL) + def enableVeloxCache: Boolean = conf.getConf(COLUMNAR_VELOX_CACHE_ENABLED) def veloxMemCacheSize: Long = conf.getConf(COLUMNAR_VELOX_MEM_CACHE_SIZE) @@ -705,15 +708,6 @@ object GlutenConfig { .booleanConf .createWithDefault(GLUTEN_ENABLE_BY_DEFAULT) - val RAS_ENABLED = - buildConf("spark.gluten.sql.ras.enabled") - .doc( - "Experimental: Enables RAS (relational algebra selector) during physical " + - "planning to generate more efficient query plan. Note, this feature is still in " + - "development and may not bring performance profits.") - .booleanConf - .createWithDefault(false) - // FIXME the option currently controls both JVM and native validation against a Substrait plan. val NATIVE_VALIDATION_ENABLED = buildConf("spark.gluten.sql.enable.native.validation") @@ -1202,7 +1196,25 @@ object GlutenConfig { .bytesConf(ByteUnit.BYTE) .createWithDefaultString("8MB") - // velox caching options + // Options used by RAS. + val RAS_ENABLED = + buildConf("spark.gluten.ras.enabled") + .doc( + "Experimental: Enables RAS (relational algebra selector) during physical " + + "planning to generate more efficient query plan. Note, this feature is still in " + + "development and may not bring performance profits.") + .booleanConf + .createWithDefault(false) + + val RAS_COST_MODEL = + buildConf("spark.gluten.ras.costModel") + .doc( + "Experimental: The classpath of user-defined cost model that will be used by RAS. " + + "If not specified, a rough built-in cost model will be used.") + .stringConf + .createWithDefaultString("rough") + + // velox caching options. val COLUMNAR_VELOX_CACHE_ENABLED = buildStaticConf("spark.gluten.sql.columnar.backend.velox.cacheEnabled") .internal()