spark_sql_test_ansi.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Spark SQL Tests (ANSI mode)

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

on:
  push:
    paths-ignore:
      - "docs/**"
      - "**.md"
  pull_request:
    paths-ignore:
      - "docs/**"
      - "**.md"
  # manual trigger
  # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
  workflow_dispatch:

env:
  RUST_VERSION: stable

jobs:
  spark-sql-catalyst:
    strategy:
      matrix:
        os: [ubuntu-24.04]
        java-version: [17]
        spark-version: [{short: '4.0', full: '4.0.0-preview1'}]
        module:
          - {name: "catalyst", args1: "catalyst/test", args2: ""}
          - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}
          - {name: "sql/core-2", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.ExtendedSQLTest"}
          - {name: "sql/core-3", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.SlowSQLTest"}
          - {name: "sql/hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"}
          - {name: "sql/hive-2", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.ExtendedHiveTest"}
          - {name: "sql/hive-3", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.SlowHiveTest"}
      fail-fast: false
    name: spark-sql-${{ matrix.module.name }}/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }}
    runs-on: ${{ matrix.os }}
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v4
      - name: Setup Rust & Java toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: ${{env.RUST_VERSION}}
          jdk-version: ${{ matrix.java-version }}
      - name: Setup Spark
        uses: ./.github/actions/setup-spark-builder
        with:
          spark-version: ${{ matrix.spark-version.full }}
          spark-short-version: ${{ matrix.spark-version.short }}
          comet-version: '0.2.0-SNAPSHOT' # TODO: get this from pom.xml
      - name: Run Spark tests
        run: |
          cd apache-spark
          rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
          RUST_BACKTRACE=1 ENABLE_COMET=true ENABLE_COMET_ANSI_MODE=true ENABLE_COMET_SHUFFLE=${{ matrix.module.name == 'sql/core-1' && 'false' || 'true' }} build/sbt ${{ matrix.module.args1 }} "${{ matrix.module.args2 }}"
        env:
          LC_ALL: "C.UTF-8"