[GLUTEN-7143][VL] CI: Add GHA job for running all UTs with RAS=ON (ap…

…ache#7702)
yikf · Oct 29, 2024 · b008a05 · b008a05
1 parent 8324a88
commit b008a05
Showing 1 changed file with 91 additions and 3 deletions.
diff --git a/.github/workflows/velox_backend.yml b/.github/workflows/velox_backend.yml
@@ -854,7 +854,7 @@ jobs:
         run: |
           yum install sudo patch java-1.8.0-openjdk-devel wget -y
           $SETUP install_maven
-      - name: Prepare spark.test.home for Spark 3.5.2 (other tests)
+      - name: Prepare spark.test.home for Spark 3.5.3 (other tests)
         run: |
           bash .github/workflows/util/install_spark_resources.sh 3.5
           dnf module -y install python39 && \
@@ -900,7 +900,7 @@ jobs:
         run: |
           yum install sudo patch java-1.8.0-openjdk-devel wget -y
           $SETUP install_maven
-      - name: Prepare spark.test.home for Spark 3.5.2 (other tests)
+      - name: Prepare spark.test.home for Spark 3.5.3 (other tests)
         run: |
           bash .github/workflows/util/install_spark_resources.sh 3.5-scala2.13
           dnf module -y install python39 && \
@@ -946,7 +946,7 @@ jobs:
         run: |
           yum install sudo patch java-1.8.0-openjdk-devel wget -y
           $SETUP install_maven
-      - name: Prepare spark.test.home for Spark 3.5.2 (other tests)
+      - name: Prepare spark.test.home for Spark 3.5.3 (slow tests)
         run: |
           bash .github/workflows/util/install_spark_resources.sh 3.5
       - name: Build and Run unit test for Spark 3.5.3 (slow tests)
@@ -962,6 +962,94 @@ jobs:
           name: test-report-spark35-slow
           path: '**/surefire-reports/TEST-*.xml'
 
+  run-spark-test-spark35-ras:
+    needs: build-native-lib-centos-7
+    runs-on: ubuntu-20.04
+    container: apache/gluten:centos-8
+    steps:
+      - uses: actions/checkout@v2
+      - name: Download All Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: velox-native-lib-centos-7-${{github.sha}}
+          path: ./cpp/build/releases
+      - name: Download Arrow Jars
+        uses: actions/download-artifact@v3
+        with:
+          name: arrow-jars-centos-7-${{github.sha}}
+          path: /root/.m2/repository/org/apache/arrow/
+      - name: Update mirror list
+        run: |
+          sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
+          sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
+      - name: Setup build dependency
+        run: |
+          yum install sudo patch java-1.8.0-openjdk-devel wget -y
+          $SETUP install_maven
+      - name: Prepare spark.test.home for Spark 3.5.3 (other tests)
+        run: |
+          bash .github/workflows/util/install_spark_resources.sh 3.5
+          dnf module -y install python39 && \
+          alternatives --set python3 /usr/bin/python3.9 && \
+          pip3 install setuptools && \
+          pip3 install pyspark==3.5.3 cython && \
+          pip3 install pandas pyarrow
+      - name: (To be fixed) Build and Run unit test for Spark 3.5.3 (other tests)
+        continue-on-error: true
+        run: |
+          cd $GITHUB_WORKSPACE/
+          export SPARK_SCALA_VERSION=2.12
+          $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \
+          -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/ -Dspark.gluten.ras.enabled=true" \
+          -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
+      - name: (To be enabled) Upload test report
+        if: false
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-report-spark35-ras
+          path: '**/surefire-reports/TEST-*.xml'
+
+  run-spark-test-spark35-slow-ras:
+    needs: build-native-lib-centos-7
+    runs-on: ubuntu-20.04
+    container: apache/gluten:centos-8
+    steps:
+      - uses: actions/checkout@v2
+      - name: Download All Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: velox-native-lib-centos-7-${{github.sha}}
+          path: ./cpp/build/releases
+      - name: Download Arrow Jars
+        uses: actions/download-artifact@v3
+        with:
+          name: arrow-jars-centos-7-${{github.sha}}
+          path: /root/.m2/repository/org/apache/arrow/
+      - name: Update mirror list
+        run: |
+          sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
+          sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
+      - name: Setup build dependency
+        run: |
+          yum install sudo patch java-1.8.0-openjdk-devel wget -y
+          $SETUP install_maven
+      - name: Prepare spark.test.home for Spark 3.5.3 (slow tests)
+        run: |
+          bash .github/workflows/util/install_spark_resources.sh 3.5
+      - name: (To be fixed) Build and Run unit test for Spark 3.5.3 (slow tests)
+        continue-on-error: true
+        run: |
+          cd $GITHUB_WORKSPACE/
+          $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \
+          -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/ -Dspark.gluten.ras.enabled=true" \
+          -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
+      - name: (To be enabled) Upload test report
+        if: false
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-report-spark35-slow-ras
+          path: '**/surefire-reports/TEST-*.xml'
+
   run-cpp-test-udf-test:
     runs-on: ubuntu-20.04
     container: apache/gluten:centos-8