From 2649fa7b34edf99268bff0d95d9d9621683f839c Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 27 Nov 2024 09:06:50 +0800 Subject: [PATCH] [GLUTEN-8046][VL] CI: fix velox cache/bundle package script (#8051) This patch fixes the cache/build script due to nodejs version update in GHA This patch also adds centos7 tests back fixes #8046 Signed-off-by: Yuan Zhou --- .github/workflows/build_bundle_package.yml | 19 ++++-- .github/workflows/velox_backend.yml | 78 +++++++++++++++++++--- .github/workflows/velox_backend_cache.yml | 8 +-- 3 files changed, 86 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_bundle_package.yml b/.github/workflows/build_bundle_package.yml index 9b94d302f962..c7ddd8748b25 100644 --- a/.github/workflows/build_bundle_package.yml +++ b/.github/workflows/build_bundle_package.yml @@ -38,9 +38,8 @@ on: jobs: build-native-lib: runs-on: ubuntu-20.04 - container: apache/gluten:vcpkg-centos-7 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Get Ccache uses: actions/cache/restore@v3 with: @@ -50,9 +49,15 @@ jobs: ccache-centos7-release-default - name: Build Gluten velox third party run: | - df -a - cd $GITHUB_WORKSPACE/ - bash dev/ci-velox-buildstatic-centos-7.sh + docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7 bash -c " + df -a + cd /work + export CCACHE_DIR=/work/.ccache + bash dev/ci-velox-buildstatic-centos-7.sh + ccache -s + mkdir -p /work/.m2/repository/org/apache/arrow/ + cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/ + " - name: Upload native libs uses: actions/upload-artifact@v3 with: @@ -65,10 +70,10 @@ jobs: path: /root/.m2/repository/org/apache/arrow/ name: velox-arrow-jar-centos-7-${{github.sha}} - build-bundle-package-centos7: + build-bundle-package-centos8: needs: build-native-lib runs-on: ubuntu-20.04 - container: centos:7 + container: centos:8 steps: - uses: actions/checkout@v2 - name: Download All Artifacts diff --git a/.github/workflows/velox_backend.yml b/.github/workflows/velox_backend.yml index 56b43975acee..3c4bb3e2e97d 100644 --- a/.github/workflows/velox_backend.yml +++ b/.github/workflows/velox_backend.yml @@ -164,12 +164,12 @@ jobs: && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 - run-tpc-test-centos: + run-tpc-test-centos8: needs: build-native-lib-centos-7 strategy: fail-fast: false matrix: - os: [ "centos:8" ] #TODO: add centos7 tests back with docker run + os: [ "centos:8" ] spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5" ] java: [ "java-8", "java-11", "java-17" ] # Spark supports JDK17 since 3.3 and later, see https://issues.apache.org/jira/browse/SPARK-33772 @@ -255,6 +255,68 @@ jobs: --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ --extra-conf=spark.gluten.ras.enabled=true + run-tpc-test-centos7: + needs: build-native-lib-centos-7 + strategy: + fail-fast: false + matrix: + spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5" ] + java: [ "java-8" ] + # Spark supports JDK17 since 3.3 and later, see https://issues.apache.org/jira/browse/SPARK-33772 + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - name: Download All Native Artifacts + uses: actions/download-artifact@v3 + with: + name: velox-native-lib-centos-7-${{github.sha}} + path: ./cpp/build/releases/ + - name: Download All Arrow Jar Artifacts + uses: actions/download-artifact@v3 + with: + name: arrow-jars-centos-7-${{github.sha}} + path: .m2/repository/org/apache/arrow/ + - name: Build and run TPCH/DS tests + run: | + docker pull centos:7 + docker run -v $GITHUB_WORKSPACE:/work -v /$GITHUB_WORKSPACE/.m2:/root/.m2/ -w /work \ + -e matrix.java=${{ matrix.java }} -e matrix.spark=${{ matrix.spark }} \ + centos:7 \ + bash -c " + sed -i -e 's|mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-* || true + sed -i -e 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true + + # Setup java and maven + yum update -y && yum install -y java-1.8.0-openjdk-devel wget + + wget -nv https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven + export PATH=${PATH}:/usr/lib/maven/bin + + # Set environment variables + export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk + + # Build gluten-it + mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -DskipTests + cd /work/tools/gluten-it + mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }} + + # Run TPC-H / TPC-DS + GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ + --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 + GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ + --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 + + # Run TPC-H / TPC-DS with RAS + cd /work/tools/gluten-it + GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ + --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ + --extra-conf=spark.gluten.ras.enabled=true \ + && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ + --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ + --extra-conf=spark.gluten.ras.enabled=true + " + run-tpc-test-ubuntu-oom: needs: build-native-lib-centos-7 strategy: @@ -962,12 +1024,12 @@ jobs: df -a bash dev/ci-velox-buildshared-centos-8.sh ccache -s - - name: "Save ccache" - uses: actions/cache/save@v3 - id: ccache - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-centos8-release-default-${{github.sha}} + # - name: "Save ccache" + # uses: actions/cache/save@v3 + # id: ccache + # with: + # path: '${{ env.CCACHE_DIR }}' + # key: ccache-centos8-release-default-${{github.sha}} - name: Run CPP unit test run: | cd ./cpp/build && ctest -V diff --git a/.github/workflows/velox_backend_cache.yml b/.github/workflows/velox_backend_cache.yml index e969ffb7415e..d0b295b5f48e 100644 --- a/.github/workflows/velox_backend_cache.yml +++ b/.github/workflows/velox_backend_cache.yml @@ -31,9 +31,8 @@ concurrency: jobs: cache-native-lib-centos-7: runs-on: ubuntu-20.04 - container: apache/gluten:vcpkg-centos-7 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Get Ccache uses: actions/cache/restore@v3 with: @@ -43,8 +42,9 @@ jobs: ccache-centos7-release-default - name: Build Gluten native libraries run: | - df -a - bash dev/ci-velox-buildstatic-centos-7.sh + docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7 bash -c " + bash dev/ci-velox-buildstatic-centos-7.sh + " - name: Save Ccache uses: actions/cache/save@v3 id: ccache