Merge branch 'main' into wip_spark344

apache · Dec 9, 2024 · 619df7c · 619df7c
2 parents 36c0ee6 + 15f4cde
commit 619df7c
Show file tree

Hide file tree

Showing 966 changed files with 30,626 additions and 16,996 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -27,6 +27,9 @@ github:
     - spark-sql
     - vectorization
     - velox
+  collaborators:
+    - majetideepak
+    - pedroerp
   enabled_merge_buttons:
     squash: true
     merge: false

diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -60,7 +60,7 @@ CORE:
 VELOX:
   - changed-files:
     - any-glob-to-any-file: [
-      'gluten-data/**/*',
+      'gluten-arrow/**/*',
       'backends-velox/**/*',
       'ep/build-velox/**/*',
       'cpp/**/*'

diff --git a/.github/workflows/build_bundle_package.yml b/.github/workflows/build_bundle_package.yml
@@ -17,6 +17,7 @@ name: Build bundle package
 
 env:
   ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+  CCACHE_DIR: "${{ github.workspace }}/.ccache"
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
@@ -25,10 +26,6 @@ concurrency:
 on:
   workflow_dispatch:
     inputs:
-      os:
-        description: 'OS version: ubuntu:20.04, ubuntu:22.04, centos:7 or centos:8'
-        required: true
-        default: 'ubuntu:20.04'
       spark:
         description: 'Spark version: spark-3.2, spark-3.3, spark-3.4 or spark-3.5'
         required: true
@@ -41,123 +38,58 @@ on:
 jobs:
   build-native-lib:
     runs-on: ubuntu-20.04
-    container: apache/gluten:gluten-vcpkg-builder_2024_05_29
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+      - name: Get Ccache
+        uses: actions/cache/restore@v3
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-centos7-release-default-${{github.sha}}
+          restore-keys: |
+            ccache-centos7-release-default
       - name: Build Gluten velox third party
         run: |
-          yum install sudo patch java-1.8.0-openjdk-devel -y && \
-          cd $GITHUB_WORKSPACE/ep/build-velox/src && \
-          ./get_velox.sh && \
-          source /opt/rh/devtoolset-11/enable && \
-          cd $GITHUB_WORKSPACE/ && \
-          export NUM_THREADS=4
-          ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=OFF  --build_benchmarks=OFF --enable_s3=OFF \
-          --enable_gcs=OFF --enable_hdfs=ON --enable_abfs=OFF
+          docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7 bash -c "
+            df -a
+            cd /work
+            export CCACHE_DIR=/work/.ccache
+            bash dev/ci-velox-buildstatic-centos-7.sh
+            ccache -s
+            mkdir -p /work/.m2/repository/org/apache/arrow/
+            cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/
+          "
       - name: Upload native libs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           path: ./cpp/build/releases/
           name: velox-native-lib-${{github.sha}}
           retention-days: 1
       - name: Upload Artifact Arrow Jar
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           path: /root/.m2/repository/org/apache/arrow/
           name: velox-arrow-jar-centos-7-${{github.sha}}
 
-  build-bundle-package-ubuntu:
-    if: startsWith(github.event.inputs.os, 'ubuntu')
-    needs: build-native-lib
-    runs-on: ubuntu-20.04
-    container: ${{ github.event.inputs.os }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Download All Artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: velox-native-lib-${{github.sha}}
-          path: ./cpp/build/releases
-      - name: Download All Arrow Jar Artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: velox-arrow-jar-centos-7-${{github.sha}}
-          path: /root/.m2/repository/org/apache/arrow/
-      - name: Setup java and maven
-        run: |
-          apt-get update && \
-          apt-get install -y openjdk-8-jdk maven && \
-          apt remove openjdk-11* -y
-      - name: Build for Spark ${{ github.event.inputs.spark }}
-        run: |
-          cd $GITHUB_WORKSPACE/ && \
-          mvn clean install -P${{ github.event.inputs.spark }} -Dhadoop.version=${{ github.event.inputs.hadoop }} -Pbackends-velox -Pceleborn -DskipTests -Dmaven.source.skip
-      - name: Upload bundle package
-        uses: actions/upload-artifact@v2
-        with:
-          name: gluten-velox-bundle-package
-          path: package/target/gluten-velox-bundle-*.jar
-          retention-days: 7
-
-  build-bundle-package-centos7:
-    if: ${{ github.event.inputs.os == 'centos:7' }}
-    needs: build-native-lib
-    runs-on: ubuntu-20.04
-    container: ${{ github.event.inputs.os }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Download All Artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: velox-native-lib-${{github.sha}}
-          path: ./cpp/build/releases
-      - name: Download All Arrow Jar Artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: velox-arrow-jar-centos-7-${{github.sha}}
-          path: /root/.m2/repository/org/apache/arrow/
-      - name: Setup java and maven
-        run: |
-          yum update -y && yum install -y java-1.8.0-openjdk-devel wget && \
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz && \
-          tar -xvf apache-maven-3.8.8-bin.tar.gz && \
-          mv apache-maven-3.8.8 /usr/lib/maven
-      - name: Build for Spark ${{ github.event.inputs.spark }}
-        run: |
-          cd $GITHUB_WORKSPACE/ && \
-          export MAVEN_HOME=/usr/lib/maven && \
-          export PATH=${PATH}:${MAVEN_HOME}/bin && \
-          mvn clean install -P${{ github.event.inputs.spark }} -Dhadoop.version=${{ github.event.inputs.hadoop }} -Pbackends-velox -Pceleborn -DskipTests -Dmaven.source.skip
-      - name: Upload bundle package
-        uses: actions/upload-artifact@v2
-        with:
-          name: gluten-velox-bundle-package
-          path: package/target/gluten-velox-bundle-*.jar
-          retention-days: 7
-
   build-bundle-package-centos8:
-    if: ${{ github.event.inputs.os == 'centos:8' }}
     needs: build-native-lib
     runs-on: ubuntu-20.04
-    container: ${{ github.event.inputs.os }}
+    container: centos:8
     steps:
       - uses: actions/checkout@v2
       - name: Download All Artifacts
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: velox-native-lib-${{github.sha}}
           path: ./cpp/build/releases
       - name: Download All Arrow Jar Artifacts
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: velox-arrow-jar-centos-7-${{github.sha}}
           path: /root/.m2/repository/org/apache/arrow/
-      - name: Update mirror list
-        run: |
-          sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true && \
-          sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
       - name: Setup java and maven
         run: |
+          sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \
+          sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* && \
           yum update -y && yum install -y java-1.8.0-openjdk-devel wget && \
           wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz && \
           tar -xvf apache-maven-3.8.8-bin.tar.gz && \
@@ -167,10 +99,11 @@ jobs:
           cd $GITHUB_WORKSPACE/ && \
           export MAVEN_HOME=/usr/lib/maven && \
           export PATH=${PATH}:${MAVEN_HOME}/bin && \
-          mvn clean install -P${{ github.event.inputs.spark }} -Dhadoop.version=${{ github.event.inputs.hadoop }} -Pbackends-velox -Pceleborn -DskipTests -Dmaven.source.skip
+          mvn clean install -P${{ github.event.inputs.spark }} -Dhadoop.version=${{ github.event.inputs.hadoop }} -Pbackends-velox -Pceleborn -Puniffle -DskipTests -Dmaven.source.skip
       - name: Upload bundle package
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: gluten-velox-bundle-package
           path: package/target/gluten-velox-bundle-*.jar
           retention-days: 7
+
diff --git a/.github/workflows/clickhouse_be_trigger.yml b/.github/workflows/clickhouse_be_trigger.yml
@@ -47,9 +47,16 @@ jobs:
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
+            const issueNumber = context.payload.number;
+            let body;
+            if (issueNumber % 10 === 0) {
+              body = "Run Gluten ClickHouse CI on ARM";
+            } else {
+              body = "Run Gluten Clickhouse CI on x86";
+            }
             await github.rest.issues.createComment({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              issue_number: context.payload.number,
-              body: "Run Gluten Clickhouse CI"
+              issue_number: issueNumber,
+              body: body
             });
diff --git a/.github/workflows/util/install_spark_resources.sh b/.github/workflows/util/install_spark_resources.sh
@@ -63,26 +63,26 @@ case "$1" in
 3.5)
     # Spark-3.5
     cd ${INSTALL_DIR} && \
-    wget -nv https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz && \
-    tar --strip-components=1 -xf spark-3.5.3-bin-hadoop3.tgz spark-3.5.3-bin-hadoop3/jars/ && \
-    rm -rf spark-3.5.3-bin-hadoop3.tgz && \
+    wget -nv https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz && \
+    tar --strip-components=1 -xf spark-3.5.2-bin-hadoop3.tgz spark-3.5.2-bin-hadoop3/jars/ && \
+    rm -rf spark-3.5.2-bin-hadoop3.tgz && \
     mkdir -p ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.12 && \
     mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.12 && \
-    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.3.tar.gz && \
-    tar --strip-components=1 -xf v3.5.3.tar.gz spark-3.5.3/sql/core/src/test/resources/  && \
+    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.2.tar.gz && \
+    tar --strip-components=1 -xf v3.5.2.tar.gz spark-3.5.2/sql/core/src/test/resources/  && \
     mkdir -p shims/spark35/spark_home/ && \
     mv sql shims/spark35/spark_home/
     ;;
 3.5-scala2.13)
     # Spark-3.5, scala 2.13
     cd ${INSTALL_DIR} && \
-    wget -nv https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz && \
-    tar --strip-components=1 -xf spark-3.5.3-bin-hadoop3.tgz spark-3.5.3-bin-hadoop3/jars/ && \
-    rm -rf spark-3.5.3-bin-hadoop3.tgz && \
+    wget -nv https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz && \
+    tar --strip-components=1 -xf spark-3.5.2-bin-hadoop3.tgz spark-3.5.2-bin-hadoop3/jars/ && \
+    rm -rf spark-3.5.2-bin-hadoop3.tgz && \
     mkdir -p ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \
     mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \
-    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.3.tar.gz && \
-    tar --strip-components=1 -xf v3.5.3.tar.gz spark-3.5.3/sql/core/src/test/resources/  && \
+    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.2.tar.gz && \
+    tar --strip-components=1 -xf v3.5.2.tar.gz spark-3.5.2/sql/core/src/test/resources/  && \
     mkdir -p shims/spark35/spark_home/ && \
     mv sql shims/spark35/spark_home/
     ;;