[VL] Add a docker build job and use pre-installed arrow libs (apache#…

…6826)

.github/workflows/build_bundle_package.yml

@@ -52,7 +52,6 @@ jobs:
           source /opt/rh/devtoolset-9/enable && \
           source /opt/gluten//dev/vcpkg/env.sh && \
           cd $GITHUB_WORKSPACE/ && \
-          sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt && \
           export NUM_THREADS=4
           ./dev/builddeps-veloxbe.sh --build_tests=OFF  --build_benchmarks=OFF --enable_s3=OFF \
           --enable_gcs=OFF --enable_hdfs=ON --enable_abfs=OFF

.github/workflows/docker_image.yml

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build and Push Docker Image
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/docker_image.yml'
+  schedule:
+    - cron: '0 20 * * 0'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: dev/vcpkg/docker/Dockerfile.gha
+          push: true
+          tags: apache/gluten:vcpkg-centos-7

.github/workflows/velox_docker.yml

@@ -51,7 +51,7 @@ concurrency:
 jobs:
   build-native-lib-centos-7:
     runs-on: ubuntu-20.04
-    container: apache/gluten:gluten-vcpkg-builder_2024_08_05 # centos7 with dependencies installed
+    container: apache/gluten:vcpkg-centos-7
     steps:
       - uses: actions/checkout@v2
       - name: Generate cache key
@@ -63,7 +63,6 @@ jobs:
         with:
           path: |
             ./cpp/build/releases/
-            /root/.m2/repository/org/apache/arrow/
           key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
       - name: Build Gluten native libraries
         if: ${{ steps.cache.outputs.cache-hit != 'true' }}

.github/workflows/velox_docker_cache.yml

@@ -30,7 +30,7 @@ concurrency:
 jobs:
   cache-native-lib-centos-7:
     runs-on: ubuntu-20.04
-    container: apache/gluten:gluten-vcpkg-builder_2024_08_05 # centos7 with dependencies installed
+    container: apache/gluten:vcpkg-centos-7
     steps:
       - uses: actions/checkout@v2
       - name: Generate cache key
@@ -43,7 +43,6 @@ jobs:
           lookup-only: true
           path: |
             ./cpp/build/releases/
-            /root/.m2/repository/org/apache/arrow/
           key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
       - name: Build Gluten native libraries
         if: steps.check-cache.outputs.cache-hit != 'true'
@@ -57,7 +56,6 @@ jobs:
         with:
           path: |
             ./cpp/build/releases/
-            /root/.m2/repository/org/apache/arrow/  
           key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
 
   cache-native-lib-centos-8:

dev/ci-velox-buildstatic-centos-7.sh

@@ -2,12 +2,9 @@
 
 set -e
 
-yum install sudo patch java-1.8.0-openjdk-devel -y
-cd $GITHUB_WORKSPACE/ep/build-velox/src
-./get_velox.sh
 source /opt/rh/devtoolset-9/enable
 cd $GITHUB_WORKSPACE/
 source ./dev/vcpkg/env.sh
-sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt
 export NUM_THREADS=4
-./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON
+./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --build_arrow=OFF --enable_s3=ON \
+                           --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON

dev/vcpkg/docker/Dockerfile.gha

@@ -11,15 +11,20 @@ RUN sed -i \
        -e 's/mirror\.centos\.org/vault.centos.org/' \
        /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
 
-RUN yum install -y git patch wget sudo
+RUN yum install -y git patch wget sudo java-1.8.0-openjdk-devel
 
-# build
 RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten
 
 RUN echo "check_certificate = off" >> ~/.wgetrc
-# deps
+
 RUN cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh
 
-# vcpkg env
+# An actual path used for vcpkg cache.
+RUN mkdir -p /var/cache/vcpkg
+
+# Set vcpkg cache path.
 ENV VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite
-RUN source /opt/rh/devtoolset-9/enable && cd /opt/gluten && source dev/vcpkg/env.sh
+
+# Build arrow, then install the native libs to system paths and jar package to .m2/ directory.
+RUN cd /opt/gluten && source /opt/rh/devtoolset-9/enable && source ./dev/vcpkg/env.sh && \
+    bash ./dev/builddeps-veloxbe.sh build_arrow && rm -rf /opt/gluten

dev/vcpkg/env.sh

@@ -1,4 +1,5 @@
 #! /bin/bash
+set -e
 
 if [ -z "${BASH_SOURCE[0]}" ] || [ "$0" == "${BASH_SOURCE[0]}" ]; then
     echo "env.sh should only be sourced in bash" >&2