From 1289369cbbe70da8d8db4f17b6df0b22c64baca4 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 13 Aug 2024 20:24:09 +0800 Subject: [PATCH 01/13] Initial --- .github/workflows/docker_image.yml | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/docker_image.yml diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml new file mode 100644 index 000000000000..9ad1e1bb2f0e --- /dev/null +++ b/.github/workflows/docker_image.yml @@ -0,0 +1,33 @@ +name: Build and Push Docker Image + +on: + pull_request: + paths: + - '.github/workflows/docker_image.yml' + schedule: + - cron: '0 20 * * 0' + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + file: dev/vcpkg/docker/Dockerfile.gha + push: true + tags: apache/gluten-vcpkg-centos-7:latest From 0e11ebc373b1ae014331f24376588462f0a680fd Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 13 Aug 2024 22:14:14 +0800 Subject: [PATCH 02/13] Fix issue --- .github/workflows/docker_image.yml | 2 +- dev/ci-velox-buildstatic-centos-7.sh | 4 ---- dev/vcpkg/docker/Dockerfile.gha | 6 ++++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 9ad1e1bb2f0e..7e3993308448 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -30,4 +30,4 @@ jobs: context: . file: dev/vcpkg/docker/Dockerfile.gha push: true - tags: apache/gluten-vcpkg-centos-7:latest + tags: apache/gluten:vcpkg-centos-7 diff --git a/dev/ci-velox-buildstatic-centos-7.sh b/dev/ci-velox-buildstatic-centos-7.sh index 50f8ee06763e..76c7f48b718b 100755 --- a/dev/ci-velox-buildstatic-centos-7.sh +++ b/dev/ci-velox-buildstatic-centos-7.sh @@ -2,12 +2,8 @@ set -e -yum install sudo patch java-1.8.0-openjdk-devel -y -cd $GITHUB_WORKSPACE/ep/build-velox/src -./get_velox.sh source /opt/rh/devtoolset-9/enable cd $GITHUB_WORKSPACE/ source ./dev/vcpkg/env.sh -sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt export NUM_THREADS=4 ./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON diff --git a/dev/vcpkg/docker/Dockerfile.gha b/dev/vcpkg/docker/Dockerfile.gha index c54725f043b8..75976481483f 100644 --- a/dev/vcpkg/docker/Dockerfile.gha +++ b/dev/vcpkg/docker/Dockerfile.gha @@ -11,7 +11,7 @@ RUN sed -i \ -e 's/mirror\.centos\.org/vault.centos.org/' \ /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo -RUN yum install -y git patch wget sudo +RUN yum install -y git patch wget sudo java-1.8.0-openjdk-devel # build RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten @@ -22,4 +22,6 @@ RUN cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh # vcpkg env ENV VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite -RUN source /opt/rh/devtoolset-9/enable && cd /opt/gluten && source dev/vcpkg/env.sh + +RUN cd /opt/gluten && source ./dev/vcpkg/env.sh && \ + bash ./dev/builddeps-veloxbe.sh --build_arrow=ON From 24db5943de9f9a9c46cfec528bf1250919aad1e0 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 14 Aug 2024 10:33:43 +0800 Subject: [PATCH 03/13] Change secret name --- .github/workflows/docker_image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 7e3993308448..f2a5613157ae 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -21,8 +21,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push Docker image uses: docker/build-push-action@v2 From b099023fb9ccdc5dbef3c8288a6cb5851e5de5e8 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 14 Aug 2024 10:38:17 +0800 Subject: [PATCH 04/13] Only build arrow --- dev/vcpkg/docker/Dockerfile.gha | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/vcpkg/docker/Dockerfile.gha b/dev/vcpkg/docker/Dockerfile.gha index 75976481483f..8ba6b525d074 100644 --- a/dev/vcpkg/docker/Dockerfile.gha +++ b/dev/vcpkg/docker/Dockerfile.gha @@ -23,5 +23,5 @@ RUN cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh # vcpkg env ENV VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite -RUN cd /opt/gluten && source ./dev/vcpkg/env.sh && \ - bash ./dev/builddeps-veloxbe.sh --build_arrow=ON +# Build arrow, then install the native libs to system paths and jar package to .m2/ directory. +RUN cd /opt/gluten && source ./dev/vcpkg/env.sh && bash ./dev/builddeps-veloxbe.sh build_arrow From ffd33fb92786281d025a734255418af73ec3b8f5 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 14 Aug 2024 10:40:39 +0800 Subject: [PATCH 05/13] Port glog fix, will be reverted --- dev/vcpkg/ports/gflags/portfile.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/vcpkg/ports/gflags/portfile.cmake b/dev/vcpkg/ports/gflags/portfile.cmake index f1c22ff1aa60..5e8715c83724 100644 --- a/dev/vcpkg/ports/gflags/portfile.cmake +++ b/dev/vcpkg/ports/gflags/portfile.cmake @@ -22,6 +22,9 @@ vcpkg_configure_cmake( -DBUILD_gflags_nothreads_LIB:BOOL=OFF -DGFLAGS_USE_TARGET_NAMESPACE:BOOL=ON -DCMAKE_DEBUG_POSTFIX=d + -DGFLAGS_BUILD_STATIC_LIBS:BOOL=ON + -DGFLAGS_BUILD_SHARED_LIBS:BOOL=ON + -DGFLAGS_BUILD_gflags_LIB:BOOL=ON ) vcpkg_install_cmake() From 27f9ed6934f1b141d009654a6926bde2f7bee38e Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 14 Aug 2024 10:41:58 +0800 Subject: [PATCH 06/13] Add license --- .github/workflows/docker_image.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index f2a5613157ae..7eecfdecc58d 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Build and Push Docker Image on: From b383244a180188c152de4aa8bf262ef5ed66ff5b Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 14 Aug 2024 10:53:31 +0800 Subject: [PATCH 07/13] Remove _ep --- dev/vcpkg/docker/Dockerfile.gha | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/vcpkg/docker/Dockerfile.gha b/dev/vcpkg/docker/Dockerfile.gha index 8ba6b525d074..e4ffe889bb0d 100644 --- a/dev/vcpkg/docker/Dockerfile.gha +++ b/dev/vcpkg/docker/Dockerfile.gha @@ -24,4 +24,5 @@ RUN cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh ENV VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite # Build arrow, then install the native libs to system paths and jar package to .m2/ directory. -RUN cd /opt/gluten && source ./dev/vcpkg/env.sh && bash ./dev/builddeps-veloxbe.sh build_arrow +RUN cd /opt/gluten && source ./dev/vcpkg/env.sh && bash ./dev/builddeps-veloxbe.sh build_arrow && \ + rm -rf ep/_ep/ && rm -rf /tmp/velox-deps/ From 3ba2febfae1088e1864af3ead83d38ff5bb39afd Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 14 Aug 2024 13:43:44 +0800 Subject: [PATCH 08/13] Rename dockerfile --- .github/workflows/docker_image.yml | 2 +- dev/vcpkg/Makefile | 2 +- dev/vcpkg/docker/{Dockerfile.gha => gha-centos-7.dockerfile} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename dev/vcpkg/docker/{Dockerfile.gha => gha-centos-7.dockerfile} (100%) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 7eecfdecc58d..f93dda153927 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -43,6 +43,6 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: dev/vcpkg/docker/Dockerfile.gha + file: dev/vcpkg/docker/gha-centos-7.dockerfile push: true tags: apache/gluten:vcpkg-centos-7 diff --git a/dev/vcpkg/Makefile b/dev/vcpkg/Makefile index 874b0b579896..15bcab60e023 100644 --- a/dev/vcpkg/Makefile +++ b/dev/vcpkg/Makefile @@ -31,7 +31,7 @@ docker-image: docker-image-gha: docker build \ - --file docker/Dockerfile.gha \ + --file docker/gha-centos-7.dockerfile \ --tag "$(DOCKER_IMAGE)" \ --build-arg HTTPS_PROXY="" \ --build-arg HTTP_PROXY="" \ diff --git a/dev/vcpkg/docker/Dockerfile.gha b/dev/vcpkg/docker/gha-centos-7.dockerfile similarity index 100% rename from dev/vcpkg/docker/Dockerfile.gha rename to dev/vcpkg/docker/gha-centos-7.dockerfile From 9de4b1034879699af358cb308df6bc8704569b9d Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 14 Aug 2024 15:00:38 +0800 Subject: [PATCH 09/13] Use the new docker and disable arrow build --- .github/workflows/velox_docker.yml | 3 +-- .github/workflows/velox_docker_cache.yml | 4 +--- dev/ci-velox-buildstatic-centos-7.sh | 3 ++- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 866ea48f18a3..b6e7e50a6325 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -51,7 +51,7 @@ concurrency: jobs: build-native-lib-centos-7: runs-on: ubuntu-20.04 - container: apache/gluten:gluten-vcpkg-builder_2024_08_05 # centos7 with dependencies installed + container: apache/gluten:vcpkg-centos-7 steps: - uses: actions/checkout@v2 - name: Generate cache key @@ -63,7 +63,6 @@ jobs: with: path: | ./cpp/build/releases/ - /root/.m2/repository/org/apache/arrow/ key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }} - name: Build Gluten native libraries if: ${{ steps.cache.outputs.cache-hit != 'true' }} diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml index 3fc0fc50f847..1aca4d6bd850 100644 --- a/.github/workflows/velox_docker_cache.yml +++ b/.github/workflows/velox_docker_cache.yml @@ -30,7 +30,7 @@ concurrency: jobs: cache-native-lib-centos-7: runs-on: ubuntu-20.04 - container: apache/gluten:gluten-vcpkg-builder_2024_08_05 # centos7 with dependencies installed + container: apache/gluten:vcpkg-centos-7 steps: - uses: actions/checkout@v2 - name: Generate cache key @@ -43,7 +43,6 @@ jobs: lookup-only: true path: | ./cpp/build/releases/ - /root/.m2/repository/org/apache/arrow/ key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }} - name: Build Gluten native libraries if: steps.check-cache.outputs.cache-hit != 'true' @@ -57,7 +56,6 @@ jobs: with: path: | ./cpp/build/releases/ - /root/.m2/repository/org/apache/arrow/ key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }} cache-native-lib-centos-8: diff --git a/dev/ci-velox-buildstatic-centos-7.sh b/dev/ci-velox-buildstatic-centos-7.sh index 76c7f48b718b..d83d443fc489 100755 --- a/dev/ci-velox-buildstatic-centos-7.sh +++ b/dev/ci-velox-buildstatic-centos-7.sh @@ -6,4 +6,5 @@ source /opt/rh/devtoolset-9/enable cd $GITHUB_WORKSPACE/ source ./dev/vcpkg/env.sh export NUM_THREADS=4 -./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON +./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --build_arrow=OFF --enable_s3=ON \ + --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON From ec31730e38795ecce34d8de75b66d85e2a1f3f4b Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 14 Aug 2024 20:44:12 +0800 Subject: [PATCH 10/13] make vcpkg cache path --- dev/vcpkg/docker/gha-centos-7.dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dev/vcpkg/docker/gha-centos-7.dockerfile b/dev/vcpkg/docker/gha-centos-7.dockerfile index e4ffe889bb0d..fb782b20e02c 100644 --- a/dev/vcpkg/docker/gha-centos-7.dockerfile +++ b/dev/vcpkg/docker/gha-centos-7.dockerfile @@ -13,14 +13,16 @@ RUN sed -i \ RUN yum install -y git patch wget sudo java-1.8.0-openjdk-devel -# build RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten RUN echo "check_certificate = off" >> ~/.wgetrc -# deps + RUN cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh -# vcpkg env +# An actual path used for vcpkg cache. +RUN mkdir -p /var/cache/vcpkg + +# Set vcpkg cache path. ENV VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite # Build arrow, then install the native libs to system paths and jar package to .m2/ directory. From 08ebc1b615002c04b36c90a07aeae029ced72192 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 14 Aug 2024 23:26:24 +0800 Subject: [PATCH 11/13] Fix vcpkg caching --- .github/workflows/build_bundle_package.yml | 1 - dev/vcpkg/docker/gha-centos-7.dockerfile | 4 ++-- dev/vcpkg/env.sh | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_bundle_package.yml b/.github/workflows/build_bundle_package.yml index 950f07ce21a8..cf70eeb1ac1a 100644 --- a/.github/workflows/build_bundle_package.yml +++ b/.github/workflows/build_bundle_package.yml @@ -52,7 +52,6 @@ jobs: source /opt/rh/devtoolset-9/enable && \ source /opt/gluten//dev/vcpkg/env.sh && \ cd $GITHUB_WORKSPACE/ && \ - sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt && \ export NUM_THREADS=4 ./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --enable_s3=OFF \ --enable_gcs=OFF --enable_hdfs=ON --enable_abfs=OFF diff --git a/dev/vcpkg/docker/gha-centos-7.dockerfile b/dev/vcpkg/docker/gha-centos-7.dockerfile index fb782b20e02c..d37c449cb1ff 100644 --- a/dev/vcpkg/docker/gha-centos-7.dockerfile +++ b/dev/vcpkg/docker/gha-centos-7.dockerfile @@ -26,5 +26,5 @@ RUN mkdir -p /var/cache/vcpkg ENV VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite # Build arrow, then install the native libs to system paths and jar package to .m2/ directory. -RUN cd /opt/gluten && source ./dev/vcpkg/env.sh && bash ./dev/builddeps-veloxbe.sh build_arrow && \ - rm -rf ep/_ep/ && rm -rf /tmp/velox-deps/ +RUN cd /opt/gluten && source /opt/rh/devtoolset-9/enable && source ./dev/vcpkg/env.sh && \ + bash ./dev/builddeps-veloxbe.sh build_arrow && rm -rf /opt/gluten diff --git a/dev/vcpkg/env.sh b/dev/vcpkg/env.sh index cf3126f0a1b9..8b247a907a05 100755 --- a/dev/vcpkg/env.sh +++ b/dev/vcpkg/env.sh @@ -1,4 +1,5 @@ #! /bin/bash +set -e if [ -z "${BASH_SOURCE[0]}" ] || [ "$0" == "${BASH_SOURCE[0]}" ]; then echo "env.sh should only be sourced in bash" >&2 From 47bf4e6030185c500cd50c6a973929b0e8fc07e6 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 15 Aug 2024 17:22:32 +0800 Subject: [PATCH 12/13] Revert "Port glog fix, will be reverted" This reverts commit ffd33fb92786281d025a734255418af73ec3b8f5. --- dev/vcpkg/ports/gflags/portfile.cmake | 3 --- 1 file changed, 3 deletions(-) diff --git a/dev/vcpkg/ports/gflags/portfile.cmake b/dev/vcpkg/ports/gflags/portfile.cmake index 5e8715c83724..f1c22ff1aa60 100644 --- a/dev/vcpkg/ports/gflags/portfile.cmake +++ b/dev/vcpkg/ports/gflags/portfile.cmake @@ -22,9 +22,6 @@ vcpkg_configure_cmake( -DBUILD_gflags_nothreads_LIB:BOOL=OFF -DGFLAGS_USE_TARGET_NAMESPACE:BOOL=ON -DCMAKE_DEBUG_POSTFIX=d - -DGFLAGS_BUILD_STATIC_LIBS:BOOL=ON - -DGFLAGS_BUILD_SHARED_LIBS:BOOL=ON - -DGFLAGS_BUILD_gflags_LIB:BOOL=ON ) vcpkg_install_cmake() From 824f54f7dacf6de0636e4b89800c3fbfb1f102ca Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 15 Aug 2024 21:10:53 +0800 Subject: [PATCH 13/13] Revert "Rename dockerfile" This reverts commit 3ba2febfae1088e1864af3ead83d38ff5bb39afd. --- .github/workflows/docker_image.yml | 2 +- dev/vcpkg/Makefile | 2 +- dev/vcpkg/docker/{gha-centos-7.dockerfile => Dockerfile.gha} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename dev/vcpkg/docker/{gha-centos-7.dockerfile => Dockerfile.gha} (100%) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index f93dda153927..7eecfdecc58d 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -43,6 +43,6 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: dev/vcpkg/docker/gha-centos-7.dockerfile + file: dev/vcpkg/docker/Dockerfile.gha push: true tags: apache/gluten:vcpkg-centos-7 diff --git a/dev/vcpkg/Makefile b/dev/vcpkg/Makefile index 15bcab60e023..874b0b579896 100644 --- a/dev/vcpkg/Makefile +++ b/dev/vcpkg/Makefile @@ -31,7 +31,7 @@ docker-image: docker-image-gha: docker build \ - --file docker/gha-centos-7.dockerfile \ + --file docker/Dockerfile.gha \ --tag "$(DOCKER_IMAGE)" \ --build-arg HTTPS_PROXY="" \ --build-arg HTTP_PROXY="" \ diff --git a/dev/vcpkg/docker/gha-centos-7.dockerfile b/dev/vcpkg/docker/Dockerfile.gha similarity index 100% rename from dev/vcpkg/docker/gha-centos-7.dockerfile rename to dev/vcpkg/docker/Dockerfile.gha