[VL] Avoid reapplying code patches for external projects when ENABLE_…

…EP_CACHE=ON (#3916)
apache · Dec 5, 2023 · 144df96 · 144df96
1 parent 9d07d48
commit 144df96
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 27 deletions.
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
@@ -21,7 +21,6 @@ ENABLE_GCS=OFF
 ENABLE_S3=OFF
 ENABLE_HDFS=OFF
 ENABLE_EP_CACHE=OFF
-SKIP_BUILD_EP=OFF
 ARROW_ENABLE_CUSTOM_CODEC=OFF
 ENABLE_VCPKG=OFF
 
@@ -82,10 +81,6 @@ do
         ENABLE_EP_CACHE=("${arg#*=}")
         shift # Remove argument name from processing
         ;;
-        --skip_build_ep=*)
-        SKIP_BUILD_EP=("${arg#*=}")
-        shift # Remove argument name from processing
-        ;;
         --enable_vcpkg=*)
         ENABLE_VCPKG=("${arg#*=}")
         shift # Remove argument name from processing
@@ -104,12 +99,10 @@ if [ "$ENABLE_VCPKG" = "ON" ]; then
 fi
 
 ##install velox
-if [ "$SKIP_BUILD_EP" != "ON" ]; then
-    cd $GLUTEN_DIR/ep/build-velox/src
-    ./get_velox.sh --enable_hdfs=$ENABLE_HDFS --build_protobuf=$BUILD_PROTOBUF --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS
-    ./build_velox.sh --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \
-                   --enable_ep_cache=$ENABLE_EP_CACHE --build_tests=$BUILD_TESTS --build_benchmarks=$BUILD_BENCHMARKS
-fi
+cd $GLUTEN_DIR/ep/build-velox/src
+./get_velox.sh --enable_hdfs=$ENABLE_HDFS --build_protobuf=$BUILD_PROTOBUF --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS
+./build_velox.sh --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \
+               --enable_ep_cache=$ENABLE_EP_CACHE --build_tests=$BUILD_TESTS --build_benchmarks=$BUILD_BENCHMARKS
 
 ## compile gluten cpp
 cd $GLUTEN_DIR/cpp

diff --git a/docs/get-started/GlutenUsage.md b/docs/get-started/GlutenUsage.md
@@ -22,7 +22,6 @@ Please set them via `--`, e.g. `--build_type=Release`.
 | enable_gcs       | build with gcs lib                                  | OFF           |
 | enable_hdfs      | build with hdfs lib                                 | OFF           |
 | enable_ep_cache  | enable caching for external project build (Velox)   | OFF           |
-| skip_build_ep    | skip the build of external projects (velox)         | OFF           |
 | enable_vcpkg     | enable vcpkg for static build                       | OFF           |
 
 ## Parameters for build_velox.sh

diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md
@@ -55,7 +55,7 @@ cd /path/to/gluten
 
 ## After a complete build, if you need to re-build the project and only some gluten code is changed,
 ## you can use the following command to skip building velox and protobuf.
-# ./dev/buildbundle-veloxbe.sh --skip_build_ep=ON --build_protobuf=OFF
+# ./dev/buildbundle-veloxbe.sh --enable_ep_cache=ON --build_protobuf=OFF
 ```
 
 **For aarch64 build:**

diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh
@@ -76,19 +76,6 @@ for arg in "$@"; do
   esac
 done
 
-function apply_compilation_fixes {
-  current_dir=$1
-  velox_home=$2
-  sudo cp ${current_dir}/modify_velox.patch ${velox_home}/
-  sudo cp ${current_dir}/modify_arrow.patch ${velox_home}/third_party/
-  cd ${velox_home}
-  git apply modify_velox.patch
-  if [ $? -ne 0 ]; then
-    echo "Failed to apply compilation fixes to Velox: $?."
-    exit 1
-  fi
-}
-
 function compile {
   TARGET_BUILD_COMMIT=$(git rev-parse --verify HEAD)
 
@@ -171,6 +158,7 @@ function check_commit {
       fi
     fi
   else
+    # Branch-new build requires all untracked files to be deleted. We only need the source code.
     git clean -dffx :/
   fi
 
@@ -265,7 +253,6 @@ fi
 echo "Target Velox commit: $TARGET_BUILD_COMMIT"
 
 check_commit
-apply_compilation_fixes $CURRENT_DIR $VELOX_HOME
 compile
 
 echo "Successfully built Velox from Source."

diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
@@ -205,6 +205,22 @@ fi
 git submodule sync --recursive
 git submodule update --init --recursive
 
+function apply_compilation_fixes {
+  current_dir=$1
+  velox_home=$2
+  sudo cp ${current_dir}/modify_velox.patch ${velox_home}/
+  sudo cp ${current_dir}/modify_arrow.patch ${velox_home}/third_party/
+  git add ${velox_home}/modify_velox.patch # to avoid the file from being deleted by git clean -dffx :/
+  git add ${velox_home}/third_party/modify_arrow.patch # to avoid the file from being deleted by git clean -dffx :/
+  cd ${velox_home}
+  echo "Applying patch to Velox source code..."
+  git apply modify_velox.patch
+  if [ $? -ne 0 ]; then
+    echo "Failed to apply compilation fixes to Velox: $?."
+    exit 1
+  fi
+}
+
 function setup_linux {
   local LINUX_DISTRIBUTION=$(. /etc/os-release && echo ${ID})
   local LINUX_VERSION_ID=$(. /etc/os-release && echo ${VERSION_ID})
@@ -268,4 +284,6 @@ else
   exit 1
 fi
 
+apply_compilation_fixes $CURRENT_DIR $VELOX_SOURCE_DIR
+
 echo "Velox-get finished."