[VL] Remove compile option ENABLE_EP_CACHE (#8350)

apache · Dec 26, 2024 · 2b99fff · 2b99fff
1 parent 547f8b7
commit 2b99fff
Show file tree

Hide file tree

Showing 8 changed files with 9 additions and 62 deletions.
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
@@ -22,7 +22,6 @@ ENABLE_GCS=OFF
 ENABLE_S3=OFF
 ENABLE_HDFS=OFF
 ENABLE_ABFS=OFF
-ENABLE_EP_CACHE=OFF
 ENABLE_VCPKG=OFF
 RUN_SETUP_SCRIPT=ON
 VELOX_REPO=""
@@ -96,10 +95,6 @@ do
         ENABLE_ABFS=("${arg#*=}")
         shift # Remove argument name from processing
         ;;
-        --enable_ep_cache=*)
-        ENABLE_EP_CACHE=("${arg#*=}")
-        shift # Remove argument name from processing
-        ;;
         --enable_vcpkg=*)
         ENABLE_VCPKG=("${arg#*=}")
         shift # Remove argument name from processing
@@ -192,7 +187,7 @@ function build_velox {
   cd $GLUTEN_DIR/ep/build-velox/src
   # When BUILD_TESTS is on for gluten cpp, we need turn on VELOX_BUILD_TEST_UTILS via build_test_utils.
   ./build_velox.sh --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \
-                   --enable_abfs=$ENABLE_ABFS --enable_ep_cache=$ENABLE_EP_CACHE --build_test_utils=$BUILD_TESTS \
+                   --enable_abfs=$ENABLE_ABFS --build_test_utils=$BUILD_TESTS \
                    --build_tests=$BUILD_VELOX_TESTS --build_benchmarks=$BUILD_VELOX_BENCHMARKS --num_threads=$NUM_THREADS \
                    --velox_home=$VELOX_HOME
 }

diff --git a/dev/ci-velox-buildshared-centos-8.sh b/dev/ci-velox-buildshared-centos-8.sh
@@ -3,5 +3,5 @@
 set -e
 
 source /opt/rh/gcc-toolset-11/enable
-./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --enable_ep_cache=OFF --build_tests=ON \
+./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --build_tests=ON \
     --build_examples=ON --build_benchmarks=ON
diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md
@@ -61,11 +61,11 @@ First time build for all supported spark versions.
 ./dev/buildbundle-veloxbe.sh
 ```
 
-After a complete build, if only some gluten code is changed, you can use the following command to skip building velox/arrow and
+After a complete build, if only some gluten code is changed, you can use the following command to skip building arrow and
 setting up build dependencies.
 
 ```bash
-./dev/buildbundle-veloxbe.sh --enable_ep_cache=ON --build_arrow=OFF --run_setup_script=OFF
+./dev/buildbundle-veloxbe.sh --build_arrow=OFF --run_setup_script=OFF
 ```
 
 **For aarch64 build**

diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md
@@ -22,7 +22,6 @@ Please set them via `--`, e.g. `--build_type=Release`.
 | enable_gcs             | Build with GCS support.                                                                            | OFF     |
 | enable_hdfs            | Build with HDFS support.                                                                           | OFF     |
 | enable_abfs            | Build with ABFS support.                                                                           | OFF     |
-| enable_ep_cache        | Enable caching for external project build (Velox).                                                 | OFF     |
 | enable_vcpkg           | Enable vcpkg for static build.                                                                     | OFF     |
 | run_setup_script       | Run setup script to install Velox dependencies.                                                    | ON      |
 | velox_repo             | Specify your own Velox repo to build.                                                              | ""      |
@@ -45,7 +44,6 @@ Please set them via `--`, e.g., `--velox_home=/YOUR/PATH`.
 | enable_hdfs      | Build Velox with HDFS support.                                | OFF                                      |
 | enable_abfs      | Build Velox with ABFS support.                                | OFF                                      |
 | run_setup_script | Run setup script to install Velox dependencies before build.  | ON                                       |
-| enable_ep_cache  | Enable and reuse cache of Velox build.                        | OFF                                      |
 | build_test_utils | Build Velox with cmake arg -DVELOX_BUILD_TEST_UTILS=ON if ON. | OFF                                      |
 | build_tests      | Build Velox test.                                             | OFF                                      |
 | build_benchmarks | Build Velox benchmarks.                                       | OFF                                      |

diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh
@@ -26,7 +26,6 @@ ENABLE_HDFS=OFF
 ENABLE_ABFS=OFF
 BUILD_TYPE=release
 VELOX_HOME=""
-ENABLE_EP_CACHE=OFF
 # May be deprecated in Gluten build.
 ENABLE_BENCHMARK=OFF
 # May be deprecated in Gluten build.
@@ -65,10 +64,6 @@ for arg in "$@"; do
     BUILD_TYPE=("${arg#*=}")
     shift # Remove argument name from processing
     ;;
-  --enable_ep_cache=*)
-    ENABLE_EP_CACHE=("${arg#*=}")
-    shift # Remove argument name from processing
-    ;;
   --build_test_utils=*)
     BUILD_TEST_UTILS=("${arg#*=}")
     shift # Remove argument name from processing
@@ -167,40 +162,6 @@ function compile {
   fi
 }
 
-function get_build_summary {
-  COMMIT_HASH=$1
-  # Ideally all script arguments should be put into build summary.
-  # ENABLE_EP_CACHE is excluded. Thus, in current build with ENABLE_EP_CACHE=ON, we can use EP cache
-  # from last build with ENABLE_EP_CACHE=OFF,
-  echo "ENABLE_S3=$ENABLE_S3,ENABLE_GCS=$ENABLE_GCS,ENABLE_HDFS=$ENABLE_HDFS,ENABLE_ABFS=$ENABLE_ABFS,\
-BUILD_TYPE=$BUILD_TYPE,VELOX_HOME=$VELOX_HOME,ENABLE_BENCHMARK=$ENABLE_BENCHMARK,\
-ENABLE_TESTS=$ENABLE_TESTS,BUILD_TEST_UTILS=$BUILD_TEST_UTILS,\
-OTHER_ARGUMENTS=$OTHER_ARGUMENTS,COMMIT_HASH=$COMMIT_HASH"
-}
-
-function check_commit {
-  if [ $ENABLE_EP_CACHE == "ON" ]; then
-    if [ -f ${VELOX_HOME}/velox-build.cache ]; then
-      CACHED_BUILD_SUMMARY="$(cat ${VELOX_HOME}/velox-build.cache)"
-      if [ -n "$CACHED_BUILD_SUMMARY" ]; then
-        if [ "$TARGET_BUILD_SUMMARY" = "$CACHED_BUILD_SUMMARY" ]; then
-          echo "Velox build $TARGET_BUILD_SUMMARY was cached."
-          exit 0
-        else
-          echo "Found cached build $CACHED_BUILD_SUMMARY for Velox which is different with target build $TARGET_BUILD_SUMMARY."
-        fi
-      fi
-    fi
-  else
-    # Branch-new build requires all untracked files to be deleted. We only need the source code.
-    sudo git clean -dffx :/
-  fi
-
-  if [ -f ${VELOX_HOME}/velox-build.cache ]; then
-    rm -f ${VELOX_HOME}/velox-build.cache
-  fi
-}
-
 CURRENT_DIR=$(
   cd "$(dirname "$BASH_SOURCE")"
   pwd
@@ -220,15 +181,8 @@ echo "ENABLE_ABFS=${ENABLE_ABFS}"
 echo "BUILD_TYPE=${BUILD_TYPE}"
 
 cd ${VELOX_HOME}
-TARGET_BUILD_SUMMARY=$(get_build_summary "$(git rev-parse --verify HEAD)")
-if [ -z "$TARGET_BUILD_SUMMARY" ]; then
-  echo "Unable to parse Velox build: $TARGET_BUILD_SUMMARY."
-  exit 1
-fi
-echo "Target Velox build: $TARGET_BUILD_SUMMARY"
-
-check_commit
+# Branch-new build requires all untracked files to be deleted. We only need the source code.
+sudo git clean -dffx :/
 compile
 
 echo "Successfully built Velox from Source."
-echo $TARGET_BUILD_SUMMARY > "${VELOX_HOME}/velox-build.cache"
diff --git a/tools/gluten-te/centos/dockerfile-build b/tools/gluten-te/centos/dockerfile-build
@@ -62,7 +62,7 @@ RUN if [ "$BUILD_BACKEND_TYPE" == "velox" ]; \
       if [ "$GLUTEN_DEBUG_BUILD" == "ON" ]; then GLUTEN_BUILD_TYPE="Debug"; else GLUTEN_BUILD_TYPE="Release"; fi; \
       DEPS_INSTALL_SCRIPT="source /env.sh && bash /opt/gluten/dev/builddeps-veloxbe.sh \
                            --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON \
-                           --build_type=$GLUTEN_BUILD_TYPE --enable_ep_cache=ON"; \
+                           --build_type=$GLUTEN_BUILD_TYPE"; \
       EXTRA_MAVEN_OPTIONS="-Pspark-3.2 \
                            -Pbackends-velox \
                            -Pceleborn \

diff --git a/tools/gluten-te/ubuntu/dockerfile-build b/tools/gluten-te/ubuntu/dockerfile-build
@@ -62,7 +62,7 @@ RUN if [ "$BUILD_BACKEND_TYPE" == "velox" ]; \
       if [ "$GLUTEN_DEBUG_BUILD" == "ON" ]; then GLUTEN_BUILD_TYPE="Debug"; else GLUTEN_BUILD_TYPE="Release"; fi; \
       DEPS_INSTALL_SCRIPT="bash /opt/gluten/dev/builddeps-veloxbe.sh \
                            --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON \
-                           --build_type=$GLUTEN_BUILD_TYPE --enable_ep_cache=ON"; \
+                           --build_type=$GLUTEN_BUILD_TYPE"; \
       EXTRA_MAVEN_OPTIONS="-Pspark-3.2 \
                            -Pbackends-velox \
                            -Pceleborn \

diff --git a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/scripts/all.sh b/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/scripts/all.sh
@@ -26,7 +26,7 @@ EXTRA_MAVEN_OPTIONS="-Pspark-3.2 \
                      -Dcheckstyle.skip=true"
 
 cd /opt/gluten
-bash dev/builddeps-veloxbe.sh --build_type=Debug --enable_ep_cache=ON
+bash dev/builddeps-veloxbe.sh --build_type=Debug
 mvn clean install $EXTRA_MAVEN_OPTIONS
 
 apt-get -y -q --no-install-recommends install firefox tmux openjdk-8-source