diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index f2db425f37e1..4a2595238092 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -63,6 +63,7 @@ jobs: simdjson_SOURCE: BUNDLED xsimd_SOURCE: BUNDLED Arrow_SOURCE: AUTO + stemmer_SOURCE: AUTO CUDA_VERSION: "12.4" steps: - uses: actions/checkout@v4 diff --git a/CMake/Findstemmer.cmake b/CMake/Findstemmer.cmake new file mode 100644 index 000000000000..8796ce461ac5 --- /dev/null +++ b/CMake/Findstemmer.cmake @@ -0,0 +1,30 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +find_library(STEMMER_LIB libstemmer.a) +if("${STEMMER_LIB}" STREQUAL "STEMMER_LIB-NOTFOUND") + set(stemmer_FOUND false) + return() +endif() + +set(stemmer_FOUND true) +if(NOT TARGET stemmer::stemmer) + add_library(stemmer::stemmer STATIC IMPORTED GLOBAL) + + find_path(STEMMER_INCLUDE_PATH libstemmer.h) + set_target_properties( + stemmer::stemmer + PROPERTIES IMPORTED_LOCATION ${STEMMER_LIB} INTERFACE_INCLUDE_DIRECTORIES + ${STEMMER_INCLUDE_PATH}) +endif() diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f7cbc92f250..421de103f9ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -569,7 +569,7 @@ endif() set_source(xsimd) resolve_dependency(xsimd 10.0.0) -set(stemmer_SOURCE BUNDLED) +set_source(stemmer) resolve_dependency(stemmer) if(VELOX_BUILD_TESTING) diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh index 825d2d29e32a..1583334cd860 100755 --- a/scripts/setup-centos9.sh +++ b/scripts/setup-centos9.sh @@ -41,6 +41,7 @@ FB_OS_VERSION="v2024.05.20.00" FMT_VERSION="10.1.1" BOOST_VERSION="boost-1.84.0" ARROW_VERSION="15.0.0" +STEMMER_VERSION="2.2.0" function dnf_install { dnf install -y -q --setopt=install_weak_deps=False "$@" @@ -175,6 +176,20 @@ function install_duckdb { fi } +function install_stemmer { + wget_and_untar https://snowballstem.org/dist/libstemmer_c-${STEMMER_VERSION}.tar.gz stemmer + pushd $SCRIPTDIR/../CMake/resolve_dependency_modules/libstemmer + PATCH_DIR=$(pwd) + popd + ( + cd stemmer + git apply ${PATCH_DIR}/Makefile.patch + make clean && make "-j${NPROC}" + ${SUDO} cp libstemmer.a /usr/local/lib/ + ${SUDO} cp include/libstemmer.h /usr/local/include/ + ) +} + function install_arrow { wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow ( @@ -224,6 +239,7 @@ function install_velox_deps { run_and_time install_mvfst run_and_time install_fbthrift run_and_time install_duckdb + run_and_time install_stemmer run_and_time install_arrow } diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index e40bba1d9923..2d26bbf707a1 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -57,6 +57,7 @@ FB_OS_VERSION="v2024.05.20.00" FMT_VERSION="10.1.1" BOOST_VERSION="boost-1.84.0" ARROW_VERSION="15.0.0" +STEMMER_VERSION="2.2.0" # Install packages required for build. function install_build_prerequisites { @@ -184,6 +185,20 @@ function install_duckdb { fi } +function install_stemmer { + wget_and_untar https://snowballstem.org/dist/libstemmer_c-${STEMMER_VERSION}.tar.gz stemmer + pushd $SCRIPTDIR/../CMake/resolve_dependency_modules/libstemmer + PATCH_DIR=$(pwd) + popd + ( + cd stemmer + git apply ${PATCH_DIR}/Makefile.patch + make clean && make "-j${NPROC}" + ${SUDO} cp libstemmer.a /usr/local/lib/ + ${SUDO} cp include/libstemmer.h /usr/local/include/ + ) +} + function install_arrow { wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow ( @@ -233,6 +248,7 @@ function install_velox_deps { run_and_time install_fbthrift run_and_time install_conda run_and_time install_duckdb + run_and_time install_stemmer run_and_time install_arrow }