diff --git a/native-sql-engine/core/pom.xml b/native-sql-engine/core/pom.xml
index 1e9b07a58..868bde3dc 100644
--- a/native-sql-engine/core/pom.xml
+++ b/native-sql-engine/core/pom.xml
@@ -43,6 +43,7 @@
${project.basedir}/../../arrow-data-source/script/build/arrow_install
${arrow_root}
${build_protobuf}
+ ${build_jemalloc}
@@ -333,6 +334,7 @@
${nativesql.build_protobuf}
${nativesql.arrow_root}
${nativesql.arrow.bfs.install.dir}
+ ${nativesql.build_jemalloc}
diff --git a/native-sql-engine/cpp/compile.sh b/native-sql-engine/cpp/compile.sh
index 4a7eaad33..db4d37b41 100755
--- a/native-sql-engine/cpp/compile.sh
+++ b/native-sql-engine/cpp/compile.sh
@@ -8,6 +8,7 @@ STATIC_ARROW=${3:-OFF}
BUILD_PROTOBUF=${4:-ON}
ARROW_ROOT=${5:-/usr/local}
ARROW_BFS_INSTALL_DIR=${6}
+BUILD_JEMALLOC=${7:-ON}
echo "CMAKE Arguments:"
echo "TESTS=${TESTS}"
@@ -16,6 +17,7 @@ echo "STATIC_ARROW=${STATIC_ARROW}"
echo "BUILD_PROTOBUF=${BUILD_PROTOBUF}"
echo "ARROW_ROOT=${ARROW_ROOT}"
echo "ARROW_BUILD_FROM_SOURCE_INSTALL_DIR=${ARROW_BFS_INSTALL_DIR}"
+echo "BUILD_JEMALLOC=${BUILD_JEMALLOC}"
CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
echo $CURRENT_DIR
@@ -26,7 +28,11 @@ if [ -d build ]; then
fi
mkdir build
cd build
-cmake .. -DTESTS=${TESTS} -DBUILD_ARROW=${BUILD_ARROW} -DSTATIC_ARROW=${STATIC_ARROW} -DBUILD_PROTOBUF=${BUILD_PROTOBUF} -DARROW_ROOT=${ARROW_ROOT} -DARROW_BFS_INSTALL_DIR=${ARROW_BFS_INSTALL_DIR}
+cmake .. -DTESTS=${TESTS} -DBUILD_ARROW=${BUILD_ARROW} -DSTATIC_ARROW=${STATIC_ARROW} -DBUILD_PROTOBUF=${BUILD_PROTOBUF} -DARROW_ROOT=${ARROW_ROOT} -DARROW_BFS_INSTALL_DIR=${ARROW_BFS_INSTALL_DIR} -DBUILD_JEMALLOC=${BUILD_JEMALLOC}
+make -j2
+
+set +eu
+
make -j2
set +eu
diff --git a/native-sql-engine/cpp/src/CMakeLists.txt b/native-sql-engine/cpp/src/CMakeLists.txt
index 29c170cf0..5935c6a05 100644
--- a/native-sql-engine/cpp/src/CMakeLists.txt
+++ b/native-sql-engine/cpp/src/CMakeLists.txt
@@ -25,12 +25,15 @@ option(BUILD_PROTOBUF "Build Protobuf from Source" ON)
option(USE_AVX512 "Build with AVX-512 optimizations" OFF)
option(TESTS "Build the tests" OFF)
option(BENCHMARKS "Build the benchmarks" OFF)
+option(BUILD_JEMALLOC "Build Jemalloc from Source" OFF)
option(DEBUG "Enable Debug Info" OFF)
set(BOOST_MIN_VERSION "1.42.0")
find_package(Boost REQUIRED)
INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS})
+set(JEMALLOC_BUILD_VERSION "5.2.1")
+
find_package(JNI REQUIRED)
set(source_root_directory ${CMAKE_CURRENT_SOURCE_DIR})
@@ -40,6 +43,9 @@ if(CCACHE_FOUND)
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
endif(CCACHE_FOUND)
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+
# Building Protobuf
macro(build_protobuf)
message(STATUS "Building Protocol Buffers from Source")
@@ -332,6 +338,70 @@ macro(find_arrow)
endmacro()
+# Building Jemalloc
+macro(build_jemalloc)
+ message(STATUS "Building Jemalloc from Source")
+ set (JEMALLOC_SOURCE_URL
+ "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_BUILD_VERSION}/jemalloc-${JEMALLOC_BUILD_VERSION}.tar.bz2"
+ "https://github.com/ursa-labs/thirdparty/releases/download/latest/jemalloc-${JEMALLOC_BUILD_VERSION}.tar.bz2"
+ )
+ set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-install")
+ set(JEMALLOC_LIB_DIR "${JEMALLOC_PREFIX}/lib")
+ set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
+ set(
+ JEMALLOC_STATIC_LIB
+ "${JEMALLOC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}jemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}"
+ )
+ set(
+ JEMALLOC_INCLUDE
+ "${JEMALLOC_PREFIX}/include"
+ )
+ set(JEMALLOC_CONFIGURE_ARGS
+ "AR=${CMAKE_AR}"
+ "CC=${CMAKE_C_COMPILER}"
+ "--prefix=${JEMALLOC_PREFIX}"
+ "--libdir=${JEMALLOC_LIB_DIR}"
+ "--with-jemalloc-prefix=je_gazelle_"
+ "--with-private-namespace=je_gazelle_private_"
+ "--without-export"
+ "--disable-shared"
+ "--disable-cxx"
+ "--disable-libdl"
+ "--disable-initial-exec-tls"
+ "CFLAGS=-fPIC"
+ "CXXFLAGS=-fPIC")
+ set(JEMALLOC_BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS})
+ ExternalProject_Add(jemalloc_ep
+ URL ${JEMALLOC_SOURCE_URL}
+ PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html
+ CONFIGURE_COMMAND "./configure" ${JEMALLOC_CONFIGURE_ARGS}
+ BUILD_COMMAND ${JEMALLOC_BUILD_COMMAND}
+ BUILD_IN_SOURCE 1
+ BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}"
+ INSTALL_COMMAND make install)
+
+ file(MAKE_DIRECTORY "${JEMALLOC_INCLUDE_DIR}")
+ add_library(jemalloc::libjemalloc STATIC IMPORTED)
+ set_target_properties(
+ jemalloc::libjemalloc
+ PROPERTIES INTERFACE_LINK_LIBRARIES Threads::Threads
+ IMPORTED_LOCATION "${JEMALLOC_STATIC_LIB}"
+ INTERFACE_INCLUDE_DIRECTORIES
+ "${JEMALLOC_INCLUDE_DIR}")
+ add_dependencies(jemalloc::libjemalloc protobuf_ep)
+endmacro()
+
+# Find Jemalloc
+macro(find_jemalloc)
+ # Find the existing Protobuf
+ set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
+ find_package(jemalloc_pic)
+ if ("${Jemalloc_LIBRARY}" STREQUAL "Jemalloc_LIBRARY-NOTFOUND")
+ message(FATAL_ERROR "Jemalloc Library Not Found")
+ endif()
+ set(PROTOC_BIN ${Jemalloc_PROTOC_EXECUTABLE})
+endmacro()
+
# Set up Proto
file(MAKE_DIRECTORY ${root_directory}/src/proto)
set(PROTO_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/proto")
@@ -495,6 +565,15 @@ else() #
message(STATUS "Use existing ARROW libraries")
endif()
+# Build Jemalloc
+if(BUILD_JEMALLOC)
+ build_jemalloc(${STATIC_JEMALLOC})
+ message(STATUS "Building Jemalloc: ${STATIC_JEMALLOC}")
+else() #
+ find_jemalloc()
+ message(STATUS "Use existing Jemalloc libraries")
+endif()
+
if(DEFINED ENV{HADOOP_HOME})
set(LIBHDFS3_DESTINATION $ENV{HADOOP_HOME}/lib/native)
else()
diff --git a/pom.xml b/pom.xml
index 1fefadcdd..d6a862b56 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,6 +126,7 @@
${arrow.script.dir}/build/arrow_install
/usr/local
ON
+ ON
spark-sql-columnar
OAP Project Spark Columnar Plugin
3.1.1