[NSE-433]Support pre-built Jemalloc (#439)

* [NSE-433]Support pre-built Jemalloc * Update default value for BUILD_JEMALLOC
oap-project · Aug 11, 2021 · 9a3f58f · 9a3f58f
1 parent 5eec759
commit 9a3f58f
Show file tree

Hide file tree

Showing 4 changed files with 89 additions and 1 deletion.
diff --git a/native-sql-engine/core/pom.xml b/native-sql-engine/core/pom.xml
@@ -43,6 +43,7 @@
     <nativesql.arrow.bfs.install.dir>${project.basedir}/../../arrow-data-source/script/build/arrow_install</nativesql.arrow.bfs.install.dir>
     <nativesql.arrow_root>${arrow_root}</nativesql.arrow_root>
     <nativesql.build_protobuf>${build_protobuf}</nativesql.build_protobuf>
+    <nativesql.build_jemalloc>${build_jemalloc}</nativesql.build_jemalloc>
   </properties>
   <dependencies>
     <!-- Prevent our dummy JAR from being included in Spark distributions or uploaded to YARN -->
@@ -333,6 +334,7 @@
                           <argument>${nativesql.build_protobuf}</argument>
                           <argument>${nativesql.arrow_root}</argument>
                           <argument>${nativesql.arrow.bfs.install.dir}</argument>
+                          <argument>${nativesql.build_jemalloc}</argument>
                       </arguments>
                   </configuration>
               </execution>

diff --git a/native-sql-engine/cpp/compile.sh b/native-sql-engine/cpp/compile.sh
@@ -8,6 +8,7 @@ STATIC_ARROW=${3:-OFF}
 BUILD_PROTOBUF=${4:-ON}
 ARROW_ROOT=${5:-/usr/local}
 ARROW_BFS_INSTALL_DIR=${6}
+BUILD_JEMALLOC=${7:-ON}
 
 echo "CMAKE Arguments:"
 echo "TESTS=${TESTS}"
@@ -16,6 +17,7 @@ echo "STATIC_ARROW=${STATIC_ARROW}"
 echo "BUILD_PROTOBUF=${BUILD_PROTOBUF}"
 echo "ARROW_ROOT=${ARROW_ROOT}"
 echo "ARROW_BUILD_FROM_SOURCE_INSTALL_DIR=${ARROW_BFS_INSTALL_DIR}"
+echo "BUILD_JEMALLOC=${BUILD_JEMALLOC}"
 
 CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
 echo $CURRENT_DIR
@@ -26,7 +28,11 @@ if [ -d build ]; then
 fi
 mkdir build
 cd build
-cmake .. -DTESTS=${TESTS} -DBUILD_ARROW=${BUILD_ARROW} -DSTATIC_ARROW=${STATIC_ARROW} -DBUILD_PROTOBUF=${BUILD_PROTOBUF} -DARROW_ROOT=${ARROW_ROOT} -DARROW_BFS_INSTALL_DIR=${ARROW_BFS_INSTALL_DIR}
+cmake .. -DTESTS=${TESTS} -DBUILD_ARROW=${BUILD_ARROW} -DSTATIC_ARROW=${STATIC_ARROW} -DBUILD_PROTOBUF=${BUILD_PROTOBUF} -DARROW_ROOT=${ARROW_ROOT} -DARROW_BFS_INSTALL_DIR=${ARROW_BFS_INSTALL_DIR} -DBUILD_JEMALLOC=${BUILD_JEMALLOC}
+make -j2
+
+set +eu
+
 make -j2
 
 set +eu

diff --git a/native-sql-engine/cpp/src/CMakeLists.txt b/native-sql-engine/cpp/src/CMakeLists.txt
@@ -25,12 +25,15 @@ option(BUILD_PROTOBUF "Build Protobuf from Source" ON)
 option(USE_AVX512 "Build with AVX-512 optimizations" OFF)
 option(TESTS "Build the tests" OFF)
 option(BENCHMARKS "Build the benchmarks" OFF)
+option(BUILD_JEMALLOC "Build Jemalloc from Source" OFF)
 option(DEBUG "Enable Debug Info" OFF)
 
 set(BOOST_MIN_VERSION "1.42.0")
 find_package(Boost REQUIRED)
 INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS})
 
+set(JEMALLOC_BUILD_VERSION "5.2.1")
+
 find_package(JNI REQUIRED)
 set(source_root_directory ${CMAKE_CURRENT_SOURCE_DIR})
 
@@ -40,6 +43,9 @@ if(CCACHE_FOUND)
   set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
 endif(CCACHE_FOUND)
 
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+
 # Building Protobuf
 macro(build_protobuf)
   message(STATUS "Building Protocol Buffers from Source")
@@ -332,6 +338,70 @@ macro(find_arrow)
 
 endmacro()
 
+# Building Jemalloc
+macro(build_jemalloc)
+  message(STATUS "Building Jemalloc from Source")
+  set (JEMALLOC_SOURCE_URL
+       "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_BUILD_VERSION}/jemalloc-${JEMALLOC_BUILD_VERSION}.tar.bz2"
+       "https://github.com/ursa-labs/thirdparty/releases/download/latest/jemalloc-${JEMALLOC_BUILD_VERSION}.tar.bz2"
+  )
+  set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-install")
+  set(JEMALLOC_LIB_DIR "${JEMALLOC_PREFIX}/lib")
+  set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
+  set(
+    JEMALLOC_STATIC_LIB
+    "${JEMALLOC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}jemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}"
+    )
+  set(
+    JEMALLOC_INCLUDE
+    "${JEMALLOC_PREFIX}/include"
+    )
+  set(JEMALLOC_CONFIGURE_ARGS
+      "AR=${CMAKE_AR}"
+      "CC=${CMAKE_C_COMPILER}"
+      "--prefix=${JEMALLOC_PREFIX}"
+      "--libdir=${JEMALLOC_LIB_DIR}"
+      "--with-jemalloc-prefix=je_gazelle_"
+      "--with-private-namespace=je_gazelle_private_"
+      "--without-export"
+      "--disable-shared"
+      "--disable-cxx"
+      "--disable-libdl"
+      "--disable-initial-exec-tls"
+      "CFLAGS=-fPIC"
+      "CXXFLAGS=-fPIC")
+  set(JEMALLOC_BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS})
+  ExternalProject_Add(jemalloc_ep
+                      URL ${JEMALLOC_SOURCE_URL}
+                      PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html
+                      CONFIGURE_COMMAND "./configure" ${JEMALLOC_CONFIGURE_ARGS}
+                      BUILD_COMMAND ${JEMALLOC_BUILD_COMMAND}
+                      BUILD_IN_SOURCE 1
+                      BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}"
+                      INSTALL_COMMAND make install)
+
+  file(MAKE_DIRECTORY "${JEMALLOC_INCLUDE_DIR}")
+  add_library(jemalloc::libjemalloc STATIC IMPORTED)
+  set_target_properties(
+    jemalloc::libjemalloc
+    PROPERTIES INTERFACE_LINK_LIBRARIES Threads::Threads
+               IMPORTED_LOCATION "${JEMALLOC_STATIC_LIB}"
+               INTERFACE_INCLUDE_DIRECTORIES
+               "${JEMALLOC_INCLUDE_DIR}")
+  add_dependencies(jemalloc::libjemalloc protobuf_ep)
+endmacro()
+
+# Find Jemalloc
+macro(find_jemalloc)
+  # Find the existing Protobuf
+  set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
+  find_package(jemalloc_pic)
+  if ("${Jemalloc_LIBRARY}" STREQUAL "Jemalloc_LIBRARY-NOTFOUND")
+    message(FATAL_ERROR "Jemalloc Library Not Found")
+  endif()
+  set(PROTOC_BIN ${Jemalloc_PROTOC_EXECUTABLE})
+endmacro()
+
 # Set up Proto
 file(MAKE_DIRECTORY ${root_directory}/src/proto)
 set(PROTO_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/proto")
@@ -495,6 +565,15 @@ else() #
   message(STATUS "Use existing ARROW libraries")
 endif()
 
+# Build Jemalloc
+if(BUILD_JEMALLOC)
+  build_jemalloc(${STATIC_JEMALLOC})
+  message(STATUS "Building Jemalloc: ${STATIC_JEMALLOC}")
+else() #
+  find_jemalloc()
+  message(STATUS "Use existing Jemalloc libraries")
+endif()
+
 if(DEFINED ENV{HADOOP_HOME})
   set(LIBHDFS3_DESTINATION $ENV{HADOOP_HOME}/lib/native)
 else()

diff --git a/pom.xml b/pom.xml
@@ -126,6 +126,7 @@
     <arrow.install.dir>${arrow.script.dir}/build/arrow_install</arrow.install.dir>
     <arrow_root>/usr/local</arrow_root>
     <build_protobuf>ON</build_protobuf>
+    <build_jemalloc>ON</build_jemalloc>
     <project.prefix>spark-sql-columnar</project.prefix>
     <project.name.prefix>OAP Project Spark Columnar Plugin</project.name.prefix>
     <spark311.version>3.1.1</spark311.version>