Skip to content
This repository has been archived by the owner on Sep 18, 2023. It is now read-only.

Commit

Permalink
[NSE-451] backport master code to 1.2 branch (#454)
Browse files Browse the repository at this point in the history 
* [NSE-254]Fix redundant arrow library issue. (#452)

* [NSE-254]Fix redundant arrow library issue.

* remote the unneccessary marked code

* remove arrow-data-source-common from jar with dependency (#453)

native sql engine is bundled with arrow data source now so
it should be safe to assume those dataset related class will
be 'pre-loaded'

with this fix the jar size is ~97MB(was ~147MB) on my local env

Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>

Co-authored-by: Wei-Ting Chen <weiting.chen@intel.com>
  • Loading branch information
@zhouyuan @weiting-chen
zhouyuan and weiting-chen authored Aug 9, 2021
1 parent d4c3a21 commit d1d6658
Show file tree
Hide file tree
Showing 4 changed files with 27 additions and 36 deletions.
1 change: 1 addition & 0 deletions native-sql-engine/core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@
<artifactId>spark-arrow-datasource-common</artifactId>
<groupId>com.intel.oap</groupId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scalacheck</groupId>
Expand Down
20 changes: 18 additions & 2 deletions native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,10 @@
/** Helper class for JNI related operations. */
public class JniUtils {
private static final String LIBRARY_NAME = "spark_columnar_jni";
private static final String ARROW_LIBRARY_NAME = "libarrow.so.400";
private static final String GANDIVA_LIBRARY_NAME = "libgandiva.so.400";
private static final String ARROW_LIBRARY_NAME = "libarrow.so.400.0.0";
private static final String ARROW_PARENT_LIBRARY_NAME = "libarrow.so.400";
private static final String GANDIVA_LIBRARY_NAME = "libgandiva.so.400.0.0";
private static final String GANDIVA_PARENT_LIBRARY_NAME = "libgandiva.so.400";
private static boolean isLoaded = false;
private static boolean isCodegenDependencyLoaded = false;
private static List<String> codegenJarsLoadedCache = new ArrayList<>();
Expand Down Expand Up @@ -113,9 +115,23 @@ static void loadLibraryFromJar(String tmp_dir) throws IOException, IllegalAccess
tmp_dir = System.getProperty("java.io.tmpdir");
}
final File arrowlibraryFile = moveFileFromJarToTemp(tmp_dir, ARROW_LIBRARY_NAME);
Path arrow_target = Paths.get(arrowlibraryFile.getPath());
Path arrow_link = Paths.get(tmp_dir, ARROW_PARENT_LIBRARY_NAME);
if (Files.exists(arrow_link)) {
Files.delete(arrow_link);
}
Path symLink = Files.createSymbolicLink(arrow_link, arrow_target);
System.load(arrowlibraryFile.getAbsolutePath());

final File gandivalibraryFile = moveFileFromJarToTemp(tmp_dir, GANDIVA_LIBRARY_NAME);
Path gandiva_target = Paths.get(gandivalibraryFile.getPath());
Path gandiva_link = Paths.get(tmp_dir, GANDIVA_PARENT_LIBRARY_NAME);
if (Files.exists(gandiva_link)) {
Files.delete(gandiva_link);
}
Files.createSymbolicLink(gandiva_link, gandiva_target);
System.load(gandivalibraryFile.getAbsolutePath());

final String libraryToLoad = System.mapLibraryName(LIBRARY_NAME);
final File libraryFile = moveFileFromJarToTemp(tmp_dir, libraryToLoad);
System.load(libraryFile.getAbsolutePath());
Expand Down
37 changes: 3 additions & 34 deletions native-sql-engine/cpp/src/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ set(ARROW_ROOT "/usr/local" CACHE PATH "Arrow Root dir")
set(ARROW_BFS_INSTALL_DIR "/usr/local" CACHE PATH "Arrow Build from Source dir")
set(ARROW_LIB_NAME arrow)
set(GANDIVA_LIB_NAME gandiva)
set(ARROW_SHARED_LIBRARY_SUFFIX ".so")
set(ARROW_SHARED_LIBRARY_SUFFIX ".so.400")

option(BUILD_ARROW "Build Arrow from Source" ON)
option(STATIC_ARROW "Build Arrow with Static Libraries" OFF)
Expand Down Expand Up @@ -213,46 +213,15 @@ macro(build_arrow STATIC_ARROW)

else()

# Copy Arrow Shared Library to releases directory for package jar
ExternalProject_Add_Step(arrow_ep copy_arrow_binary
ExternalProject_Add_Step(arrow_ep copy_arrow_binary_400_0_0
COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX} ${root_directory}/releases/
COMMENT "Copy libarrow.so to releases/"
DEPENDEES mkdir download update patch configure build install java_install
WORKING_DIRECTORY "${ARROW_PREFIX}/"
)

ExternalProject_Add_Step(arrow_ep copy_arrow_binary_300
COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400 ${root_directory}/releases/
COMMENT "Copy libarrow.so.400 to releases/"
DEPENDEES mkdir download update patch configure build install java_install
WORKING_DIRECTORY "${ARROW_PREFIX}/"
)

ExternalProject_Add_Step(arrow_ep copy_arrow_binary_300_0_0
COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400.0.0 ${root_directory}/releases/
COMMENT "Copy libarrow.so.400.0.0 to releases/"
DEPENDEES mkdir download update patch configure build install java_install
WORKING_DIRECTORY "${ARROW_PREFIX}/"
)


# Copy Gandiva Shared Library to releases directory for package jar
ExternalProject_Add_Step(arrow_ep copy_gandiva_binary
ExternalProject_Add_Step(arrow_ep copy_gandiva_binary_400_0_0
COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX} ${root_directory}/releases/
COMMENT "Copy libgandiva.so to releases/"
DEPENDEES mkdir download update patch configure build install java_install
WORKING_DIRECTORY "${ARROW_PREFIX}/"
)

ExternalProject_Add_Step(arrow_ep copy_gandiva_binary_300
COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400 ${root_directory}/releases/
COMMENT "Copy libgandiva.so.400 to releases/"
DEPENDEES mkdir download update patch configure build install java_install
WORKING_DIRECTORY "${ARROW_PREFIX}/"
)

ExternalProject_Add_Step(arrow_ep copy_gandiva_binary_300_0_0
COMMAND cp -a ${ARROW_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}.400.0.0 ${root_directory}/releases/
COMMENT "Copy libgandiva.so.400.0.0 to releases/"
DEPENDEES mkdir download update patch configure build install java_install
WORKING_DIRECTORY "${ARROW_PREFIX}/"
Expand Down
5 changes: 5 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,11 @@

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.2.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
Expand Down

0 comments on commit d1d6658

Please sign in to comment.