Skip to content
This repository was archived by the owner on Nov 17, 2023. It is now read-only.

[WIP] Add SOVERSION when build shared libmxnet.so library #17815

Merged
merged 6 commits into from
Aug 7, 2020
Merged

[WIP] Add SOVERSION when build shared libmxnet.so library #17815

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b055ae3
Add SOVERSION when build shared libmxnet.so library
sl1pkn07 Mar 11, 2020
135964a
Merge remote-tracking branch 'origin/master' into soname_soversion
leezu Jul 27, 2020
d2659e2
Resolve remaining merge conflicts
leezu Jul 28, 2020
2e4187d
Merge branch 'master' into soname_soversion
leezu Aug 6, 2020
598be31
Delete unnecessary files
leezu Aug 6, 2020
3e2f02b
Remove duplicate project()
leezu Aug 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
cmake_minimum_required(VERSION 3.13)

file(READ "include/mxnet/base.h" ver)
string(REGEX MATCH "MXNET_MAJOR ([0-9]*)" _ ${ver})
set(ver_major ${CMAKE_MATCH_1})
string(REGEX MATCH "MXNET_MINOR ([0-9]*)" _ ${ver})
set(ver_minor ${CMAKE_MATCH_1})
string(REGEX MATCH "MXNET_PATCH ([0-9]*)" _ ${ver})
set(ver_patch ${CMAKE_MATCH_1})

# workaround to store CMAKE_CROSSCOMPILING because is getting reset by the project command
if(CMAKE_CROSSCOMPILING)
set(__CMAKE_CROSSCOMPILING ${CMAKE_CROSSCOMPILING})
set(__CMAKE_CROSSCOMPILING_OVERRIDE ON)
endif()

project(mxnet C CXX)
project(mxnet VERSION ${ver_major}.${ver_minor}.${ver_patch} LANGUAGES C CXX)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS ON) # GNU extensions used by src/operator/random/shuffle_op.cc
Expand Down Expand Up @@ -103,8 +111,6 @@ if(USE_TVM_OP)
add_definitions(-DMXNET_USE_TVM_OP=1)
endif()

message(STATUS "CMake version '${CMAKE_VERSION}' using generator '${CMAKE_GENERATOR}'")
project(mxnet C CXX)
if(USE_CUDA)
cmake_minimum_required(VERSION 3.13.2) # CUDA 10 (Turing) detection available starting 3.13.2
include(CheckLanguage)
Expand Down Expand Up @@ -707,6 +713,7 @@ elseif(MSVC)
target_link_libraries(mxnet PUBLIC mshadow)
endif()
endif()
set_target_properties(mxnet PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR})
target_compile_definitions(mxnet PUBLIC DMLC_LOG_FATAL_THROW=$<BOOL:${LOG_FATAL_THROW}>)

# extension libraries (custom operators, custom subgraphs) are built by default
Expand Down
2 changes: 1 addition & 1 deletion cd/Jenkinsfile_utils.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ def restore_artifact(variant, libtype) {

// move libraries to lib directory
dir('lib') {
sh "mv ../mxnet_artifact/libmxnet.so ."
sh "mv ../mxnet_artifact/libmxnet.so* ."
if (fileExists('../mxnet_artifact/dependencies')) {
sh """find "../mxnet_artifact/dependencies" -type f -name "*.so*" -exec mv {} . \\;"""
sh "ls ."
Expand Down
2 changes: 1 addition & 1 deletion cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

// NOTE: the following variables are referenced in the mxnet_lib_pipeline jenkins file imported bellow
// libmxnet location
libmxnet = 'build/libmxnet.so'
libmxnet = 'build/libmxnet.so*'

// licenses
licenses = 'licenses/*'
Expand Down
2 changes: 1 addition & 1 deletion cd/mxnet_lib/static/Jenkins_pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
// NOTE: the following variables are referenced in the mxnet_lib_pipeline jenkins file imported bellow

// libmxnet location
libmxnet = 'lib/libmxnet.so'
libmxnet = 'lib/libmxnet.so*'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With this change, are we planning to package all three binaries (viz. libmxnet.so, libmxnet.so.2, libmxnet.so.2.0.0) into the nightly wheel? @sl1pkn07 @leezu


// licenses
licenses = 'licenses/*'
Expand Down
10 changes: 5 additions & 5 deletions cd/utils/test_artifact_repository.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,15 +164,15 @@ def test_probe_variant_native(self, mock_features):
Tests 'native' is returned if MKLDNN and CUDA features are OFF
"""
mock_features.return_value = {'MKLDNN': False, 'CUDA': False}
self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'native')
self.assertEqual(probe_mxnet_variant('libmxnet.so*'), 'native')

@patch('artifact_repository.get_libmxnet_features')
def test_probe_variant_cpu(self, mock_features):
"""
Tests 'cpu' is returned if MKLDNN is ON and CUDA is OFF
"""
mock_features.return_value = {'MKLDNN': True, 'CUDA': False}
self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'cpu')
self.assertEqual(probe_mxnet_variant('libmxnet.so*'), 'cpu')

@patch('artifact_repository.get_libmxnet_features')
@patch('artifact_repository.get_cuda_version')
Expand All @@ -182,15 +182,15 @@ def test_probe_variant_cuda(self, mock_cuda_version, mock_features):
"""
mock_features.return_value = {'MKLDNN': True, 'CUDA': True}
mock_cuda_version.return_value = '100'
self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'cu100')
self.assertEqual(probe_mxnet_variant('libmxnet.so*'), 'cu100')

@patch('artifact_repository.get_libmxnet_features')
def test_probe_variant_cuda_returns_none_on_no_features(self, mock_features):
"""
Tests None is returned if the mxnet features could not be extracted from the libmxnet.so file
"""
mock_features.return_value = None
self.assertIsNone(probe_mxnet_variant('libmxnet.so'))
self.assertIsNone(probe_mxnet_variant('libmxnet.so*'))

@patch('artifact_repository.get_libmxnet_features')
@patch('artifact_repository.get_cuda_version')
Expand All @@ -201,7 +201,7 @@ def test_probe_variant_cuda_mkl(self, mock_cuda_version, mock_features):
mock_features.return_value = {'MKLDNN': True, 'CUDA': True}
mock_cuda_version.return_value = None
with self.assertRaises(RuntimeError):
probe_mxnet_variant('libmxnet.so')
probe_mxnet_variant('libmxnet.so*')

def test_probe_artifact_repository_bucket(self):
"""
Expand Down
2 changes: 1 addition & 1 deletion ci/docker/runtime_functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ build_wheel() {
unzip -d ${TMPDIR} ${WHEEL}
rm ${WHEEL}
cd ${TMPDIR}
mv *.data/data/mxnet/libmxnet.so mxnet
mv *.data/data/mxnet/libmxnet.so* mxnet
zip -r ${WHEEL} .
cp ${WHEEL} ${BUILD_DIR}
rm -rf ${TMPDIR}
Expand Down
24 changes: 12 additions & 12 deletions ci/jenkins/Jenkins_steps.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,21 @@
utils = load('ci/Jenkinsfile_utils.groovy')

// mxnet libraries
mx_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so'
mx_lib_cython = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, python/mxnet/_cy3/*.so, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_ffi/_cy3/*.so'
mx_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so'
mx_lib_cython = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, python/mxnet/_cy3/*.so, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_ffi/_cy3/*.so'

// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
mx_cmake_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
mx_cmake_lib_no_tvm_op = 'build/libmxnet.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
mx_cmake_lib_cython = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so'
mx_cmake_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
mx_cmake_lib_no_tvm_op = 'build/libmxnet.so*, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
mx_cmake_lib_cython = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy3/*.so*, python/mxnet/_ffi/_cy3/*.so*'
// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
mx_cmake_lib_debug = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/tests/mxnet_unit_tests'
mx_mkldnn_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so'
mx_tensorrt_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
mx_lib_cpp_examples = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so'
mx_lib_cpp_examples_no_tvm_op = 'build/libmxnet.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so'
mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so'
mx_cd_lib = 'lib/libmxnet.so, licenses/*, lib/libgfortran.so.4, lib/libquadmath.so.0, lib/libopenblas.so.0, include/mkldnn/dnnl_version.h, include/mkldnn/dnnl_config.h'
mx_cmake_lib_debug = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/tests/mxnet_unit_tests'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unrelated to your PR, but potential improvement: Have only one variable for makefile build and one for cmake build instead of 10+. If for a particular build more files are listed than are present, the additional ones are just ignored.

mx_mkldnn_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so'
mx_tensorrt_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
mx_lib_cpp_examples = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so'
mx_lib_cpp_examples_no_tvm_op = 'build/libmxnet.so*, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so'
mx_lib_cpp_examples_cpu = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so'
mx_cd_lib = 'lib/libmxnet.so*, licenses/*, lib/libgfortran.so.4, lib/libquadmath.so.0, lib/libopenblas.so.0, include/mkldnn/dnnl_version.h, include/mkldnn/dnnl_config.h'

// Python unittest for CPU
// Python 3
Expand Down
2 changes: 1 addition & 1 deletion tests/nightly/Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
//
//This is a Jenkinsfile for nightly tests. The format and some functions have been picked up from the top-level Jenkinsfile

mx_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so'
mx_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so'

node('utility') {
// Loading the utilities requires a node context unfortunately
Expand Down
2 changes: 1 addition & 1 deletion tests/nightly/JenkinsfileForBinaries
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
//
//This is a Jenkinsfile for nightly tests. The format and some functions have been picked up from the top-level Jenkinsfile

mx_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so'
mx_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so'

node('utility') {
// Loading the utilities requires a node context unfortunately
Expand Down
2 changes: 1 addition & 1 deletion tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
//
//This is a Jenkinsfile for the model backwards compatibility checker. The format and some functions have been picked up from the top-level Jenkinsfile.

mx_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so'
mx_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so'

node('restricted-utility') {
// Loading the utilities requires a node context unfortunately
Expand Down
2 changes: 1 addition & 1 deletion tools/staticbuild/build_lib.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ cd -
# Move to lib
rm -rf lib; mkdir lib;
if [[ $PLATFORM == 'linux' ]]; then
cp -L build/libmxnet.so lib/libmxnet.so
cp -L build/libmxnet.so* lib/
cp -L staticdeps/lib/libopenblas.so lib/libopenblas.so.0
cp -L $(ldd lib/libmxnet.so | grep libgfortran | awk '{print $3}') lib/
cp -L $(ldd lib/libmxnet.so | grep libquadmath | awk '{print $3}') lib/
Expand Down