From 82a8b6e9796d352beabc19351d729edcf9219990 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Mon, 9 Sep 2019 05:01:02 +0000 Subject: [PATCH 01/13] julia: migrate build process to cmake The original build process triggered by `Pkg.add` is done by GNU make. --- julia/Project.toml | 1 + julia/deps/build.jl | 141 ++++++++++++++------------------------------ julia/src/base.jl | 19 +++--- 3 files changed, 53 insertions(+), 108 deletions(-) diff --git a/julia/Project.toml b/julia/Project.toml index 994a696c1399..ca41bb711063 100644 --- a/julia/Project.toml +++ b/julia/Project.toml @@ -5,6 +5,7 @@ version = "1.6.0" [deps] BinDeps = "9e28174c-4ba2-5203-b857-d8d62c4213ee" +CMake = "631607c0-34d2-5d66-819e-eb0f9aa2061a" Formatting = "59287772-0a20-5a39-b81b-1366585eb4c0" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/julia/deps/build.jl b/julia/deps/build.jl index dfd3f78379b0..96b1634be143 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +using CMake using JSON using Libdl using LinearAlgebra @@ -75,7 +76,7 @@ if Sys.isunix() nvcc_path = Sys.which("nvcc") if nvcc_path ≢ nothing @info "Found nvcc: $nvcc_path" - push!(CUDAPATHS, replace(nvcc_path, "bin/nvcc", "lib64")) + push!(CUDAPATHS, replace(nvcc_path, "bin/nvcc" => "lib64")) end end @@ -114,19 +115,17 @@ else end # propagate more build flags from ENV -const CC = get(ENV, "CC", nothing) -const CXX = get(ENV, "CXX", nothing) -const ADD_CFLAGS = get(ENV, "ADD_CFLAGS", nothing) -const ADD_LDFLAGS = get(ENV, "ADD_LDFLAGS", nothing) -const USE_JEMALLOC = get(ENV, "USE_JEMALLOC", nothing) # "0" or "1" - -function get_cpucore() - if haskey(ENV, "TRAVIS") # on travis-ci - 2 - else - min(Sys.CPU_THREADS, 32) - end -end +const USE_JEMALLOC = get(ENV, "USE_JEMALLOC", nothing) # "ON" or "OFF" + +get_cpucore() = min(Sys.CPU_THREADS, 32) + +cmake_bool(x::Bool) = ifelse(x ≡ true, "ON", "OFF") + +cmake_jemalloc(::Nothing) = "" +cmake_jemalloc(x::Bool) = "-DUSE_JEMALLOC=" * cmake_bool(x) + +cmake_cuda_path(::Nothing) = "" +cmake_cuda_path(x::String) = "-DUSE_CUDA_PATH=" * x using BinDeps @BinDeps.setup @@ -167,7 +166,7 @@ if !libmxnet_detected # testing run(`cmd /c dir "usr\\lib"`) return - end + end # if Sys.iswindows() ################################################################################ # If not found, try to build automatically using BinDeps @@ -181,7 +180,6 @@ if !libmxnet_detected ilp64 = "-DINTERFACE64" end - FORCE_LAPACK = false if blas_vendor == :unknown @info("Julia is built with an unkown blas library ($blas_path).") @info("Attempting build without reusing the blas library") @@ -191,33 +189,34 @@ if !libmxnet_detected @info("Attempting build anyway.") USE_JULIA_BLAS = true else - USE_JULIA_BLAS = true - FORCE_LAPACK = true + USE_JULIA_BLAS = false end @info("USE_JULIA_BLAS -> $USE_JULIA_BLAS") - blas_name = blas_vendor == :openblas64 ? "openblas" : string(blas_vendor) - MSHADOW_LDFLAGS = "MSHADOW_LDFLAGS=-lm $blas_path" + blas_name = occursin("openblas", string(blas_vendor)) ? "open" : string(blas_vendor) #-------------------------------------------------------------------------------- # Build libmxnet mxnet = library_dependency("mxnet", aliases=["mxnet", "libmxnet", "libmxnet.so"]) - _prefix = joinpath(BinDeps.depsdir(mxnet), "usr") + _blddir = joinpath(BinDeps.depsdir(mxnet), "build") _srcdir = joinpath(BinDeps.depsdir(mxnet), "src") _mxdir = joinpath(_srcdir, "mxnet") - _libdir = joinpath(_prefix, "lib") - # We have do eagerly delete the installed libmxnet.so + + # We have do eagerly delete the build stuffs. # Otherwise we won't rebuild on an update. - run(`rm -f $_libdir/libmxnet.$(Libdl.dlext)`) + rm(_blddir, recursive=true, force=true) + + @debug "build dir -> $_blddir" + provides(BuildProcess, (@build_steps begin + CreateDirectory(_blddir) CreateDirectory(_srcdir) - CreateDirectory(_libdir) @build_steps begin BinDeps.DirectoryRule(_mxdir, @build_steps begin ChangeDirectory(_srcdir) - `git clone https://github.com/apache/incubator-mxnet mxnet` + `git clone --recursive https://github.com/apache/incubator-mxnet mxnet` end) @build_steps begin ChangeDirectory(_mxdir) @@ -227,80 +226,28 @@ if !libmxnet_detected else `git checkout origin/$libmxnet_curr_ver` end - `git submodule update --init --recursive` - `git -C 3rdparty/mshadow checkout -- make/mshadow.mk` - `cp -v ../../cblas.h include/cblas.h` - `sed -i -s "s/MSHADOW_CFLAGS = \(.*\)/MSHADOW_CFLAGS = \1 $ilp64/" 3rdparty/mshadow/make/mshadow.mk` - - # Copy config.mk, always override the file - if Sys.isapple() - `cp make/osx.mk config.mk` - else - `cp make/config.mk config.mk` - end - - # Configure OpenCV - `sed -i -s 's/USE_OPENCV = 1/USE_OPENCV = 0/' config.mk` - - # Configure CUDA - if HAS_CUDA - @build_steps begin - `sed -i -s 's/USE_CUDA = 0/USE_CUDA = 1/' config.mk` - # address https://github.com/apache/incubator-mxnet/pull/7856 - `sed -i -s "s/ADD_LDFLAGS =\(.*\)/ADD_LDFLAGS =\1 -lcublas -lcusolver -lcurand -lcudart/" config.mk` - if haskey(ENV, "CUDA_HOME") - `sed -i -s "s@USE_CUDA_PATH = NONE@USE_CUDA_PATH = $(ENV["CUDA_HOME"])@" config.mk` - end - if haskey(ENV, "CUDA_HOME") - # address https://github.com/apache/incubator-mxnet/pull/7838 - flag = "-L$(ENV["CUDA_HOME"])/lib64 -L$(ENV["CUDA_HOME"])/lib" - `sed -i -s "s@ADD_LDFLAGS =\(.*\)@ADD_LDFLAGS =\1 $flag@" config.mk` - end - if HAS_CUDNN - `sed -i -s 's/USE_CUDNN = 0/USE_CUDNN = 1/' config.mk` - end - end - end - - # Force enable LAPACK build - # Julia's OpenBLAS has LAPACK functionality already - if FORCE_LAPACK - if Sys.isapple() - MSHADOW_LDFLAGS *= " -framework Accelerate" - end - `sed -i -s 's/ADD_CFLAGS =\(.*\)/ADD_CFLAGS =\1 -DMXNET_USE_LAPACK/' config.mk` - end - - # propagate more build flags from ENV - if CC != nothing - `sed -i -s "s@^export CC =\(.*\)@export CC = $CC@" config.mk` - end - if CXX != nothing - `sed -i -s "s@^export CXX =\(.*\)@export CXX = $CXX@" config.mk` - end - if ADD_CFLAGS != nothing - `sed -i -s "s@ADD_CFLAGS =\(.*\)@ADD_CFLAGS =\1 $ADD_CFLAGS@" config.mk` - end - if ADD_LDFLAGS != nothing - `sed -i -s "s@ADD_LDFLAGS =\(.*\)@ADD_LDFLAGS =\1 $ADD_LDFLAGS@" config.mk` - end - if USE_JEMALLOC != nothing - `sed -i -s "s@USE_JEMALLOC =\(.*\)@USE_JEMALLOC = $USE_JEMALLOC@" config.mk` - end - - if USE_JULIA_BLAS - `make -j$(get_cpucore()) USE_BLAS=$blas_name $MSHADOW_LDFLAGS` - else - `make -j$(get_cpucore())` - end + `cp -f -v ../../cblas.h include/cblas.h` + end + @build_steps begin + ChangeDirectory(_blddir) + `$cmake + -DCMAKE_BUILD_TYPE=$(libmxnet_curr_ver == "master" ? "Debug" : "Release") + -DUSE_BLAS=$blas_name + -DUSE_OPENCV=$(cmake_bool(false)) + -DUSE_CUDA=$(cmake_bool(HAS_CUDA)) + -DUSE_CUDNN=$(cmake_bool(HAS_CUDNN)) + $(cmake_jemalloc(USE_JEMALLOC)) + $(cmake_cuda_path(get(ENV, "CUDA_HOME", nothing))) + $_mxdir` + `make -j$(get_cpucore()) VERBOSE=$(Int(libmxnet_curr_ver == "master"))` end - FileRule(joinpath(_libdir, "libmxnet.$(Libdl.dlext)"), @build_steps begin - # the output file on macos is still in `.so` suffix - # so we rename it - `cp $_mxdir/lib/libmxnet.so $_libdir/libmxnet.$(Libdl.dlext)` + FileRule(joinpath(_blddir, "libmxnet.$(Libdl.dlext)"), @build_steps begin + # the output file on macos is still in `.so` suffix, + # so we create a soft link for it. + `ln -s libmxnet.so $_blddir/libmxnet.$(Libdl.dlext)` end) end - end), mxnet, installed_libpath=_libdir) + end), mxnet, installed_libpath=_blddir) @BinDeps.install Dict(:mxnet => :mxnet) end diff --git a/julia/src/base.jl b/julia/src/base.jl index 7a2bde0907c1..3460c9d0eeb7 100644 --- a/julia/src/base.jl +++ b/julia/src/base.jl @@ -58,11 +58,11 @@ function _get_search_names() end function _get_search_dirs() - # TODO: remove MXNET_HOME backward compatibility in v1.7 + # TODO: remove MXNET_HOME backward compatibility in v2.0 if haskey(ENV, "MXNET_HOME") @warn "The environment variable `MXNET_HOME` has been renamed, please use `MXNET_ROOT` instead." end - A = [joinpath(@__DIR__, "..", "deps", "usr", "lib")] + A = [joinpath(@__DIR__, "..", "deps", "build")] MXNET_ROOT = get(ENV, "MXNET_ROOT", get(ENV, "MXNET_HOME", "")) if !isempty(MXNET_ROOT) !isabspath(MXNET_ROOT) && error("MXNET_ROOT should be a absolute path") @@ -71,19 +71,16 @@ function _get_search_dirs() A end -const MXNET_LIB = Libdl.find_library(_get_search_names(), _get_search_dirs()) -const LIB_VERSION = Ref{Cint}(0) +const MXNET_LIB = Libdl.find_library(_get_search_names(), _get_search_dirs()) +const LIB_VERSION = Ref{Cint}(C_NULL) if isempty(MXNET_LIB) - # touch this file, so that after the user properly build libmxnet, the precompiled - # MXNet.ji will be re-compiled to get MXNET_LIB properly. - touch(@__FILE__) error("Cannot find or load libmxnet.$(Libdl.dlext). " * "Please see the document on how to build it.") -else - include_dependency(MXNET_LIB) end +include_dependency(MXNET_LIB) + function __init__() # TODO: bug in nnvm, if do not call this, call get handle "_copyto" will fail _get_libmx_op_names() @@ -92,12 +89,12 @@ function __init__() atexit() do # notify libmxnet we are shutting down - ccall( ("MXNotifyShutdown", MXNET_LIB), Cint, () ) + ccall(("MXNotifyShutdown", MXNET_LIB), Cint, ()) end end function mx_get_last_error() - msg = ccall( ("MXGetLastError", MXNET_LIB), char_p, () ) + msg = ccall(("MXGetLastError", MXNET_LIB), char_p, ()) if msg == C_NULL throw(MXError("Failed to get last error message")) end From 6f27f0e038cd7129e2c1636df7583192564baa51 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 14 Sep 2019 13:16:11 +0000 Subject: [PATCH 02/13] build with Julia's private libopenblas --- cmake/Modules/FindOpenBLAS.cmake | 25 ++++++++++++++++++++++--- julia/deps/build.jl | 10 ++++------ julia/docs/src/user-guide/install.md | 9 +++++---- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/cmake/Modules/FindOpenBLAS.cmake b/cmake/Modules/FindOpenBLAS.cmake index a3a79caae461..9212ac275184 100644 --- a/cmake/Modules/FindOpenBLAS.cmake +++ b/cmake/Modules/FindOpenBLAS.cmake @@ -46,14 +46,15 @@ SET(Open_BLAS_LIB_SEARCH_PATHS /usr/local/opt/openblas/lib ${PROJECT_SOURCE_DIR}/3rdparty/OpenBLAS/lib ${PROJECT_SOURCE_DIR}/thirdparty/OpenBLAS/lib - ${OpenBLAS_DIR} - ${OpenBLAS_DIR}/lib + ${OpenBLAS_DIR} + ${OpenBLAS_DIR}/lib ${OpenBLAS_HOME} ${OpenBLAS_HOME}/lib ) FIND_PATH(OpenBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Open_BLAS_INCLUDE_SEARCH_PATHS}) -FIND_LIBRARY(OpenBLAS_LIB NAMES openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS}) +# the Julia's private OpenBLAS is named as `libopenblas64_.so` on x86-64 Linux +FIND_LIBRARY(OpenBLAS_LIB NAMES openblas64_ openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS}) IF(NOT OpenBLAS_LIB) FIND_FILE(OpenBLAS_LIB NAMES libopenblas.dll.a PATHS ${Open_BLAS_LIB_SEARCH_PATHS}) ENDIF() @@ -89,3 +90,21 @@ MARK_AS_ADVANCED( OpenBLAS ) +# Check ILP64 data model for the case of Julia self-shipped `libopenblas64_.so` +SET(detect_interface64_src " + #include + char* openblas_get_config64_(void)\; + int main() { + return strstr(openblas_get_config64_(), \"USE64BITINT\") == NULL\; + } +") +FILE(WRITE "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c" ${detect_interface64_src}) +TRY_RUN( + run_detect_interface64 compile_detect_interface64 + "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c" + LINK_LIBRARIES ${OpenBLAS_LIB} +) +IF(run_detect_interface64 EQUAL 0) + add_definitions(-DINTERFACE64=1) # see julia/deps/cblas.h +ENDIF(run_detect_interface64 EQUAL 0) +FILE(REMOVE "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c") diff --git a/julia/deps/build.jl b/julia/deps/build.jl index 96b1634be143..97f25160a954 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -127,6 +127,8 @@ cmake_jemalloc(x::Bool) = "-DUSE_JEMALLOC=" * cmake_bool(x) cmake_cuda_path(::Nothing) = "" cmake_cuda_path(x::String) = "-DUSE_CUDA_PATH=" * x +cmake_jl_blas(x::Bool, blas_path) = ifelse(x, "-DOpenBLAS_LIB=$blas_path", "") + using BinDeps @BinDeps.setup if !libmxnet_detected @@ -175,11 +177,6 @@ if !libmxnet_detected blas_path = Libdl.dlpath(Libdl.dlopen(Base.libblas_name)) blas_vendor = LinearAlgebra.BLAS.vendor() - ilp64 = "" - if blas_vendor == :openblas64 - ilp64 = "-DINTERFACE64" - end - if blas_vendor == :unknown @info("Julia is built with an unkown blas library ($blas_path).") @info("Attempting build without reusing the blas library") @@ -191,7 +188,7 @@ if !libmxnet_detected else USE_JULIA_BLAS = false end - @info("USE_JULIA_BLAS -> $USE_JULIA_BLAS") + @info "USE_JULIA_BLAS -> $USE_JULIA_BLAS" blas_name = occursin("openblas", string(blas_vendor)) ? "open" : string(blas_vendor) @@ -238,6 +235,7 @@ if !libmxnet_detected -DUSE_CUDNN=$(cmake_bool(HAS_CUDNN)) $(cmake_jemalloc(USE_JEMALLOC)) $(cmake_cuda_path(get(ENV, "CUDA_HOME", nothing))) + $(cmake_jl_blas(USE_JULIA_BLAS, blas_path)) $_mxdir` `make -j$(get_cpucore()) VERBOSE=$(Int(libmxnet_curr_ver == "master"))` end diff --git a/julia/docs/src/user-guide/install.md b/julia/docs/src/user-guide/install.md index d9d5574b025d..aa38a378d695 100644 --- a/julia/docs/src/user-guide/install.md +++ b/julia/docs/src/user-guide/install.md @@ -54,11 +54,12 @@ There are several environment variables that change this behaviour. (e.g. `a0b1c2d3`). - `CC`: The path of C compiler. - `CXX`: The path of C++ compiler. -- `ADD_CFLAGS`: Additional C flags. For instance, +- `CFLAGS`: Additional C flags. For instance, if you need to point non-standard include directory, please set it as - `ENV["ADD_CFLAGS"] = "-I'/path/to/include/dir'"`. -- `ADD_LDFLAGS`: Additional linker flags. -- `USE_JEMALLOC`: Default is enabled if jemalloc available. + `ENV["CFLAGS"] = "-I'/path/to/include/dir'"`. +- `LDFLAGS`: Additional linker flags. +- `USE_JEMALLOC`: Set it to `ON` or `OFF`. + Default is enabled if jemalloc available. If you ran into segfault cause by jemalloc, Please try to disable it. From 8c73cd93833634b22d5c95094bc7cf51a8191de0 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 14 Sep 2019 21:32:02 +0800 Subject: [PATCH 03/13] submodule update --- julia/deps/build.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/julia/deps/build.jl b/julia/deps/build.jl index 97f25160a954..972d6d3342ba 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -218,6 +218,7 @@ if !libmxnet_detected @build_steps begin ChangeDirectory(_mxdir) `git fetch` + `git submodule update` if libmxnet_curr_ver != "master" `git checkout $libmxnet_curr_ver` else From 3ef9cf0b38e524a487a6d28af88c63c7f15fcc41 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 14 Sep 2019 13:59:29 +0000 Subject: [PATCH 04/13] USE_JULIA_BLAS --- julia/deps/build.jl | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/julia/deps/build.jl b/julia/deps/build.jl index 972d6d3342ba..c8f6134dd197 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -177,17 +177,7 @@ if !libmxnet_detected blas_path = Libdl.dlpath(Libdl.dlopen(Base.libblas_name)) blas_vendor = LinearAlgebra.BLAS.vendor() - if blas_vendor == :unknown - @info("Julia is built with an unkown blas library ($blas_path).") - @info("Attempting build without reusing the blas library") - USE_JULIA_BLAS = false - elseif !(blas_vendor in (:openblas, :openblas64)) - @info("Unsure if we can build against $blas_vendor.") - @info("Attempting build anyway.") - USE_JULIA_BLAS = true - else - USE_JULIA_BLAS = false - end + USE_JULIA_BLAS = (blas_vendor in (:openblas, :openblas64)) @info "USE_JULIA_BLAS -> $USE_JULIA_BLAS" blas_name = occursin("openblas", string(blas_vendor)) ? "open" : string(blas_vendor) From 27f996da8952b73f8eaa880bb0ca713ee3a6a7b5 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Mon, 9 Sep 2019 05:01:02 +0000 Subject: [PATCH 05/13] julia: migrate build process to cmake The original build process triggered by `Pkg.add` is done by GNU make. --- julia/Project.toml | 1 + julia/deps/build.jl | 137 ++++++++++++++------------------------------ julia/src/base.jl | 43 +++++++++----- 3 files changed, 73 insertions(+), 108 deletions(-) diff --git a/julia/Project.toml b/julia/Project.toml index 994a696c1399..ca41bb711063 100644 --- a/julia/Project.toml +++ b/julia/Project.toml @@ -5,6 +5,7 @@ version = "1.6.0" [deps] BinDeps = "9e28174c-4ba2-5203-b857-d8d62c4213ee" +CMake = "631607c0-34d2-5d66-819e-eb0f9aa2061a" Formatting = "59287772-0a20-5a39-b81b-1366585eb4c0" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/julia/deps/build.jl b/julia/deps/build.jl index a79d2a062c18..03c5db66e18e 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +using CMake using JSON using Libdl using LinearAlgebra @@ -93,19 +94,17 @@ else end # propagate more build flags from ENV -const CC = get(ENV, "CC", nothing) -const CXX = get(ENV, "CXX", nothing) -const ADD_CFLAGS = get(ENV, "ADD_CFLAGS", nothing) -const ADD_LDFLAGS = get(ENV, "ADD_LDFLAGS", nothing) -const USE_JEMALLOC = get(ENV, "USE_JEMALLOC", nothing) # "0" or "1" +const USE_JEMALLOC = get(ENV, "USE_JEMALLOC", nothing) # "ON" or "OFF" -function get_cpucore() - if haskey(ENV, "TRAVIS") # on travis-ci - 2 - else - min(Sys.CPU_THREADS, 32) - end -end +get_cpucore() = min(Sys.CPU_THREADS, 32) + +cmake_bool(x::Bool) = ifelse(x ≡ true, "ON", "OFF") + +cmake_jemalloc(::Nothing) = "" +cmake_jemalloc(x::Bool) = "-DUSE_JEMALLOC=" * cmake_bool(x) + +cmake_cuda_path(::Nothing) = "" +cmake_cuda_path(x::String) = "-DUSE_CUDA_PATH=" * x using BinDeps @BinDeps.setup @@ -146,7 +145,7 @@ if !libmxnet_detected # testing run(`cmd /c dir "usr\\lib"`) return - end + end # if Sys.iswindows() ################################################################################ # If not found, try to build automatically using BinDeps @@ -160,7 +159,6 @@ if !libmxnet_detected ilp64 = "-DINTERFACE64" end - FORCE_LAPACK = false if blas_vendor == :unknown @info("Julia is built with an unkown blas library ($blas_path).") @info("Attempting build without reusing the blas library") @@ -170,33 +168,34 @@ if !libmxnet_detected @info("Attempting build anyway.") USE_JULIA_BLAS = true else - USE_JULIA_BLAS = true - FORCE_LAPACK = true + USE_JULIA_BLAS = false end @info("USE_JULIA_BLAS -> $USE_JULIA_BLAS") - blas_name = blas_vendor == :openblas64 ? "openblas" : string(blas_vendor) - MSHADOW_LDFLAGS = "MSHADOW_LDFLAGS=-lm $blas_path" + blas_name = occursin("openblas", string(blas_vendor)) ? "open" : string(blas_vendor) #-------------------------------------------------------------------------------- # Build libmxnet mxnet = library_dependency("mxnet", aliases=["mxnet", "libmxnet", "libmxnet.so"]) - _prefix = joinpath(BinDeps.depsdir(mxnet), "usr") + _blddir = joinpath(BinDeps.depsdir(mxnet), "build") _srcdir = joinpath(BinDeps.depsdir(mxnet), "src") _mxdir = joinpath(_srcdir, "mxnet") - _libdir = joinpath(_prefix, "lib") - # We have do eagerly delete the installed libmxnet.so + + # We have do eagerly delete the build stuffs. # Otherwise we won't rebuild on an update. - run(`rm -f $_libdir/libmxnet.$(Libdl.dlext)`) + rm(_blddir, recursive=true, force=true) + + @debug "build dir -> $_blddir" + provides(BuildProcess, (@build_steps begin + CreateDirectory(_blddir) CreateDirectory(_srcdir) - CreateDirectory(_libdir) @build_steps begin BinDeps.DirectoryRule(_mxdir, @build_steps begin ChangeDirectory(_srcdir) - `git clone https://github.com/apache/incubator-mxnet mxnet` + `git clone --recursive https://github.com/apache/incubator-mxnet mxnet` end) @build_steps begin ChangeDirectory(_mxdir) @@ -206,80 +205,28 @@ if !libmxnet_detected else `git checkout origin/$libmxnet_curr_ver` end - `git submodule update --init --recursive` - `git -C 3rdparty/mshadow checkout -- make/mshadow.mk` - `cp -v ../../cblas.h include/cblas.h` - `sed -i -s "s/MSHADOW_CFLAGS = \(.*\)/MSHADOW_CFLAGS = \1 $ilp64/" 3rdparty/mshadow/make/mshadow.mk` - - # Copy config.mk, always override the file - if Sys.isapple() - `cp make/osx.mk config.mk` - else - `cp make/config.mk config.mk` - end - - # Configure OpenCV - `sed -i -s 's/USE_OPENCV = 1/USE_OPENCV = 0/' config.mk` - - # Configure CUDA - if HAS_CUDA - @build_steps begin - `sed -i -s 's/USE_CUDA = 0/USE_CUDA = 1/' config.mk` - # address https://github.com/apache/incubator-mxnet/pull/7856 - `sed -i -s "s/ADD_LDFLAGS =\(.*\)/ADD_LDFLAGS =\1 -lcublas -lcusolver -lcurand -lcudart/" config.mk` - if haskey(ENV, "CUDA_HOME") - `sed -i -s "s@USE_CUDA_PATH = NONE@USE_CUDA_PATH = $(ENV["CUDA_HOME"])@" config.mk` - end - if haskey(ENV, "CUDA_HOME") - # address https://github.com/apache/incubator-mxnet/pull/7838 - flag = "-L$(ENV["CUDA_HOME"])/lib64 -L$(ENV["CUDA_HOME"])/lib" - `sed -i -s "s@ADD_LDFLAGS =\(.*\)@ADD_LDFLAGS =\1 $flag@" config.mk` - end - if HAS_CUDNN - `sed -i -s 's/USE_CUDNN = 0/USE_CUDNN = 1/' config.mk` - end - end - end - - # Force enable LAPACK build - # Julia's OpenBLAS has LAPACK functionality already - if FORCE_LAPACK - if Sys.isapple() - MSHADOW_LDFLAGS *= " -framework Accelerate" - end - `sed -i -s 's/ADD_CFLAGS =\(.*\)/ADD_CFLAGS =\1 -DMXNET_USE_LAPACK/' config.mk` - end - - # propagate more build flags from ENV - if CC != nothing - `sed -i -s "s@^export CC =\(.*\)@export CC = $CC@" config.mk` - end - if CXX != nothing - `sed -i -s "s@^export CXX =\(.*\)@export CXX = $CXX@" config.mk` - end - if ADD_CFLAGS != nothing - `sed -i -s "s@ADD_CFLAGS =\(.*\)@ADD_CFLAGS =\1 $ADD_CFLAGS@" config.mk` - end - if ADD_LDFLAGS != nothing - `sed -i -s "s@ADD_LDFLAGS =\(.*\)@ADD_LDFLAGS =\1 $ADD_LDFLAGS@" config.mk` - end - if USE_JEMALLOC != nothing - `sed -i -s "s@USE_JEMALLOC =\(.*\)@USE_JEMALLOC = $USE_JEMALLOC@" config.mk` - end - - if USE_JULIA_BLAS - `make -j$(get_cpucore()) USE_BLAS=$blas_name $MSHADOW_LDFLAGS` - else - `make -j$(get_cpucore())` - end + `cp -f -v ../../cblas.h include/cblas.h` + end + @build_steps begin + ChangeDirectory(_blddir) + `$cmake + -DCMAKE_BUILD_TYPE=$(libmxnet_curr_ver == "master" ? "Debug" : "Release") + -DUSE_BLAS=$blas_name + -DUSE_OPENCV=$(cmake_bool(false)) + -DUSE_CUDA=$(cmake_bool(HAS_CUDA)) + -DUSE_CUDNN=$(cmake_bool(HAS_CUDNN)) + $(cmake_jemalloc(USE_JEMALLOC)) + $(cmake_cuda_path(get(ENV, "CUDA_HOME", nothing))) + $_mxdir` + `make -j$(get_cpucore()) VERBOSE=$(Int(libmxnet_curr_ver == "master"))` end - FileRule(joinpath(_libdir, "libmxnet.$(Libdl.dlext)"), @build_steps begin - # the output file on macos is still in `.so` suffix - # so we rename it - `cp $_mxdir/lib/libmxnet.so $_libdir/libmxnet.$(Libdl.dlext)` + FileRule(joinpath(_blddir, "libmxnet.$(Libdl.dlext)"), @build_steps begin + # the output file on macos is still in `.so` suffix, + # so we create a soft link for it. + `ln -s libmxnet.so $_blddir/libmxnet.$(Libdl.dlext)` end) end - end), mxnet, installed_libpath=_libdir) + end), mxnet, installed_libpath=_blddir) @BinDeps.install Dict(:mxnet => :mxnet) end diff --git a/julia/src/base.jl b/julia/src/base.jl index 2b60dca4f783..8a7d555b1b6a 100644 --- a/julia/src/base.jl +++ b/julia/src/base.jl @@ -40,23 +40,40 @@ const grad_req_map = Dict{Symbol,GRAD_REQ}( ################################################################################ # Initialization and library API entrance ################################################################################ -const MXNET_LIB = Libdl.find_library(["libmxnet.$(Libdl.dlext)", "libmxnet.so"], # see build.jl - [joinpath(get(ENV, "MXNET_HOME", ""), "lib"), - get(ENV, "MXNET_HOME", ""), - joinpath(@__DIR__, "..", - "deps", "usr", "lib")]) -const LIB_VERSION = Ref{Cint}(0) +function _get_search_names() + MXNET_LIBRARY_PATH = get(ENV, "MXNET_LIBRARY_PATH", "") + A = ["libmxnet.$(Libdl.dlext)", "libmxnet.so"] # see build.jl + if !isempty(MXNET_LIBRARY_PATH) + !isabspath(MXNET_LIBRARY_PATH) && error("MXNET_LIBRARY_PATH should be a absolute path") + pushfirst!(A, MXNET_LIBRARY_PATH) + end + A +end + +function _get_search_dirs() + # TODO: remove MXNET_HOME backward compatibility in v2.0 + if haskey(ENV, "MXNET_HOME") + @warn "The environment variable `MXNET_HOME` has been renamed, please use `MXNET_ROOT` instead." + end + A = [joinpath(@__DIR__, "..", "deps", "build")] + MXNET_ROOT = get(ENV, "MXNET_ROOT", get(ENV, "MXNET_HOME", "")) + if !isempty(MXNET_ROOT) + !isabspath(MXNET_ROOT) && error("MXNET_ROOT should be a absolute path") + prepend!(A, [joinpath(MXNET_ROOT, "lib"), MXNET_ROOT]) + end + A +end + +const MXNET_LIB = Libdl.find_library(_get_search_names(), _get_search_dirs()) +const LIB_VERSION = Ref{Cint}(C_NULL) if isempty(MXNET_LIB) - # touch this file, so that after the user properly build libmxnet, the precompiled - # MXNet.ji will be re-compiled to get MXNET_LIB properly. - touch(@__FILE__) error("Cannot find or load libmxnet.$(Libdl.dlext). " * "Please see the document on how to build it.") -else - include_dependency(MXNET_LIB) end +include_dependency(MXNET_LIB) + function __init__() # TODO: bug in nnvm, if do not call this, call get handle "_copyto" will fail _get_libmx_op_names() @@ -65,12 +82,12 @@ function __init__() atexit() do # notify libmxnet we are shutting down - ccall( ("MXNotifyShutdown", MXNET_LIB), Cint, () ) + ccall(("MXNotifyShutdown", MXNET_LIB), Cint, ()) end end function mx_get_last_error() - msg = ccall( ("MXGetLastError", MXNET_LIB), char_p, () ) + msg = ccall(("MXGetLastError", MXNET_LIB), char_p, ()) if msg == C_NULL throw(MXError("Failed to get last error message")) end From 46e7bd27db91d759bc60fa8bc79857e88f15c087 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 14 Sep 2019 13:16:11 +0000 Subject: [PATCH 06/13] build with Julia's private libopenblas --- cmake/Modules/FindOpenBLAS.cmake | 25 ++++++++++++++++++++++--- julia/deps/build.jl | 10 ++++------ julia/docs/src/user-guide/install.md | 9 +++++---- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/cmake/Modules/FindOpenBLAS.cmake b/cmake/Modules/FindOpenBLAS.cmake index a3a79caae461..9212ac275184 100644 --- a/cmake/Modules/FindOpenBLAS.cmake +++ b/cmake/Modules/FindOpenBLAS.cmake @@ -46,14 +46,15 @@ SET(Open_BLAS_LIB_SEARCH_PATHS /usr/local/opt/openblas/lib ${PROJECT_SOURCE_DIR}/3rdparty/OpenBLAS/lib ${PROJECT_SOURCE_DIR}/thirdparty/OpenBLAS/lib - ${OpenBLAS_DIR} - ${OpenBLAS_DIR}/lib + ${OpenBLAS_DIR} + ${OpenBLAS_DIR}/lib ${OpenBLAS_HOME} ${OpenBLAS_HOME}/lib ) FIND_PATH(OpenBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Open_BLAS_INCLUDE_SEARCH_PATHS}) -FIND_LIBRARY(OpenBLAS_LIB NAMES openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS}) +# the Julia's private OpenBLAS is named as `libopenblas64_.so` on x86-64 Linux +FIND_LIBRARY(OpenBLAS_LIB NAMES openblas64_ openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS}) IF(NOT OpenBLAS_LIB) FIND_FILE(OpenBLAS_LIB NAMES libopenblas.dll.a PATHS ${Open_BLAS_LIB_SEARCH_PATHS}) ENDIF() @@ -89,3 +90,21 @@ MARK_AS_ADVANCED( OpenBLAS ) +# Check ILP64 data model for the case of Julia self-shipped `libopenblas64_.so` +SET(detect_interface64_src " + #include + char* openblas_get_config64_(void)\; + int main() { + return strstr(openblas_get_config64_(), \"USE64BITINT\") == NULL\; + } +") +FILE(WRITE "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c" ${detect_interface64_src}) +TRY_RUN( + run_detect_interface64 compile_detect_interface64 + "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c" + LINK_LIBRARIES ${OpenBLAS_LIB} +) +IF(run_detect_interface64 EQUAL 0) + add_definitions(-DINTERFACE64=1) # see julia/deps/cblas.h +ENDIF(run_detect_interface64 EQUAL 0) +FILE(REMOVE "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c") diff --git a/julia/deps/build.jl b/julia/deps/build.jl index 03c5db66e18e..203eb9597eef 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -106,6 +106,8 @@ cmake_jemalloc(x::Bool) = "-DUSE_JEMALLOC=" * cmake_bool(x) cmake_cuda_path(::Nothing) = "" cmake_cuda_path(x::String) = "-DUSE_CUDA_PATH=" * x +cmake_jl_blas(x::Bool, blas_path) = ifelse(x, "-DOpenBLAS_LIB=$blas_path", "") + using BinDeps @BinDeps.setup if !libmxnet_detected @@ -154,11 +156,6 @@ if !libmxnet_detected blas_path = Libdl.dlpath(Libdl.dlopen(Base.libblas_name)) blas_vendor = LinearAlgebra.BLAS.vendor() - ilp64 = "" - if blas_vendor == :openblas64 - ilp64 = "-DINTERFACE64" - end - if blas_vendor == :unknown @info("Julia is built with an unkown blas library ($blas_path).") @info("Attempting build without reusing the blas library") @@ -170,7 +167,7 @@ if !libmxnet_detected else USE_JULIA_BLAS = false end - @info("USE_JULIA_BLAS -> $USE_JULIA_BLAS") + @info "USE_JULIA_BLAS -> $USE_JULIA_BLAS" blas_name = occursin("openblas", string(blas_vendor)) ? "open" : string(blas_vendor) @@ -217,6 +214,7 @@ if !libmxnet_detected -DUSE_CUDNN=$(cmake_bool(HAS_CUDNN)) $(cmake_jemalloc(USE_JEMALLOC)) $(cmake_cuda_path(get(ENV, "CUDA_HOME", nothing))) + $(cmake_jl_blas(USE_JULIA_BLAS, blas_path)) $_mxdir` `make -j$(get_cpucore()) VERBOSE=$(Int(libmxnet_curr_ver == "master"))` end diff --git a/julia/docs/src/user-guide/install.md b/julia/docs/src/user-guide/install.md index 129b6a190c5b..8fb5274f91b5 100644 --- a/julia/docs/src/user-guide/install.md +++ b/julia/docs/src/user-guide/install.md @@ -50,11 +50,12 @@ There are several environment variables that change this behaviour. (e.g. `a0b1c2d3`). - `CC`: The path of C compiler. - `CXX`: The path of C++ compiler. -- `ADD_CFLAGS`: Additional C flags. For instance, +- `CFLAGS`: Additional C flags. For instance, if you need to point non-standard include directory, please set it as - `ENV["ADD_CFLAGS"] = "-I'/path/to/include/dir'"`. -- `ADD_LDFLAGS`: Additional linker flags. -- `USE_JEMALLOC`: Default is enabled if jemalloc available. + `ENV["CFLAGS"] = "-I'/path/to/include/dir'"`. +- `LDFLAGS`: Additional linker flags. +- `USE_JEMALLOC`: Set it to `ON` or `OFF`. + Default is enabled if jemalloc available. If you ran into segfault cause by jemalloc, Please try to disable it. From cede75b971d6252106b162d3b428050247878696 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 14 Sep 2019 21:32:02 +0800 Subject: [PATCH 07/13] submodule update --- julia/deps/build.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/julia/deps/build.jl b/julia/deps/build.jl index 203eb9597eef..508698277a83 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -197,6 +197,7 @@ if !libmxnet_detected @build_steps begin ChangeDirectory(_mxdir) `git fetch` + `git submodule update` if libmxnet_curr_ver != "master" `git checkout $libmxnet_curr_ver` else From b28b847275b3086445b11463aa93bbbf0fe76c8d Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 14 Sep 2019 13:59:29 +0000 Subject: [PATCH 08/13] USE_JULIA_BLAS --- julia/deps/build.jl | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/julia/deps/build.jl b/julia/deps/build.jl index 508698277a83..22650784a7f7 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -156,17 +156,7 @@ if !libmxnet_detected blas_path = Libdl.dlpath(Libdl.dlopen(Base.libblas_name)) blas_vendor = LinearAlgebra.BLAS.vendor() - if blas_vendor == :unknown - @info("Julia is built with an unkown blas library ($blas_path).") - @info("Attempting build without reusing the blas library") - USE_JULIA_BLAS = false - elseif !(blas_vendor in (:openblas, :openblas64)) - @info("Unsure if we can build against $blas_vendor.") - @info("Attempting build anyway.") - USE_JULIA_BLAS = true - else - USE_JULIA_BLAS = false - end + USE_JULIA_BLAS = (blas_vendor in (:openblas, :openblas64)) @info "USE_JULIA_BLAS -> $USE_JULIA_BLAS" blas_name = occursin("openblas", string(blas_vendor)) ? "open" : string(blas_vendor) From 49fad87adf44f0601e617c9f0286bb5671bbaf07 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 17 Dec 2019 13:01:31 +0000 Subject: [PATCH 09/13] submodule update --- julia/deps/build.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/julia/deps/build.jl b/julia/deps/build.jl index 22650784a7f7..bc248f44cddf 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -187,7 +187,7 @@ if !libmxnet_detected @build_steps begin ChangeDirectory(_mxdir) `git fetch` - `git submodule update` + `git submodule update --recursive --force` if libmxnet_curr_ver != "master" `git checkout $libmxnet_curr_ver` else From e5b48710177ee8a220375918f17164b933693afa Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 21 Dec 2019 09:25:21 +0000 Subject: [PATCH 10/13] OPENBLAS_INTERFACE64 --- cmake/Modules/FindOpenBLAS.cmake | 8 ++++---- julia/deps/build.jl | 2 +- julia/deps/{ => include}/cblas.h | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) rename julia/deps/{ => include}/cblas.h (99%) diff --git a/cmake/Modules/FindOpenBLAS.cmake b/cmake/Modules/FindOpenBLAS.cmake index 9212ac275184..a261f469bf82 100644 --- a/cmake/Modules/FindOpenBLAS.cmake +++ b/cmake/Modules/FindOpenBLAS.cmake @@ -100,11 +100,11 @@ SET(detect_interface64_src " ") FILE(WRITE "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c" ${detect_interface64_src}) TRY_RUN( - run_detect_interface64 compile_detect_interface64 + OpenBLAS_INTERFACE64 compile_detect_interface64 "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c" LINK_LIBRARIES ${OpenBLAS_LIB} ) -IF(run_detect_interface64 EQUAL 0) - add_definitions(-DINTERFACE64=1) # see julia/deps/cblas.h -ENDIF(run_detect_interface64 EQUAL 0) +IF(OpenBLAS_INTERFACE64 EQUAL 0) + add_definitions(-DOPENBLAS_INTERFACE64=1) # see julia/deps/cblas.h +ENDIF(OpenBLAS_INTERFACE64 EQUAL 0) FILE(REMOVE "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c") diff --git a/julia/deps/build.jl b/julia/deps/build.jl index bc248f44cddf..97b4ddd3d636 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -193,7 +193,7 @@ if !libmxnet_detected else `git checkout origin/$libmxnet_curr_ver` end - `cp -f -v ../../cblas.h include/cblas.h` + `cp -f -v julia/deps/include/cblas.h include/cblas.h` end @build_steps begin ChangeDirectory(_blddir) diff --git a/julia/deps/cblas.h b/julia/deps/include/cblas.h similarity index 99% rename from julia/deps/cblas.h rename to julia/deps/include/cblas.h index d9449dc8e21d..54569da672be 100644 --- a/julia/deps/cblas.h +++ b/julia/deps/include/cblas.h @@ -52,7 +52,7 @@ typedef long BLASLONG; typedef unsigned long BLASULONG; #endif -#ifdef INTERFACE64 +#ifdef OPENBLAS_INTERFACE64 typedef BLASLONG blasint; #else typedef int blasint; @@ -74,7 +74,7 @@ typedef int blasint; typedef struct { double real, imag; } openblas_complex_double; #endif -#ifdef INTERFACE64 +#ifdef OPENBLAS_INTERFACE64 # define cblas_sdsdot cblas_sdsdot64_ # define cblas_dsdot cblas_dsdot64_ # define cblas_sdot cblas_sdot64_ @@ -240,7 +240,7 @@ typedef int blasint; # define cblas_dgeadd cblas_dgeadd64_ # define cblas_cgeadd cblas_cgeadd64_ # define cblas_zgeadd cblas_zgeadd64_ -#endif +#endif // OPENBLAS_INTERFACE64 #define CBLAS_INDEX size_t From 877915416bc155599d6c3059e6894d1c436a5c58 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sat, 21 Dec 2019 09:28:07 +0000 Subject: [PATCH 11/13] fix command quote warning --- julia/deps/build.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/julia/deps/build.jl b/julia/deps/build.jl index 97b4ddd3d636..bac0c00c44c5 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -138,7 +138,7 @@ if !libmxnet_detected run(download_cmd(package_url, "mxnet.7z")) # this command will create the dir "usr\\lib" - run(`$exe7z e mxnet.7z *\\build\\* *\\lib\\* -y -ousr\\lib`) + run(`$exe7z e mxnet.7z "*\\build\\*" "*\\lib\\*" -y -ousr\\lib`) # TODO check it works on windows or not run(download_cmd(base_url, "mxnet_base.7z")) run(`$exe7z x mxnet_base.7z -y -ousr`) From ef7a75e556259c72cde38f7154d9701401715b8a Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 24 Dec 2019 22:47:15 +0800 Subject: [PATCH 12/13] Update julia/deps/build.jl Co-Authored-By: Chaitanya Prakash Bapat --- julia/deps/build.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/julia/deps/build.jl b/julia/deps/build.jl index bac0c00c44c5..fc915526a70a 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -169,7 +169,7 @@ if !libmxnet_detected _srcdir = joinpath(BinDeps.depsdir(mxnet), "src") _mxdir = joinpath(_srcdir, "mxnet") - # We have do eagerly delete the build stuffs. + # We have to eagerly delete the build directory. # Otherwise we won't rebuild on an update. rm(_blddir, recursive=true, force=true) From afe7bee98fc7dd5bb39f9c54006e5815b79d82eb Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Tue, 24 Dec 2019 22:51:34 +0800 Subject: [PATCH 13/13] Update julia/deps/build.jl --- julia/deps/build.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/julia/deps/build.jl b/julia/deps/build.jl index fc915526a70a..ea7510c4f040 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -138,7 +138,7 @@ if !libmxnet_detected run(download_cmd(package_url, "mxnet.7z")) # this command will create the dir "usr\\lib" - run(`$exe7z e mxnet.7z "*\\build\\*" "*\\lib\\*" -y -ousr\\lib`) # TODO check it works on windows or not + run(`$exe7z e mxnet.7z "*\\build\\*" "*\\lib\\*" -y -ousr\\lib`) run(download_cmd(base_url, "mxnet_base.7z")) run(`$exe7z x mxnet_base.7z -y -ousr`)