Skip to content
This repository was archived by the owner on Nov 17, 2023. It is now read-only.

julia: migrate build process to cmake #16125

Open
wants to merge 14 commits into
base: master
Choose a base branch
from
Open

julia: migrate build process to cmake #16125

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
82a8b6e
julia: migrate build process to cmake
iblislin Sep 9, 2019
6f27f0e
build with Julia's private libopenblas
iblislin Sep 14, 2019
8c73cd9
submodule update
iblislin Sep 14, 2019
3ef9cf0
USE_JULIA_BLAS
iblislin Sep 14, 2019
27f996d
julia: migrate build process to cmake
iblislin Sep 9, 2019
46e7bd2
build with Julia's private libopenblas
iblislin Sep 14, 2019
cede75b
submodule update
iblislin Sep 14, 2019
b28b847
USE_JULIA_BLAS
iblislin Sep 14, 2019
4a2a9c9
Merge branch 'ib/jl-cmake-build' of github.com:apache/incubator-mxnet…
iblislin Dec 17, 2019
49fad87
submodule update
iblislin Dec 17, 2019
e5b4871
OPENBLAS_INTERFACE64
iblislin Dec 21, 2019
8779154
fix command quote warning
iblislin Dec 21, 2019
ef7a75e
Update julia/deps/build.jl
iblislin Dec 24, 2019
afe7bee
Update julia/deps/build.jl
iblislin Dec 24, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 22 additions & 3 deletions cmake/Modules/FindOpenBLAS.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,15 @@ SET(Open_BLAS_LIB_SEARCH_PATHS
/usr/local/opt/openblas/lib
${PROJECT_SOURCE_DIR}/3rdparty/OpenBLAS/lib
${PROJECT_SOURCE_DIR}/thirdparty/OpenBLAS/lib
${OpenBLAS_DIR}
${OpenBLAS_DIR}/lib
${OpenBLAS_DIR}
${OpenBLAS_DIR}/lib
${OpenBLAS_HOME}
${OpenBLAS_HOME}/lib
)

FIND_PATH(OpenBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Open_BLAS_INCLUDE_SEARCH_PATHS})
FIND_LIBRARY(OpenBLAS_LIB NAMES openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS})
# the Julia's private OpenBLAS is named as `libopenblas64_.so` on x86-64 Linux
FIND_LIBRARY(OpenBLAS_LIB NAMES openblas64_ openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS})
IF(NOT OpenBLAS_LIB)
FIND_FILE(OpenBLAS_LIB NAMES libopenblas.dll.a PATHS ${Open_BLAS_LIB_SEARCH_PATHS})
ENDIF()
Expand Down Expand Up @@ -89,3 +90,21 @@ MARK_AS_ADVANCED(
OpenBLAS
)

# Check ILP64 data model for the case of Julia self-shipped `libopenblas64_.so`
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could anyone review this cmake script?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know much, but maybe you can simplify a bit using CHECK_SOURCE_COMPILES and similar. https://gitlab.kitware.com/cmake/community/wikis/doc/tutorials/How-To-Write-Platform-Checks
Overall looks ok-ish to me.

Copy link
Member Author

@iblislin iblislin Dec 24, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah, looks greate, I will try

  • CHECK_SYMBOL_EXISTS

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any updates?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sorry, I'm swamped... I will try it after 1/10.

SET(detect_interface64_src "
#include <string.h>
char* openblas_get_config64_(void)\;
int main() {
return strstr(openblas_get_config64_(), \"USE64BITINT\") == NULL\;
}
")
FILE(WRITE "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c" ${detect_interface64_src})
TRY_RUN(
OpenBLAS_INTERFACE64 compile_detect_interface64
"${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c"
LINK_LIBRARIES ${OpenBLAS_LIB}
)
IF(OpenBLAS_INTERFACE64 EQUAL 0)
add_definitions(-DOPENBLAS_INTERFACE64=1) # see julia/deps/cblas.h
ENDIF(OpenBLAS_INTERFACE64 EQUAL 0)
FILE(REMOVE "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c")
1 change: 1 addition & 0 deletions julia/Project.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ version = "1.6.0"

[deps]
BinDeps = "9e28174c-4ba2-5203-b857-d8d62c4213ee"
CMake = "631607c0-34d2-5d66-819e-eb0f9aa2061a"
Formatting = "59287772-0a20-5a39-b81b-1366585eb4c0"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
Expand Down
160 changes: 48 additions & 112 deletions julia/deps/build.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.

using CMake
using JSON
using Libdl
using LinearAlgebra
Expand Down Expand Up @@ -93,19 +94,19 @@ else
end

# propagate more build flags from ENV
const CC = get(ENV, "CC", nothing)
const CXX = get(ENV, "CXX", nothing)
const ADD_CFLAGS = get(ENV, "ADD_CFLAGS", nothing)
const ADD_LDFLAGS = get(ENV, "ADD_LDFLAGS", nothing)
const USE_JEMALLOC = get(ENV, "USE_JEMALLOC", nothing) # "0" or "1"
const USE_JEMALLOC = get(ENV, "USE_JEMALLOC", nothing) # "ON" or "OFF"

function get_cpucore()
if haskey(ENV, "TRAVIS") # on travis-ci
2
else
min(Sys.CPU_THREADS, 32)
end
end
get_cpucore() = min(Sys.CPU_THREADS, 32)

cmake_bool(x::Bool) = ifelse(x ≡ true, "ON", "OFF")

cmake_jemalloc(::Nothing) = ""
cmake_jemalloc(x::Bool) = "-DUSE_JEMALLOC=" * cmake_bool(x)

cmake_cuda_path(::Nothing) = ""
cmake_cuda_path(x::String) = "-DUSE_CUDA_PATH=" * x

cmake_jl_blas(x::Bool, blas_path) = ifelse(x, "-DOpenBLAS_LIB=$blas_path", "")

using BinDeps
@BinDeps.setup
Expand Down Expand Up @@ -137,7 +138,7 @@ if !libmxnet_detected

run(download_cmd(package_url, "mxnet.7z"))
# this command will create the dir "usr\\lib"
run(`$exe7z e mxnet.7z *\\build\\* *\\lib\\* -y -ousr\\lib`)
run(`$exe7z e mxnet.7z "*\\build\\*" "*\\lib\\*" -y -ousr\\lib`) # TODO check it works on windows or not
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this imply we "might" break cmake build for windows? Can't we check this on windows before merging PR?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah, I checked it on Window, but forgot to remove the comment.
The previous code just poped some warning about quoting.


run(download_cmd(base_url, "mxnet_base.7z"))
run(`$exe7z x mxnet_base.7z -y -ousr`)
Expand All @@ -146,7 +147,7 @@ if !libmxnet_detected
# testing
run(`cmd /c dir "usr\\lib"`)
return
end
end # if Sys.iswindows()

################################################################################
# If not found, try to build automatically using BinDeps
Expand All @@ -155,131 +156,66 @@ if !libmxnet_detected
blas_path = Libdl.dlpath(Libdl.dlopen(Base.libblas_name))
blas_vendor = LinearAlgebra.BLAS.vendor()

ilp64 = ""
if blas_vendor == :openblas64
ilp64 = "-DINTERFACE64"
end
USE_JULIA_BLAS = (blas_vendor in (:openblas, :openblas64))
@info "USE_JULIA_BLAS -> $USE_JULIA_BLAS"

FORCE_LAPACK = false
if blas_vendor == :unknown
@info("Julia is built with an unkown blas library ($blas_path).")
@info("Attempting build without reusing the blas library")
USE_JULIA_BLAS = false
elseif !(blas_vendor in (:openblas, :openblas64))
@info("Unsure if we can build against $blas_vendor.")
@info("Attempting build anyway.")
USE_JULIA_BLAS = true
else
USE_JULIA_BLAS = true
FORCE_LAPACK = true
end
@info("USE_JULIA_BLAS -> $USE_JULIA_BLAS")

blas_name = blas_vendor == :openblas64 ? "openblas" : string(blas_vendor)
MSHADOW_LDFLAGS = "MSHADOW_LDFLAGS=-lm $blas_path"
blas_name = occursin("openblas", string(blas_vendor)) ? "open" : string(blas_vendor)

#--------------------------------------------------------------------------------
# Build libmxnet
mxnet = library_dependency("mxnet", aliases=["mxnet", "libmxnet", "libmxnet.so"])

_prefix = joinpath(BinDeps.depsdir(mxnet), "usr")
_blddir = joinpath(BinDeps.depsdir(mxnet), "build")
_srcdir = joinpath(BinDeps.depsdir(mxnet), "src")
_mxdir = joinpath(_srcdir, "mxnet")
_libdir = joinpath(_prefix, "lib")
# We have do eagerly delete the installed libmxnet.so

# We have do eagerly delete the build stuffs.
# Otherwise we won't rebuild on an update.
run(`rm -f $_libdir/libmxnet.$(Libdl.dlext)`)
rm(_blddir, recursive=true, force=true)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Previously it deleted the file libmxnet.* in the libdir
Now we are deleting the entire build directory. What changed with that?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

well, the final result is the same -- making a fresh build, not increamental build.

This build script is simliar to Python's setup.py.
It's for creating fresh build for user.


@debug "build dir -> $_blddir"

provides(BuildProcess,
(@build_steps begin
CreateDirectory(_blddir)
CreateDirectory(_srcdir)
CreateDirectory(_libdir)
@build_steps begin
BinDeps.DirectoryRule(_mxdir, @build_steps begin
ChangeDirectory(_srcdir)
`git clone https://github.com/apache/incubator-mxnet mxnet`
`git clone --recursive https://github.com/apache/incubator-mxnet mxnet`
end)
@build_steps begin
ChangeDirectory(_mxdir)
`git fetch`
`git submodule update --recursive --force`
if libmxnet_curr_ver != "master"
`git checkout $libmxnet_curr_ver`
else
`git checkout origin/$libmxnet_curr_ver`
end
`git submodule update --init --recursive`
`git -C 3rdparty/mshadow checkout -- make/mshadow.mk`
`cp -v ../../cblas.h include/cblas.h`
`sed -i -s "s/MSHADOW_CFLAGS = \(.*\)/MSHADOW_CFLAGS = \1 $ilp64/" 3rdparty/mshadow/make/mshadow.mk`

# Copy config.mk, always override the file
if Sys.isapple()
`cp make/osx.mk config.mk`
else
`cp make/config.mk config.mk`
end

# Configure OpenCV
`sed -i -s 's/USE_OPENCV = 1/USE_OPENCV = 0/' config.mk`

# Configure CUDA
if HAS_CUDA
@build_steps begin
`sed -i -s 's/USE_CUDA = 0/USE_CUDA = 1/' config.mk`
# address https://github.com/apache/incubator-mxnet/pull/7856
`sed -i -s "s/ADD_LDFLAGS =\(.*\)/ADD_LDFLAGS =\1 -lcublas -lcusolver -lcurand -lcudart/" config.mk`
if haskey(ENV, "CUDA_HOME")
`sed -i -s "s@USE_CUDA_PATH = NONE@USE_CUDA_PATH = $(ENV["CUDA_HOME"])@" config.mk`
end
if haskey(ENV, "CUDA_HOME")
# address https://github.com/apache/incubator-mxnet/pull/7838
flag = "-L$(ENV["CUDA_HOME"])/lib64 -L$(ENV["CUDA_HOME"])/lib"
`sed -i -s "s@ADD_LDFLAGS =\(.*\)@ADD_LDFLAGS =\1 $flag@" config.mk`
end
if HAS_CUDNN
`sed -i -s 's/USE_CUDNN = 0/USE_CUDNN = 1/' config.mk`
end
end
end

# Force enable LAPACK build
# Julia's OpenBLAS has LAPACK functionality already
if FORCE_LAPACK
if Sys.isapple()
MSHADOW_LDFLAGS *= " -framework Accelerate"
end
`sed -i -s 's/ADD_CFLAGS =\(.*\)/ADD_CFLAGS =\1 -DMXNET_USE_LAPACK/' config.mk`
end

# propagate more build flags from ENV
if CC != nothing
`sed -i -s "s@^export CC =\(.*\)@export CC = $CC@" config.mk`
end
if CXX != nothing
`sed -i -s "s@^export CXX =\(.*\)@export CXX = $CXX@" config.mk`
end
if ADD_CFLAGS != nothing
`sed -i -s "s@ADD_CFLAGS =\(.*\)@ADD_CFLAGS =\1 $ADD_CFLAGS@" config.mk`
end
if ADD_LDFLAGS != nothing
`sed -i -s "s@ADD_LDFLAGS =\(.*\)@ADD_LDFLAGS =\1 $ADD_LDFLAGS@" config.mk`
end
if USE_JEMALLOC != nothing
`sed -i -s "s@USE_JEMALLOC =\(.*\)@USE_JEMALLOC = $USE_JEMALLOC@" config.mk`
end

if USE_JULIA_BLAS
`make -j$(get_cpucore()) USE_BLAS=$blas_name $MSHADOW_LDFLAGS`
else
`make -j$(get_cpucore())`
end
`cp -f -v julia/deps/include/cblas.h include/cblas.h`
end
@build_steps begin
ChangeDirectory(_blddir)
`$cmake
-DCMAKE_BUILD_TYPE=$(libmxnet_curr_ver == "master" ? "Debug" : "Release")
-DUSE_BLAS=$blas_name
-DUSE_OPENCV=$(cmake_bool(false))
-DUSE_CUDA=$(cmake_bool(HAS_CUDA))
-DUSE_CUDNN=$(cmake_bool(HAS_CUDNN))
$(cmake_jemalloc(USE_JEMALLOC))
$(cmake_cuda_path(get(ENV, "CUDA_HOME", nothing)))
$(cmake_jl_blas(USE_JULIA_BLAS, blas_path))
$_mxdir`
`make -j$(get_cpucore()) VERBOSE=$(Int(libmxnet_curr_ver == "master"))`
end
FileRule(joinpath(_libdir, "libmxnet.$(Libdl.dlext)"), @build_steps begin
# the output file on macos is still in `.so` suffix
# so we rename it
`cp $_mxdir/lib/libmxnet.so $_libdir/libmxnet.$(Libdl.dlext)`
FileRule(joinpath(_blddir, "libmxnet.$(Libdl.dlext)"), @build_steps begin
# the output file on macos is still in `.so` suffix,
# so we create a soft link for it.
`ln -s libmxnet.so $_blddir/libmxnet.$(Libdl.dlext)`
end)
end
end), mxnet, installed_libpath=_libdir)
end), mxnet, installed_libpath=_blddir)

@BinDeps.install Dict(:mxnet => :mxnet)
end
6 changes: 3 additions & 3 deletions julia/deps/cblas.h → julia/deps/include/cblas.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ typedef long BLASLONG;
typedef unsigned long BLASULONG;
#endif

#ifdef INTERFACE64
#ifdef OPENBLAS_INTERFACE64
typedef BLASLONG blasint;
#else
typedef int blasint;
Expand All @@ -74,7 +74,7 @@ typedef int blasint;
typedef struct { double real, imag; } openblas_complex_double;
#endif

#ifdef INTERFACE64
#ifdef OPENBLAS_INTERFACE64
# define cblas_sdsdot cblas_sdsdot64_
# define cblas_dsdot cblas_dsdot64_
# define cblas_sdot cblas_sdot64_
Expand Down Expand Up @@ -240,7 +240,7 @@ typedef int blasint;
# define cblas_dgeadd cblas_dgeadd64_
# define cblas_cgeadd cblas_cgeadd64_
# define cblas_zgeadd cblas_zgeadd64_
#endif
#endif // OPENBLAS_INTERFACE64

#define CBLAS_INDEX size_t

Expand Down
9 changes: 5 additions & 4 deletions julia/docs/src/user-guide/install.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,12 @@ There are several environment variables that change this behaviour.
(e.g. `a0b1c2d3`).
- `CC`: The path of C compiler.
- `CXX`: The path of C++ compiler.
- `ADD_CFLAGS`: Additional C flags. For instance,
- `CFLAGS`: Additional C flags. For instance,
if you need to point non-standard include directory, please set it as
`ENV["ADD_CFLAGS"] = "-I'/path/to/include/dir'"`.
- `ADD_LDFLAGS`: Additional linker flags.
- `USE_JEMALLOC`: Default is enabled if jemalloc available.
`ENV["CFLAGS"] = "-I'/path/to/include/dir'"`.
- `LDFLAGS`: Additional linker flags.
- `USE_JEMALLOC`: Set it to `ON` or `OFF`.
Default is enabled if jemalloc available.
If you ran into segfault cause by jemalloc,
Please try to disable it.

Expand Down
43 changes: 30 additions & 13 deletions julia/src/base.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,23 +40,40 @@ const grad_req_map = Dict{Symbol,GRAD_REQ}(
################################################################################
# Initialization and library API entrance
################################################################################
const MXNET_LIB = Libdl.find_library(["libmxnet.$(Libdl.dlext)", "libmxnet.so"], # see build.jl
[joinpath(get(ENV, "MXNET_HOME", ""), "lib"),
get(ENV, "MXNET_HOME", ""),
joinpath(@__DIR__, "..",
"deps", "usr", "lib")])
const LIB_VERSION = Ref{Cint}(0)
function _get_search_names()
MXNET_LIBRARY_PATH = get(ENV, "MXNET_LIBRARY_PATH", "")
A = ["libmxnet.$(Libdl.dlext)", "libmxnet.so"] # see build.jl
if !isempty(MXNET_LIBRARY_PATH)
!isabspath(MXNET_LIBRARY_PATH) && error("MXNET_LIBRARY_PATH should be a absolute path")
pushfirst!(A, MXNET_LIBRARY_PATH)
end
A
end

function _get_search_dirs()
# TODO: remove MXNET_HOME backward compatibility in v2.0
if haskey(ENV, "MXNET_HOME")
@warn "The environment variable `MXNET_HOME` has been renamed, please use `MXNET_ROOT` instead."
end
A = [joinpath(@__DIR__, "..", "deps", "build")]
MXNET_ROOT = get(ENV, "MXNET_ROOT", get(ENV, "MXNET_HOME", ""))
if !isempty(MXNET_ROOT)
!isabspath(MXNET_ROOT) && error("MXNET_ROOT should be a absolute path")
prepend!(A, [joinpath(MXNET_ROOT, "lib"), MXNET_ROOT])
end
A
end

const MXNET_LIB = Libdl.find_library(_get_search_names(), _get_search_dirs())
const LIB_VERSION = Ref{Cint}(C_NULL)

if isempty(MXNET_LIB)
# touch this file, so that after the user properly build libmxnet, the precompiled
# MXNet.ji will be re-compiled to get MXNET_LIB properly.
touch(@__FILE__)
error("Cannot find or load libmxnet.$(Libdl.dlext). " *
"Please see the document on how to build it.")
else
include_dependency(MXNET_LIB)
end

include_dependency(MXNET_LIB)

function __init__()
# TODO: bug in nnvm, if do not call this, call get handle "_copyto" will fail
_get_libmx_op_names()
Expand All @@ -65,12 +82,12 @@ function __init__()

atexit() do
# notify libmxnet we are shutting down
ccall( ("MXNotifyShutdown", MXNET_LIB), Cint, () )
ccall(("MXNotifyShutdown", MXNET_LIB), Cint, ())
end
end

function mx_get_last_error()
msg = ccall( ("MXGetLastError", MXNET_LIB), char_p, () )
msg = ccall(("MXGetLastError", MXNET_LIB), char_p, ())
if msg == C_NULL
throw(MXError("Failed to get last error message"))
end
Expand Down