diff --git a/CMakeLists.txt b/CMakeLists.txt index 483108a68419..000bbbf17ea5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -372,13 +372,13 @@ endif() # ---[ LAPack if(USE_LAPACK) + message("USE_LAPACK is ON") add_definitions(-DMXNET_USE_LAPACK=1) if (NOT MSVC) list(APPEND mxnet_LINKER_LIBS lapack) endif() endif() -message("USE LAPACK ${USE_LAPACK}") # ---[ jemalloc if(USE_JEMALLOC) diff --git a/Jenkinsfile b/Jenkinsfile index 6d21f496426e..95eeaa9da073 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -378,28 +378,8 @@ try { ws('workspace/build-cpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { init_git_win() - bat """mkdir build_vc14_cpu - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_vc14_cpu - cmake -G \"Visual Studio 14 2015 Win64\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DUSE_MKL_IF_AVAILABLE=0 ${env.WORKSPACE}""" - bat 'C:\\mxnet\\build_vc14_cpu.bat' - - bat '''rmdir /s/q pkg_vc14_cpu - mkdir pkg_vc14_cpu\\lib - mkdir pkg_vc14_cpu\\python - mkdir pkg_vc14_cpu\\include - mkdir pkg_vc14_cpu\\build - copy build_vc14_cpu\\Release\\libmxnet.lib pkg_vc14_cpu\\lib - copy build_vc14_cpu\\Release\\libmxnet.dll pkg_vc14_cpu\\build - xcopy python pkg_vc14_cpu\\python /E /I /Y - xcopy include pkg_vc14_cpu\\include /E /I /Y - xcopy 3rdparty\\dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y - xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y - xcopy 3rdparty\\nnvm\\include pkg_vc14_cpu\\nnvm\\include /E /I /Y - del /Q *.7z - 7z.exe a vc14_cpu.7z pkg_vc14_cpu\\ - ''' - stash includes: 'vc14_cpu.7z', name: 'vc14_cpu' + powershell 'python ci/build_windows.py -f WIN_CPU' + stash includes: 'windows_package.7z', name: 'windows_package_cpu' } } } @@ -411,28 +391,9 @@ try { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-gpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { - init_git_win() - bat """mkdir build_vc14_gpu - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_vc14_gpu - cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN=52 -DCUDA_ARCH_PTX=52 -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release -DUSE_MKL_IF_AVAILABLE=0 ${env.WORKSPACE}""" - bat 'C:\\mxnet\\build_vc14_gpu.bat' - bat '''rmdir /s/q pkg_vc14_gpu - mkdir pkg_vc14_gpu\\lib - mkdir pkg_vc14_gpu\\python - mkdir pkg_vc14_gpu\\include - mkdir pkg_vc14_gpu\\build - copy build_vc14_gpu\\libmxnet.lib pkg_vc14_gpu\\lib - copy build_vc14_gpu\\libmxnet.dll pkg_vc14_gpu\\build - xcopy python pkg_vc14_gpu\\python /E /I /Y - xcopy include pkg_vc14_gpu\\include /E /I /Y - xcopy 3rdparty\\dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y - xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y - xcopy 3rdparty\\nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y - del /Q *.7z - 7z.exe a vc14_gpu.7z pkg_vc14_gpu\\ - ''' - stash includes: 'vc14_gpu.7z', name: 'vc14_gpu' + init_git_win() + powershell 'python ci/build_windows.py -f WIN_GPU' + stash includes: 'windows_package.7z', name: 'windows_package_gpu' } } } @@ -443,37 +404,9 @@ try { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-gpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) { - init_git_win() - bat """mkdir build_%BUILD_NAME% - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_%BUILD_NAME% - copy ${env.WORKSPACE}\\3rdparty\\mkldnn\\config_template.vcxproj.user ${env.WORKSPACE}\\config_template.vcxproj.user /y - cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN=52 -DCUDA_ARCH_PTX=52 -DUSE_MKLDNN=1 -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}""" - bat ''' - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_%BUILD_NAME% - set /a cores=%NUMBER_OF_PROCESSORS% * 2 - jom -j %cores% - ''' - bat '''rmdir /s/q pkg_%BUILD_NAME% - mkdir pkg_%BUILD_NAME%\\lib - mkdir pkg_%BUILD_NAME%\\python - mkdir pkg_%BUILD_NAME%\\include - mkdir pkg_%BUILD_NAME%\\build - copy build_%BUILD_NAME%\\libmxnet.lib pkg_%BUILD_NAME%\\lib - copy build_%BUILD_NAME%\\libmxnet.dll pkg_%BUILD_NAME%\\build - copy build_%BUILD_NAME%\\3rdparty\\mkldnn\\src\\mkldnn.dll pkg_%BUILD_NAME%\\build - copy build_%BUILD_NAME%\\libiomp5md.dll pkg_%BUILD_NAME%\\build - copy build_%BUILD_NAME%\\mklml.dll pkg_%BUILD_NAME%\\build - xcopy python pkg_%BUILD_NAME%\\python /E /I /Y - xcopy include pkg_%BUILD_NAME%\\include /E /I /Y - xcopy 3rdparty\\dmlc-core\\include pkg_%BUILD_NAME%\\include /E /I /Y - xcopy 3rdparty\\mshadow\\mshadow pkg_%BUILD_NAME%\\include\\mshadow /E /I /Y - xcopy 3rdparty\\nnvm\\include pkg_%BUILD_NAME%\\nnvm\\include /E /I /Y - del /Q *.7z - 7z.exe a %BUILD_NAME%.7z pkg_%BUILD_NAME%\\ - ''' - stash includes: 'vc14_gpu_mkldnn.7z', name: 'vc14_gpu_mkldnn' + init_git_win() + powershell 'python ci/build_windows.py -f WIN_GPU_MKLDNN' + stash includes: 'windows_package.7z', name: 'windows_package_gpu_mkldnn' } } } @@ -843,16 +776,8 @@ try { ws('workspace/ut-python-cpu') { try { init_git_win() - unstash 'vc14_cpu' - bat '''rmdir /s/q pkg_vc14_cpu - 7z x -y vc14_cpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py2 - pip install mock - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc - C:\\mxnet\\test_cpu.bat""" + unstash 'windows_package_cpu' + powershell 'ci/windows/test_py2_cpu.ps1' } finally { collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python2_cpu.xml') } @@ -866,15 +791,8 @@ try { ws('workspace/ut-python-cpu') { try { init_git_win() - unstash 'vc14_cpu' - bat '''rmdir /s/q pkg_vc14_cpu - 7z x -y vc14_cpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py3 - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc - C:\\mxnet\\test_cpu.bat""" + unstash 'windows_package_cpu' + powershell 'ci/windows/test_py3_cpu.ps1' } finally { collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python3_cpu.xml') } @@ -888,16 +806,7 @@ try { ws('workspace/ut-python-gpu') { try { init_git_win() - unstash 'vc14_gpu' - bat '''rmdir /s/q pkg_vc14_gpu - 7z x -y vc14_gpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py2 - pip install mock - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc - C:\\mxnet\\test_gpu.bat""" + powershell 'ci/windows/test_py2_gpu.ps1' } finally { collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml') collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml') @@ -912,15 +821,8 @@ try { ws('workspace/ut-python-gpu') { try { init_git_win() - unstash 'vc14_gpu' - bat '''rmdir /s/q pkg_vc14_gpu - 7z x -y vc14_gpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py3 - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc - C:\\mxnet\\test_gpu.bat""" + unstash 'windows_package_gpu' + powershell 'ci/windows/test_py3_gpu.ps1' } finally { collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml') collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml') @@ -935,15 +837,8 @@ try { ws('workspace/ut-python-gpu') { try { init_git_win() - unstash 'vc14_gpu_mkldnn' - bat '''rmdir /s/q pkg_vc14_gpu_mkldnn - 7z x -y vc14_gpu_mkldnn.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py3 - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu_mkldnn\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu_mkldnn\\python\\*.pyc - C:\\mxnet\\test_gpu.bat""" + unstash 'windows_package_gpu_mkldnn' + powershell 'ci/windows/test_py3_gpu.ps1' } finally { collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu_mkldnn.xml') collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu_mkldnn.xml') diff --git a/ci/build.py b/ci/build.py index 09f2d4709bdd..a9d6a63537f2 100755 --- a/ci/build.py +++ b/ci/build.py @@ -39,6 +39,7 @@ from itertools import chain from subprocess import call, check_call from typing import * +from util import * CCACHE_MAXSIZE = '500G' @@ -138,24 +139,9 @@ def _get_local_image_id(docker_binary, docker_tag): return image_id -def get_mxnet_root() -> str: - curpath = os.path.abspath(os.path.dirname(__file__)) - - def is_mxnet_root(path: str) -> bool: - return os.path.exists(os.path.join(path, ".mxnet_root")) - - while not is_mxnet_root(curpath): - parent = os.path.abspath(os.path.join(curpath, os.pardir)) - if parent == curpath: - raise RuntimeError("Got to the root and couldn't find a parent folder with .mxnet_root") - curpath = parent - return curpath - - def buildir() -> str: return os.path.join(get_mxnet_root(), "build") - def default_ccache_dir() -> str: # Share ccache across containers if 'CCACHE_DIR' in os.environ: diff --git a/ci/build_windows.py b/ci/build_windows.py new file mode 100755 index 000000000000..433936278150 --- /dev/null +++ b/ci/build_windows.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""User friendly / multi platform builder script""" + +import subprocess +import logging +import os +import tempfile +import sys +from distutils import spawn +import logging +from subprocess import check_call +import platform +import argparse +from util import * +import json +from enum import Enum +import time +import datetime +import shutil +import glob +from distutils.dir_util import copy_tree + +KNOWN_VCVARS = [ + # VS 2015 + r'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat' + # VS 2017 + , r'c:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat' +] + +class BuildFlavour(Enum): + WIN_CPU = 'WIN_CPU' + WIN_CPU_MKLDNN = 'WIN_CPU_MKLDNN' + WIN_GPU = 'WIN_GPU' + WIN_GPU_MKLDNN = 'WIN_GPU_MKLDNN' + +CMAKE_FLAGS = { + 'WIN_CPU': '-DUSE_CUDA=0 \ + -DUSE_CUDNN=0 \ + -DUSE_NVRTC=0 \ + -DUSE_OPENCV=1 \ + -DUSE_OPENMP=1 \ + -DUSE_PROFILER=1 \ + -DUSE_BLAS=open \ + -DUSE_LAPACK=1 \ + -DUSE_DIST_KVSTORE=0 \ + -DBUILD_CPP_EXAMPLES=1 \ + -DUSE_MKL_IF_AVAILABLE=0' + + ,'WIN_CPU_MKLDNN': '-DUSE_CUDA=0 \ + -DUSE_CUDNN=0 \ + -DUSE_NVRTC=0 \ + -DUSE_OPENCV=1 \ + -DUSE_OPENMP=1 \ + -DUSE_PROFILER=1 \ + -DUSE_BLAS=open \ + -DUSE_LAPACK=1 \ + -DUSE_DIST_KVSTORE=0 \ + -DUSE_MKL_IF_AVAILABLE=1' + + ,'WIN_GPU': '-DUSE_CUDA=1 \ + -DUSE_CUDNN=1 \ + -DUSE_NVRTC=1 \ + -DUSE_OPENCV=1 \ + -DUSE_OPENMP=1 \ + -DUSE_PROFILER=1 \ + -DUSE_BLAS=open \ + -DUSE_LAPACK=1 \ + -DUSE_DIST_KVSTORE=0 \ + -DCUDA_ARCH_NAME=Manual \ + -DCUDA_ARCH_BIN=52 \ + -DCUDA_ARCH_PTX=52 \ + -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" \ + -DUSE_MKL_IF_AVAILABLE=0 \ + -DCMAKE_BUILD_TYPE=Release' + + ,'WIN_GPU_MKLDNN': '-DUSE_CUDA=1 \ + -DUSE_CUDNN=1 \ + -DUSE_NVRTC=1 \ + -DUSE_OPENCV=1 \ + -DUSE_OPENMP=1 \ + -DUSE_PROFILER=1 \ + -DUSE_BLAS=open \ + -DUSE_LAPACK=1 \ + -DUSE_DIST_KVSTORE=0 \ + -DCUDA_ARCH_NAME=Manual \ + -DCUDA_ARCH_BIN=52 \ + -DCUDA_ARCH_PTX=52 \ + -DUSE_MKLDNN=1 \ + -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 \ + /DNDEBUG" \ + -DCMAKE_BUILD_TYPE=Release' + +} + + +def get_vcvars_environment(architecture, vcvars): + """ + Returns a dictionary containing the environment variables set up by vcvars + """ + result = None + python = sys.executable + + vcvars_list = [vcvars] + vcvars_list.extend(KNOWN_VCVARS) + for vcvars in vcvars_list: + if os.path.isfile(vcvars): + process = subprocess.Popen('("%s" %s>nul) && "%s" -c "import os; import json; print(json.dumps(dict(os.environ)))"' % (vcvars, architecture, python), stdout=subprocess.PIPE, shell=True) + stdout, stderr = process.communicate() + exitcode = process.wait() + if exitcode == 0: + logging.info("Using build environment from: %s", vcvars) + return(json.loads(stdout.strip())) + else: + raise RuntimeError('Failed cloning environment from vcvars file: %s stdout: %s stderr: %s', vcvars, stdout, stderr) + raise RuntimeError('Couldn\'t find vcvars batch file: %s', vcvars) + + +def windows_build(args): + vcvars_env = get_vcvars_environment(args.arch, args.vcvars) + logging.debug("vcvars environment: %s", vcvars_env) + os.environ.update(vcvars_env) + + path = args.output + os.makedirs(path, exist_ok=True) + mxnet_root = get_mxnet_root() + logging.info("Found mxnet root: {}".format(mxnet_root)) + with remember_cwd(): + os.chdir(path) + logging.info("Generating project with CMake") + check_call("cmake -G \"Visual Studio 14 2015 Win64\" {} {}".format(CMAKE_FLAGS[args.flavour], mxnet_root), shell=True) + logging.info("Building with visual studio") + t0 = int(time.time()) + check_call(["msbuild", "mxnet.sln","/p:configuration=release;platform=x64"]) + logging.info("Build flavour: %s complete in directory: \"%s\"", args.flavour, os.path.abspath(path)) + logging.info("Build took %s" , datetime.timedelta(seconds=int(time.time()-t0))) + windows_package(args) + +def windows_package(args): + pkgfile = 'windows_package.7z' + pkgdir = os.path.abspath('windows_package') + logging.info("Packaging libraries and headers in package: %s", pkgfile) + j = os.path.join + pkgdir_lib = os.path.abspath(j(pkgdir, 'lib')) + with remember_cwd(): + os.chdir(args.output) + logging.info("Looking for static libraries and dlls in: \"%s", os.getcwd()) + libs = list(glob.iglob('**/*.lib', recursive=True)) + dlls = list(glob.iglob('**/*.dll', recursive=True)) + os.makedirs(pkgdir_lib, exist_ok=True) + for lib in libs: + logging.info("packing lib: %s", lib) + shutil.copy(lib, pkgdir_lib) + for dll in dlls: + logging.info("packing dll: %s", dll) + shutil.copy(lib, pkgdir_lib) + os.chdir(get_mxnet_root()) + logging.info('packing python bindings') + copy_tree('python', j(pkgdir, 'python')) + logging.info('packing headers') + copy_tree('include', j(pkgdir, 'include')) + copy_tree(j('3rdparty','dmlc-core','include'), j(pkgdir, 'include')) + copy_tree(j('3rdparty','mshadow', 'mshadow'), j(pkgdir, 'include', 'mshadow')) + copy_tree(j('3rdparty','tvm','nnvm', 'include'), j(pkgdir,'include', 'nnvm', 'include')) + logging.info("Compressing package: %s", pkgfile) + check_call(['7z', 'a', pkgfile, pkgdir]) + + +def nix_build(args): + path = args.output + os.makedirs(path, exist_ok=True) + with remember_cwd(): + os.chdir(path) + logging.info("Generating project with CMake") + check_call("cmake \ + -DUSE_CUDA=OFF \ + -DUSE_OPENCV=OFF \ + -DUSE_OPENMP=OFF \ + -DCMAKE_BUILD_TYPE=Debug \ + -GNinja ..", shell=True) + check_call("ninja", shell=True) + +def main(): + logging.getLogger().setLevel(logging.INFO) + logging.basicConfig(format='%(asctime)-15s %(message)s') + logging.info("MXNet Windows build helper") + + parser = argparse.ArgumentParser() + parser.add_argument("-o", "--output", + help="output directory", + default='build', + type=str) + + parser.add_argument("--vcvars", + help="vcvars batch file location, typically inside vs studio install dir", + default=r'c:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat', + type=str) + + parser.add_argument("--arch", + help="architecture", + default='x64', + type=str) + + parser.add_argument("-f", "--flavour", + help="build flavour", + default='WIN_CPU', + choices=[x.name for x in BuildFlavour], + type=str) + + args = parser.parse_args() + logging.info("Build flavour: %s", args.flavour) + + system = platform.system() + if system == 'Windows': + logging.info("Detected Windows platform") + if 'OpenBLAS_HOME' not in os.environ: + os.environ["OpenBLAS_HOME"] = "C:\\mxnet\\openblas" + if 'OpenCV_DIR' not in os.environ: + os.environ["OpenCV_DIR"] = "C:\\mxnet\\opencv_vc14" + if 'CUDA_PATH' not in os.environ: + os.environ["CUDA_PATH"] = "C:\\CUDA\\v8.0" + windows_build(args) + + elif system == 'Linux' or system == 'Darwin': + nix_build(args) + + else: + logging.error("Don't know how to build for {} yet".format(platform.system())) + + return 0 + + +if __name__ == '__main__': + sys.exit(1) + sys.exit(main()) + diff --git a/ci/util.py b/ci/util.py new file mode 100644 index 000000000000..22631f30435f --- /dev/null +++ b/ci/util.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import contextlib + +def get_mxnet_root() -> str: + curpath = os.path.abspath(os.path.dirname(__file__)) + + def is_mxnet_root(path: str) -> bool: + return os.path.exists(os.path.join(path, ".mxnet_root")) + + while not is_mxnet_root(curpath): + parent = os.path.abspath(os.path.join(curpath, os.pardir)) + if parent == curpath: + raise RuntimeError("Got to the root and couldn't find a parent folder with .mxnet_root") + curpath = parent + return curpath + +@contextlib.contextmanager +def remember_cwd(): + ''' + Restore current directory when exiting context + ''' + curdir = os.getcwd() + try: yield + finally: os.chdir(curdir) + + diff --git a/ci/windows/test_py2_cpu.ps1 b/ci/windows/test_py2_cpu.ps1 new file mode 100644 index 000000000000..bd1e98ca6ded --- /dev/null +++ b/ci/windows/test_py2_cpu.ps1 @@ -0,0 +1,7 @@ +7z x -y windows_package.7z +$env:PYTHONPATH=join-path $pwd.Path windows_package\python +$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 +c:\Anaconda3\envs\py2\python.exe -m nose -v tests\python\unittest +if (! $?) { Throw ("Error running unittest") } +c:\Anaconda3\envs\py2\python.exe -m nose -v tests\python\train +if (! $?) { Throw ("Error running train tests") } diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1 new file mode 100644 index 000000000000..ff00c96b0348 --- /dev/null +++ b/ci/windows/test_py2_gpu.ps1 @@ -0,0 +1,11 @@ +7z x -y windows_package.7z +$env:PYTHONPATH=join-path $pwd.Path windows_package\python +$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 +c:\Anaconda3\envs\py2\python.exe -m nose -v tests\python\unittest +if (! $?) { Throw ("Error running unittest") } +c:\Anaconda3\envs\py2\python.exe -m nose -v tests\python\gpu\test_operator_gpu.py +if (! $?) { Throw ("Error running tests") } +c:\Anaconda3\envs\py2\python.exe -m nose -v tests\python\gpu\test_forward.py +if (! $?) { Throw ("Error running tests") } +c:\Anaconda3\envs\py2\python.exe -m nose -v tests\python\train +if (! $?) { Throw ("Error running tests") } diff --git a/ci/windows/test_py3_cpu.ps1 b/ci/windows/test_py3_cpu.ps1 new file mode 100644 index 000000000000..8ca0cc41b427 --- /dev/null +++ b/ci/windows/test_py3_cpu.ps1 @@ -0,0 +1,7 @@ +7z x -y windows_package.7z +$env:PYTHONPATH=join-path $pwd.Path windows_package\python +$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 +c:\Anaconda3\envs\py3\python.exe -m nose -v tests\python\unittest +if (! $?) { Throw ("Error running unittest") } +c:\Anaconda3\envs\py3\python.exe -m nose -v tests\python\train +if (! $?) { Throw ("Error running train tests") } diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1 new file mode 100644 index 000000000000..6a6ca7d3881b --- /dev/null +++ b/ci/windows/test_py3_gpu.ps1 @@ -0,0 +1,11 @@ +7z x -y windows_package.7z +$env:PYTHONPATH=join-path $pwd.Path windows_package\python +$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 +c:\Anaconda3\envs\py3\python.exe -m nose -v tests\python\unittest +if (! $?) { Throw ("Error running unittest") } +c:\Anaconda3\envs\py3\python.exe -m nose -v tests\python\gpu\test_operator_gpu.py +if (! $?) { Throw ("Error running tests") } +c:\Anaconda3\envs\py3\python.exe -m nose -v tests\python\gpu\test_forward.py +if (! $?) { Throw ("Error running tests") } +c:\Anaconda3\envs\py3\python.exe -m nose -v tests\python\train +if (! $?) { Throw ("Error running tests") }