From 9d2c4b6747357a93336763251a5d27b4f23b1329 Mon Sep 17 00:00:00 2001 From: Marco de Abreu Date: Wed, 8 Aug 2018 14:45:48 +0200 Subject: [PATCH] Address review comments 2 --- Jenkinsfile | 36 +++++++++---------- ci/Jenkinsfile_docker_cache | 2 +- ci/Jenkinsfile_utils.groovy | 20 ++++++----- docs/Jenkinsfile | 4 +-- tests/nightly/Jenkinsfile | 2 +- tests/nightly/JenkinsfileForBinaries | 2 +- .../JenkinsfileForBLC | 2 +- .../JenkinsfileForMBCC | 2 +- 8 files changed, 36 insertions(+), 34 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 82a9907eebf4..d74f0b43c212 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,7 +86,7 @@ node('mxnetlinux-cpu') { utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage('Sanity Check') { parallel 'Lint': { node(NODE_LINUX_CPU) { @@ -113,7 +113,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('centos7_cpu', 'build_centos7_cpu', false) - utils.pack_lib('centos7_cpu') + utils.pack_lib('centos7_cpu', mx_lib) } } } @@ -124,7 +124,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('centos7_cpu', 'build_centos7_mkldnn', false) - utils.pack_lib('centos7_mkldnn') + utils.pack_lib('centos7_mkldnn', mx_lib) } } } @@ -135,7 +135,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('centos7_gpu', 'build_centos7_gpu', false) - utils.pack_lib('centos7_gpu') + utils.pack_lib('centos7_gpu', mx_lib) } } } @@ -410,7 +410,7 @@ handler: { ws('workspace/ut-python2-cpu') { try { utils.init_git() - utils.unpack_lib('cpu') + utils.unpack_lib('cpu', mx_lib) python2_ut('ubuntu_cpu') utils.publish_test_coverage() } finally { @@ -426,7 +426,7 @@ handler: { ws('workspace/ut-python3-cpu') { try { utils.init_git() - utils.unpack_lib('cpu') + utils.unpack_lib('cpu', mx_lib) python3_ut('ubuntu_cpu') utils.publish_test_coverage() } finally { @@ -589,7 +589,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { try { utils.init_git() - utils.unpack_lib('centos7_cpu') + utils.unpack_lib('centos7_cpu', mx_lib) utils.docker_run('centos7_cpu', 'unittest_centos7_cpu', false) utils.publish_test_coverage() } finally { @@ -606,7 +606,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { try { utils.init_git() - utils.unpack_lib('centos7_gpu') + utils.unpack_lib('centos7_gpu', mx_lib) utils.docker_run('centos7_gpu', 'unittest_centos7_gpu', true) utils.publish_test_coverage() } finally { @@ -645,7 +645,7 @@ handler: { ws('workspace/ut-perl-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('cpu') + utils.unpack_lib('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpugpu_perl', false) utils.publish_test_coverage() } @@ -657,7 +657,7 @@ handler: { ws('workspace/ut-perl-gpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('gpu') + utils.unpack_lib('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true) utils.publish_test_coverage() } @@ -693,7 +693,7 @@ handler: { ws('workspace/ut-r-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('cpu') + utils.unpack_lib('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_R', false) utils.publish_test_coverage() } @@ -705,7 +705,7 @@ handler: { ws('workspace/ut-r-gpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('gpu') + utils.unpack_lib('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_R', true) utils.publish_test_coverage() } @@ -796,7 +796,7 @@ handler: { ws('workspace/it-onnx-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('cpu') + utils.unpack_lib('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx', false) utils.publish_test_coverage() } @@ -808,7 +808,7 @@ handler: { ws('workspace/it-python-gpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('gpu') + utils.unpack_lib('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_python', true) utils.publish_test_coverage() } @@ -821,7 +821,7 @@ handler: { // ws('workspace/it-caffe') { // timeout(time: max_time, unit: 'MINUTES') { // utils.init_git() - // utils.unpack_lib('gpu') + // utils.unpack_lib('gpu', mx_lib) // utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_caffe', true) // utils.publish_test_coverage() // } @@ -833,7 +833,7 @@ handler: { ws('workspace/it-cpp-package') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('gpu') + utils.unpack_lib('gpu', mx_lib) unstash 'cpp_lenet' unstash 'cpp_alexnet' unstash 'cpp_googlenet' @@ -855,7 +855,7 @@ handler: { ws('workspace/it-dist-kvstore') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('gpu') + utils.unpack_lib('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_dist_kvstore', true) utils.publish_test_coverage() } @@ -871,7 +871,7 @@ handler: { ws('workspace/it-dist-kvstore') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('cpu') + utils.unpack_lib('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_dist_kvstore', false) utils.publish_test_coverage() } diff --git a/ci/Jenkinsfile_docker_cache b/ci/Jenkinsfile_docker_cache index bee769a0a321..e8556c625ff4 100644 --- a/ci/Jenkinsfile_docker_cache +++ b/ci/Jenkinsfile_docker_cache @@ -31,7 +31,7 @@ node('restricted-mxnetlinux-cpu') { utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage("Docker cache build & publish") { node(NODE_LINUX_CPU) { ws('workspace/docker_cache') { diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy index 3892ccc5173d..c9f917d6c0ff 100644 --- a/ci/Jenkinsfile_utils.groovy +++ b/ci/Jenkinsfile_utils.groovy @@ -57,7 +57,7 @@ def init_git_win() { } // pack libraries for later use -def pack_lib(name, libs=mx_lib) { +def pack_lib(name, libs) { sh """ echo "Packing ${libs} into ${name}" echo ${libs} | sed -e 's/,/ /g' | xargs md5sum @@ -66,7 +66,7 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum } // unpack libraries saved before -def unpack_lib(name, libs=mx_lib) { +def unpack_lib(name, libs) { unstash name sh """ echo "Unpacked ${libs} from ${name}" @@ -118,15 +118,17 @@ def assign_node_labels(args) { NODE_WINDOWS_GPU = args.windows_gpu } -// assign any caught errors here -err = null def main_wrapper(args) { - // hander: Core logic - // failure_handler: Failure handler - + // Main Jenkinsfile pipeline wrapper handler that allows to wrap core logic into a format + // that supports proper failure handling + // args: + // - core_logic: Jenkins pipeline containing core execution logic + // - failure_handler: Failure handler + + // assign any caught errors here + err = null try { - // Call actual handler - args['handler']() + args['core_logic']() // set build status to success at the end currentBuild.result = "SUCCESS" diff --git a/docs/Jenkinsfile b/docs/Jenkinsfile index 172aaffb5d8c..2431825c97ca 100644 --- a/docs/Jenkinsfile +++ b/docs/Jenkinsfile @@ -31,7 +31,7 @@ node('restricted-mxnetlinux-cpu') { utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage('Build Docs') { node(NODE_LINUX_CPU) { ws('workspace/docs') { @@ -50,4 +50,4 @@ failure_handler: { emailext body: 'Generating the website has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[WEBSITE FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}' } } -) \ No newline at end of file +) diff --git a/tests/nightly/Jenkinsfile b/tests/nightly/Jenkinsfile index 9f2a4000d2a3..b8debb21344f 100755 --- a/tests/nightly/Jenkinsfile +++ b/tests/nightly/Jenkinsfile @@ -28,7 +28,7 @@ node('mxnetlinux-cpu') { utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage('NightlyTests'){ parallel 'CompilationWarnings: CPU': { node(NODE_LINUX_CPU) { diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index 6086c9f7d87f..d3454dc6832b 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -28,7 +28,7 @@ node('mxnetlinux-cpu) { utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage('Build') { parallel 'GPU: CUDA9.1+cuDNN7': { node(NODE_LINUX_CPU) { diff --git a/tests/nightly/broken_link_checker_test/JenkinsfileForBLC b/tests/nightly/broken_link_checker_test/JenkinsfileForBLC index 185708d0875a..6e859ed6bf92 100755 --- a/tests/nightly/broken_link_checker_test/JenkinsfileForBLC +++ b/tests/nightly/broken_link_checker_test/JenkinsfileForBLC @@ -26,7 +26,7 @@ node('mxnetlinux-cpu') { utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage('BLC'){ parallel 'BrokenLinkChecker: CPU': { node(NODE_LINUX_CPU) { diff --git a/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC b/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC index 7f0494c442c5..c3fe4fd4cd89 100644 --- a/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC +++ b/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC @@ -28,7 +28,7 @@ node('restricted-mxnetlinux-cpu') { utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage('MBCC Train'){ node(NODE_LINUX_CPU) { ws('workspace/modelBackwardsCompat') {