diff --git a/ci_jobs/Jenkinsfile b/ci_jobs/Jenkinsfile index 99941ff12919..dea0db8c3c9f 100644 --- a/ci_jobs/Jenkinsfile +++ b/ci_jobs/Jenkinsfile @@ -2,20 +2,46 @@ library('sima-jenkins-lib') DOCKER_OPTS = '-m 32g --cpus 8 -v /jenkins/workspace/ref_repos:/jenkins/workspace/ref_repos:rw,z' +NEXUS_DOCKER_DEV = "https://${env.NEXUS_URL}:5000" +NEXUS_DOCKER_STAGING = "https://${env.NEXUS_URL}:5300" + +/* promote - promote artifacts to staging repo on tagged builds + * + */ +def promote(dev_image) { + if (utils.isTagBuild()) { + node('docker') { + docker.withRegistry(NEXUS_DOCKER_DEV, "jenkins_user") { + dev_image["image"].pull() + dev_image["image"].inside(DOCKER_OPTS) { + + stage("Promote") { + unstash('dist') + withEnv(["HOME=${env.WORKSPACE}"]) { + utils.installNexusCreds('jenkins_user') + utils.uploadPythonPackages('jenkins_user', 'sima-staging', 'python/dist/*.whl') + } + } + } + } + } + } +} def main() { def job_name = env.JOB_NAME.split('/')[1] + def image properties([ parameters([ booleanParam( name: 'SKIP_DOWNSTREAM_BUILDS', description: 'Skips building upstream jobs', - defaultValue: false + defaultValue: utils.isTagBuild() ), booleanParam( name: "PACKAGE_ONLY", - defaultValue: false, + defaultValue: utils.isTagBuild(), description: 'Only package don\'t run tests' ) ]), @@ -27,7 +53,6 @@ def main() { utils.setBuildMetadataFromVersionIn("python/VERSION.in") } - def image stage("DockerBuild") { image = utils.dockerBuild( "docker/Dockerfile", @@ -35,7 +60,9 @@ def main() { "jenkins_user", "docker_build.log", {}, - "https://${env.NEXUS_URL}:5000" + "https://${env.NEXUS_URL}:5000", + "120", + "20", ) } @@ -55,12 +82,18 @@ python3 setup.py bdist_wheel """ }, 'sima') } + dir("${env.WORKSPACE}") { + stash(name: 'dist', includes: 'python/dist/*.whl') + } } }, "../sima-regres.cmake", "clean all") stage("Package") { tvm_pkg_dir = "python/dist/*.whl" archiveArtifacts(tvm_pkg_dir) - utils.uploadPythonPackages('jenkins_user', 'sima-pypi', tvm_pkg_dir, 3) + withEnv(["HOME=${env.WORKSPACE}"]) { + utils.installNexusCreds('jenkins_user') + utils.uploadPythonPackages('jenkins_user', 'sima-pypi', tvm_pkg_dir, 3) + } } } } @@ -79,9 +112,13 @@ python3 setup.py bdist_wheel booleanParam(name: 'PACKAGE_ONLY', value: params.PACKAGE_ONLY) ]) }, psim_mla_flow: { - utils.buildUpstream("psim_mla_flow", params.SKIP_UPSTREAM_BUILDS, []) + utils.buildUpstream("psim_mla_flow", params.SKIP_DOWNSTREAM_BUILDS, [ + booleanParam(name: 'PACKAGE_ONLY', value: params.PACKAGE_ONLY) + ]) } } + + promote(image) } utils.job_wrapper( { diff --git a/ci_jobs/Jenkinsfile.release b/ci_jobs/Jenkinsfile.release index f55aa5ad23b1..3c1e8349deba 100644 --- a/ci_jobs/Jenkinsfile.release +++ b/ci_jobs/Jenkinsfile.release @@ -1,63 +1,127 @@ -#!/usr/bin/env groovy +#!groovy library('sima-jenkins-lib') DOCKER_OPTS = '-m 32g --cpus 8 -v /jenkins/workspace/ref_repos:/jenkins/workspace/ref_repos:rw,z' +NEXUS_DOCKER_DEV = "https://${env.NEXUS_URL}:5000" +NEXUS_DOCKER_STAGING = "https://${env.NEXUS_URL}:5300" + +/* promote - promote artifacts to staging repo on tagged builds + * + */ +def promote(dev_image) { + if (utils.isTagBuild()) { + node('docker') { + docker.withRegistry(NEXUS_DOCKER_DEV, "jenkins_user") { + dev_image["image"].pull() + dev_image["image"].inside(DOCKER_OPTS) { + stage("Promote") { + unstash('dist') + withEnv(["HOME=${env.WORKSPACE}"]) { + utils.installNexusCreds('jenkins_user') + utils.uploadPythonPackages('jenkins_user', 'sima-staging', 'python/dist/*.whl') + } + } + } + } + } + } +} def main() { def job_name = env.JOB_NAME.split('/')[1] def image - currentBuild.result = "SUCCESS" + properties([ + parameters([ + booleanParam( + name: 'SKIP_DOWNSTREAM_BUILDS', + description: 'Skips building upstream jobs', + defaultValue: utils.isTagBuild() + ), + booleanParam( + name: "PACKAGE_ONLY", + defaultValue: utils.isTagBuild(), + description: 'Only package don\'t run tests' + ) + ]), + ]) - node('docker') { - stage("Checkout"){ + node("docker") { + stage("Checkout") { utils.checkoutBitbucket() + utils.setBuildMetadataFromVersionIn("python/VERSION.in") } - timeout(120) { - stage("Docker Pull") { - docker.withRegistry('', 'docker_creds') { - image = docker.image("simaai/n2a_compiler:latest") - image.pull() + stage("DockerBuild") { + image = utils.dockerBuild( + "docker/Dockerfile", + "${env.NEXUS_URL}:5000/" + job_name, + "jenkins_user", + "docker_build.log", + {}, + "https://${env.NEXUS_URL}:5000", + "120", + "20", + ) + } + + parallel push: { + stage("DockerPush") { + image['post']() + } + }, build: { + image["image"].inside(DOCKER_OPTS) { + utils.cmakeBuild("build", "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", {}, { src_dir -> + stage("Python Bindings") { + dir("${env.WORKSPACE}/python") { + utils.setPythonBuildEnv([], { + sh """#!/bin/bash -ex +rm -rf dist build +python3 setup.py bdist_wheel +""" + }, 'sima') + } + dir("${env.WORKSPACE}") { + stash(name: 'dist', includes: 'python/dist/*.whl') + } + } + }, "../sima-regres.cmake", "clean all") + stage("Package") { + tvm_pkg_dir = "python/dist/*.whl" + archiveArtifacts(tvm_pkg_dir) + withEnv(["HOME=${env.WORKSPACE}"]) { + utils.installNexusCreds('jenkins_user') + utils.uploadPythonPackages('jenkins_user', 'sima-pypi', tvm_pkg_dir, 3) + } } } } - timeout(120) { - image.inside(DOCKER_OPTS) { - stage("Download Package") { - utils.record('download.log', { - sh "rm -rf *.whl" - - tvm_pkg_name = "sima-tvm" - vinfo = readYaml file: "${env.WORKSPACE}/python/VERSION.in" - tvm_version = vinfo['major'] + "." + vinfo['minor'] + "." + vinfo['patch'] - - utils.downloadPythonPackage('jenkins_user', 'sima-pypi', tvm_pkg_name + "===" + tvm_version, 3) - topi_pkg_name = "sima-topi" - vinfo = readYaml file: "${env.WORKSPACE}/topi/python/VERSION.in" - topi_version = vinfo['major'] + "." + vinfo['minor'] + "." + vinfo['patch'] - - utils.downloadPythonPackage('jenkins_user', 'sima-pypi', topi_pkg_name + "===" + topi_version, 3) + stage("Promotion") { + if (env.BRANCH_NAME=="sima") { + utils.docker_promote(image['image'], 'jenkins_user', "https://${env.NEXUS_URL}:5000") + } + } - sh "ls -alh" - }, true) - } + } - stage("Validate Package") { - //TODO: run tests on the package - } + stage("Upstream") { + parallel afe: { + utils.buildUpstream("awesome-front-end", params.SKIP_DOWNSTREAM_BUILDS, [ + booleanParam(name: 'PACKAGE_ONLY', value: params.PACKAGE_ONLY) + ]) + }, psim_mla_flow: { + utils.buildUpstream("psim_mla_flow", params.SKIP_DOWNSTREAM_BUILDS, [ + booleanParam(name: 'PACKAGE_ONLY', value: params.PACKAGE_ONLY) + ]) + } + } - stage("Promote Package(Staging)") { - utils.record('upload.log', { - utils.uploadPythonPackages('jenkins_user', 'sima-staging', '*.whl', 3) - }, true) - } - } // docker - } // timeout - } // node -} // main + promote(image) +} -utils.job_wrapper({ +utils.job_wrapper( { main() -}, 240) +}) + +return this diff --git a/python/setup.py b/python/setup.py index a8f2746c7e61..008df47074f0 100644 --- a/python/setup.py +++ b/python/setup.py @@ -199,8 +199,27 @@ def get_version(pkg_dir, main_branch = "master"): def get_package(env, pkg_name, pkg_version): - if env in os.environ and os.environ[env] != "latest": - return "%s==%s.dev0+%s" % (pkg_name, pkg_version, os.environ[env]) + if env + "_BRANCH" in os.environ and env + "_VERSION" in os.environ: + branch = os.environ[env + "_BRANCH"] + version = os.environ[env + "_VERSION"] + + if version == pkg_version or pkg_version == "latest": + deps = {} + + if os.path.exists("DEPS"): + # Append dependency to DEPS file + with open("DEPS", "r") as fh: + deps = yaml.load(fh.read(), Loader=yaml.BaseLoader) + + deps[env] = "%s==%s.dev0+%s" % (pkg_name, version, branch) + + with open("DEPS", "w") as fh: + fh.write(yaml.dump(deps)) + + return "%s==%s.dev0+%s" % (pkg_name, version, branch) + + if pkg_version == "latest": + return "%s" % (pkg_name) else: return "%s==%s" % (pkg_name, pkg_version)