diff --git a/Jenkinsfile b/Jenkinsfile index 5115238a3a0..1c9b1556528 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,254 +14,8 @@ // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ -boolean doc_only_change() { - if (cachedCommitPragma(pragma: 'Doc-only') == 'true') { - return true - } - if (cachedCommitPragma(pragma: 'Doc-only') == 'false') { - return false - } - - def rc = sh label: "Determine if doc-only change", - script: "CHANGE_ID=${env.CHANGE_ID} " + - "TARGET_BRANCH=${target_branch} " + - 'ci/doc_only_change.sh', - returnStatus: true - return rc == 1 -} - -boolean release_candidate() { - return !sh(label: "Determine if building (a PR of) an RC", - script: "git diff-index --name-only HEAD^ | grep -q TAG && " + - "grep -i '[0-9]rc[0-9]' TAG", - returnStatus: true) -} - -def scons_faults_args() { - // The default build will have BUILD_TYPE=dev; fault injection enabled - if ((cachedCommitPragma(pragma: 'faults-enabled', def_val: 'true') == 'true') && !release_candidate()) { - return "BUILD_TYPE=dev" - } else { - return "BUILD_TYPE=release" - } -} - -def skip_stage(String stage, boolean def_val = false) { - String value = 'false' - if (def_val) { - value = 'true' - } - return cachedCommitPragma(pragma: 'Skip-' + stage, - def_val: value) == 'true' -} - -boolean quickbuild() { - return cachedCommitPragma(pragma: 'Quick-build') == 'true' || - quick_functional() -} - -def functional_post_always() { - return sh(label: "Job Cleanup", - script: 'ci/functional/job_cleanup.sh', - returnStatus: true) -} - -String get_daos_packages() { - return get_daos_packages(parseStageInfo()['target']) -} - -String get_daos_packages(String distro) { - String pkgs - if (env.TEST_RPMS == 'true') { - pkgs = "daos{,-{client,tests,server}}" - } else { - pkgs = "daos{,-client}" - } - if (distro.startsWith('ubuntu20')) { - return pkgs + "=" + daos_packages_version(distro) - } - return pkgs + "-" + daos_packages_version(distro) -} - -String pr_repos() { - return pr_repos(parseStageInfo()['target']) -} - -String pr_repos(String distro) { - String repos = "" - if (distro.startsWith('el7') || distro.startsWith('centos7')) { - repos = cachedCommitPragma(pragma: 'PR-repos-el7') - } else if (distro.startsWith('leap15')) { - repos = cachedCommitPragma(pragma: 'PR-repos-leap15') - } else if (distro.startsWith('ubuntu20')) { - repos = cachedCommitPragma(pragma: 'PR-repos-ubuntu20', cache: commit_pragma_cache) - } else { - error 'pr_repos not implemented for ' + distro - } - return repos + ' ' + cachedCommitPragma(pragma: 'PR-repos') -} - -String daos_repo() { - if (cachedCommitPragma(pragma: 'RPM-test-version') == '') { - return "daos@${env.BRANCH_NAME}:${env.BUILD_NUMBER}" - } else { - return "" - } -} - -String hw_distro_target() { - if (env.STAGE_NAME.contains('Hardware')) { - if (env.STAGE_NAME.contains('Small')) { - return hw_distro('small') - } - if (env.STAGE_NAME.contains('Medium')) { - return hw_distro('medium') - } - if (env.STAGE_NAME.contains('Large')) { - return hw_distro('large') - } - } - return parseStageInfo()['target'] -} - -String daos_repos() { - String target = hw_distro_target() - return daos_repos(target) -} - -String daos_repos(String distro) { - return pr_repos(distro) + ' ' + daos_repo() -} - -String unit_packages() { - Map stage_info = parseStageInfo() - boolean need_qb = quickbuild() - if (env.STAGE_NAME.contains('Bullseye')) { - need_qb = true - } - if (stage_info['target'].startsWith('el7') || - stage_info['target'].startsWith('centos7')) { - String packages = 'gotestsum openmpi3 ' + - 'hwloc-devel argobots ' + - 'fuse3-libs fuse3 ' + - 'boost-python36-devel ' + - 'libisa-l-devel libpmem ' + - 'libpmemobj protobuf-c ' + - 'spdk-devel libfabric-devel ' + - 'pmix numactl-devel ' + - 'libipmctl-devel python36-pyxattr ' + - 'python36-tabulate numactl ' + - 'libyaml-devel ' + - 'valgrind-devel patchelf' - if (need_qb) { - unstash stage_info['target'] + '-required-mercury-rpm-version' - packages += " spdk-tools mercury-" + - readFile(stage_info['target'] + - '-required-mercury-rpm-version').trim() + - " libisa-l_crypto libfabric-debuginfo" + - " argobots-debuginfo protobuf-c-debuginfo" - } - return packages - } else { - error 'unit packages not implemented for ' + stage_info['target'] - } -} - -commit_pragma_cache = [:] -def cachedCommitPragma(Map config) { - - if (commit_pragma_cache[config['pragma']]) { - return commit_pragma_cache[config['pragma']] - } - - commit_pragma_cache[config['pragma']] = commitPragma(config) - - return commit_pragma_cache[config['pragma']] - -} - -String daos_packages_version() { - return daos_packages_version(parseStageInfo()['target']) -} - -String daos_packages_version(String distro) { - // commit pragma has highest priority - // TODO: this should actually be determined from the PR-repos artifacts - String version = cachedCommitPragma(pragma: 'RPM-test-version') - if (version != "") { - String dist = "" - if (version.indexOf('-') > -1) { - // only tack on the %{dist} if the release was specified - if (distro.startsWith('el7') || distro.startsWith('centos7')) { - dist = ".el7" - } else if (distro.startsWith('leap15')) { - dist = ".suse.lp152" - } - } - return version + dist - } - - // use the stash after that - unstash distro + '-rpm-version' - version = readFile(distro + '-rpm-version').trim() - if (version != "") { - return version - } - - error "Don't know how to determine package version for " + distro -} - -boolean parallel_build() { - // defaults to false - // true if Quick-build: true unless Parallel-build: false - def pb = cachedCommitPragma(pragma: 'Parallel-build') - if (pb == "true" || - (quickbuild() && pb != "false")) { - return true - } - - return false -} - -String hw_distro(String size) { - // Possible values: - //'leap15 - //'centos7 - return cachedCommitPragma(pragma: 'Func-hw-test-' + size + '-distro', - def_val: cachedCommitPragma(pragma: 'Func-hw-test-distro', - def_val: 'centos7')) -} - -String functional_packages() { - String target = hw_distro_target() - return functional_packages(target) -} - -String functional_packages(String distro) { - String daos_pkgs = get_daos_packages(distro) - String pkgs = " openmpi3 hwloc ndctl fio " + - "patchutils ior-hpc-daos-1 " + - "romio-tests-daos-1 " + - "testmpio " + - "mpi4py-tests " + - "hdf5-mpich-tests " + - "hdf5-openmpi3-tests " + - "hdf5-vol-daos-mpich-tests " + - "hdf5-vol-daos-openmpi3-tests " + - "MACSio-mpich " + - "MACSio-openmpi3 " + - "mpifileutils-mpich-daos-1 " - if (distro.startsWith('leap15')) { - return daos_pkgs + pkgs - } else if (distro.startsWith('el7') || distro.startsWith('centos7')) { - // need to exclude openmpi until we remove it from the repo - return "--exclude openmpi " + daos_pkgs + pkgs - } else if (distro.startsWith('ubuntu20')) { - return daos_pkgs + " openmpi-bin ndctl fio" - } else { - error 'functional_packages not implemented for ' + distro - } -} +// For master, this is just some wildly high number +next_version = "1000" // Don't define this as a type or it loses it's global scope target_branch = env.CHANGE_TARGET ? env.CHANGE_TARGET : env.BRANCH_NAME @@ -287,235 +41,13 @@ def getuid() { return cached_uid } -// Default priority is 3, lower is better. -// The parameter for a job is set using the script/Jenkinsfile from the -// previous build of that job, so the first build of any PR will always -// run at default because Jenkins sees it as a new job, but subsequent -// ones will use the value from here. -// The advantage therefore is not to change the priority of PRs, but to -// change the master branch itself to run at lower priority, resulting -// in faster time-to-result for PRs. - -String get_priority() { - if (env.BRANCH_NAME == 'master' || - env.BRANCH_NAME.startsWith("release/")) { - string p = '2' - } else { - string p = '' - } - echo "Build priority set to " + p == '' ? 'default' : p - return p -} - -String rpm_test_version() { - return cachedCommitPragma(pragma: 'RPM-test-version') -} - -boolean skip_prebuild() { - return target_branch == 'weekly-testing' -} - -boolean skip_checkpatch() { - return skip_stage('checkpatch') || - doc_only_change() || - quick_functional() -} - -boolean skip_build() { - // always build branch landings as we depend on lastSuccessfulBuild - // always having RPMs in it - return (env.BRANCH_NAME != target_branch) && - skip_stage('build') || - doc_only_change() || - rpm_test_version() != '' -} - -boolean quick_functional() { - return cachedCommitPragma(pragma: 'Quick-Functional') == 'true' -} - -boolean skip_build_rpm(String distro) { - return target_branch == 'weekly-testing' || - skip_stage('build-' + distro + '-rpm') || - distro == 'ubuntu20' && quick_functional() -} - -boolean skip_build_on_centos7_gcc() { - return skip_stage('build-centos7-gcc') || - quick_functional() -} - -boolean skip_ftest(String distro) { - return distro == 'ubuntu20' || - skip_stage('func-test') || - skip_stage('func-test-vm') || - ! tests_in_stage('vm') || - skip_stage('func-test-' + distro) -} - -boolean skip_test_rpms_centos7() { - return target_branch == 'weekly-testing' || - skip_stage('test') || - skip_stage('test-centos-rpms') || - quick_functional() -} - -boolean skip_scan_rpms_centos7() { - return target_branch == 'weekly-testing' || - skip_stage('scan-centos-rpms', true) || - quick_functional() -} - -boolean tests_in_stage(String size) { - Map stage_info = parseStageInfo() - return sh(label: "Get test list for ${size}", - script: """cd src/tests/ftest - ./list_tests.py """ + stage_info['test_tag'], - returnStatus: true) == 0 -} - -boolean skip_ftest_hw(String size) { - return env.DAOS_STACK_CI_HARDWARE_SKIP == 'true' || - skip_stage('func-test') || - skip_stage('func-hw-test') || - skip_stage('func-hw-test-' + size) || - ! tests_in_stage(size) || - (env.BRANCH_NAME == 'master' && ! startedByTimer()) -} - -boolean skip_bandit_check() { - return cachedCommitPragma(pragma: 'Skip-python-bandit', - def_val: 'true') == 'true' || - quick_functional() -} - -boolean skip_build_on_centos7_bullseye() { - return env.NO_CI_TESTING == 'true' || - skip_stage('bullseye', true) || - quick_functional() -} - -boolean skip_build_on_centos7_gcc_debug() { - return skip_stage('build-centos7-gcc-debug') || - quickbuild() -} - -boolean skip_build_on_centos7_gcc_release() { - return skip_stage('build-centos7-gcc-release') || - quickbuild() -} - -boolean skip_build_on_centos8_gcc_dev() { - return skip_stage('build-centos8-gcc-dev') || - quickbuild() -} - -boolean skip_build_on_landing_branch() { - return env.BRANCH_NAME != target_branch || - quickbuild() -} - -boolean skip_build_on_ubuntu_clang() { - return target_branch == 'weekly-testing' || - skip_stage('build-ubuntu-clang') || - quickbuild() - -} - -boolean skip_build_on_leap15_gcc() { - return skip_stage('build-leap15-gcc') || - quickbuild() -} - -boolean skip_build_on_leap15_icc() { - return target_branch == 'weekly-testing' || - skip_stage('build-leap15-icc') || - quickbuild() -} - -boolean skip_unit_testing_stage() { - return env.NO_CI_TESTING == 'true' || - (skip_stage('build') && - rpm_test_version() == '') || - doc_only_change() || - skip_build_on_centos7_gcc() || - skip_stage('unit-tests') -} - -boolean skip_coverity() { - return skip_stage('coverity-test') || - quick_functional() || - skip_stage('build') -} - -boolean skip_if_unstable() { - if (cachedCommitPragma(pragma: 'Allow-unstable-test') == 'true' || - env.BRANCH_NAME == 'master' || - env.BRANCH_NAME.startsWith("weekly-testing") || - env.BRANCH_NAME.startsWith("release/")) { - return false - } - - //Ok, it's a PR and the Allow pragma isn't set. Skip if the build is - //unstable. - - return currentBuild.currentResult == 'UNSTABLE' -} - -boolean skip_testing_stage() { - return env.NO_CI_TESTING == 'true' || - (skip_stage('build') && - rpm_test_version() == '') || - doc_only_change() || - skip_stage('test') || - skip_if_unstable() -} - -boolean skip_unit_test() { - return skip_stage('unit-test') || - skip_stage('run_test') -} - -boolean skip_bullseye_report() { - return env.BULLSEYE == null || - skip_stage('bullseye', true) -} - -// Reply with an empty string if quickbuild disabled to avoid discarding -// docker caches. -String quick_build_deps(String distro, always=false) { - String rpmspec_args = "" - if (!always) { - if (!quickbuild() ) { - return "" - } - } - if (distro.startsWith('leap15')) { - rpmspec_args = "--define dist\\ .suse.lp152 " + - "--undefine rhel " + - "--define suse_version\\ 1502" - } else if (distro.startsWith('el7') || distro.startsWith('centos7')) { - rpmspec_args = "--undefine suse_version " + - "--define rhel\\ 7" - } else { - error("Unknown distro: ${distro} in quick_build_deps()") - } - return sh(label: 'Get Quickbuild dependencies', - script: "rpmspec -q " + - "--srpm " + - rpmspec_args + ' ' + - "--requires utils/rpms/daos.spec " + - "2>/dev/null", - returnStdout: true) -} - pipeline { agent { label 'lightweight' } triggers { cron(env.BRANCH_NAME == 'master' ? 'TZ=America/Toronto\n0 0 * * *\n' : '' + env.BRANCH_NAME == 'release/1.2' ? 'TZ=America/Toronto\n0 12 * * *\n' : '' + - env.BRANCH_NAME == 'weekly-testing' ? 'H 0 * * 6' : '') + env.BRANCH_NAME.startsWith('weekly-testing') ? 'H 0 * * 6' : '') } environment { @@ -525,7 +57,7 @@ pipeline { CLUSH_ARGS = "-o$SSH_KEY_ARGS" TEST_RPMS = cachedCommitPragma(pragma: 'RPM-test', def_val: 'true') COVFN_DISABLED = cachedCommitPragma(pragma: 'Skip-fnbullseye', def_val: 'true') - SCONS_FAULTS_ARGS = scons_faults_args() + SCONS_FAULTS_ARGS = sconsFaultsArgs() } options { @@ -537,11 +69,22 @@ pipeline { parameters { string(name: 'BuildPriority', - defaultValue: get_priority(), + defaultValue: getPriority(), description: 'Priority of this build. DO NOT USE WITHOUT PERMISSION.') + string(name: 'TestTag', + defaultValue: "daily_regression", + description: 'Test-tag to use for this run (i.e pr, daily_regression, full_regression, etc.') } stages { + stage('Get Commit Message') { + steps { + script { + env.COMMIT_MESSAGE = sh(script: 'git show -s --format=%B', + returnStdout: true).trim() + } + } + } stage('Cancel Previous Builds') { when { changeRequest() } steps { @@ -551,20 +94,20 @@ pipeline { stage('Pre-build') { when { beforeAgent true - expression { ! skip_prebuild() } + expression { ! skipStage() } } parallel { stage('checkpatch') { when { beforeAgent true - expression { ! skip_checkpatch() } + expression { ! skipStage() } } agent { dockerfile { filename 'Dockerfile.checkpatch' dir 'utils/docker' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(add_repos:false) + additionalBuildArgs dockerBuildArgs(add_repos: false) } } steps { @@ -623,7 +166,7 @@ pipeline { stage('Python Bandit check') { when { beforeAgent true - expression { ! skip_bandit_check() } + expression { ! skipStage() } } agent { dockerfile { @@ -655,7 +198,7 @@ pipeline { //failFast true when { beforeAgent true - expression { ! skip_build() } + expression { ! skipStage() } } parallel { stage('Build RPM on CentOS 7') { @@ -674,11 +217,6 @@ pipeline { post { success { buildRpmPost condition: 'success' - script { - Map stage_info = parseStageInfo() - stash name: stage_info['target'] + '-required-mercury-rpm-version', - includes: stage_info['target'] + '-required-mercury-rpm-version' - } } unstable { buildRpmPost condition: 'unstable' @@ -697,7 +235,7 @@ pipeline { stage('Build RPM on Leap 15') { when { beforeAgent true - expression { ! skip_build_rpm('leap15') } + expression { ! skipStage() } } agent { dockerfile { @@ -732,7 +270,7 @@ pipeline { stage('Build DEB on Ubuntu 20.04') { when { beforeAgent true - expression { ! skip_build_rpm('ubuntu20') } + expression { ! skipStage() } } agent { dockerfile { @@ -768,24 +306,24 @@ pipeline { stage('Build on CentOS 7') { when { beforeAgent true - expression { ! skip_build_on_centos7_gcc() } + expression { ! skipStage() } } agent { dockerfile { filename 'utils/docker/Dockerfile.centos.7' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(qb: quickbuild()) + + additionalBuildArgs dockerBuildArgs(qb: quickBuild()) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg QUICKBUILD_DEPS="' + - quick_build_deps('centos7') + '"' + - ' --build-arg REPOS="' + pr_repos() + '"' + quickBuildDeps('centos7') + '"' + + ' --build-arg REPOS="' + prRepos() + '"' } } steps { - sconsBuild parallel_build: parallel_build(), + sconsBuild parallel_build: parallelBuild(), stash_files: 'ci/test_files_to_stash.txt', scons_exe: 'scons-3', - scons_args: scons_faults_args() + scons_args: sconsFaultsArgs() } post { always { @@ -806,25 +344,25 @@ pipeline { stage('Build on CentOS 7 Bullseye') { when { beforeAgent true - expression { ! skip_build_on_centos7_bullseye() } + expression { ! skipStage() } } agent { dockerfile { filename 'utils/docker/Dockerfile.centos.7' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(qb: quickbuild()) + + additionalBuildArgs dockerBuildArgs(qb: quickBuild()) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg BULLSEYE=' + env.BULLSEYE + ' --build-arg QUICKBUILD_DEPS="' + - quick_build_deps('centos7') + '"' + - ' --build-arg REPOS="' + pr_repos() + '"' + quickBuildDeps('centos7') + '"' + + ' --build-arg REPOS="' + prRepos() + '"' } } steps { - sconsBuild parallel_build: parallel_build(), + sconsBuild parallel_build: parallelBuild(), stash_files: 'ci/test_files_to_stash.txt', scons_exe: 'scons-3', - scons_args: scons_faults_args() + scons_args: sconsFaultsArgs() } post { always { @@ -845,22 +383,22 @@ pipeline { stage('Build on CentOS 7 debug') { when { beforeAgent true - expression { ! skip_build_on_centos7_gcc_debug() } + expression { ! skipStage() } } agent { dockerfile { filename 'utils/docker/Dockerfile.centos.7' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(qb: quickbuild(), - deps_build:true) + + additionalBuildArgs dockerBuildArgs(qb: quickBuild(), + deps_build: true) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg QUICKBUILD_DEPS="' + - quick_build_deps('centos7') + '"' + - ' --build-arg REPOS="' + pr_repos() + '"' + quickBuildDeps('centos7') + '"' + + ' --build-arg REPOS="' + prRepos() + '"' } } steps { - sconsBuild parallel_build: parallel_build(), + sconsBuild parallel_build: parallelBuild(), scons_exe: 'scons-3', scons_args: "PREFIX=/opt/daos TARGET_TYPE=release", build_deps: "no" @@ -884,22 +422,22 @@ pipeline { stage('Build on CentOS 7 release') { when { beforeAgent true - expression { ! skip_build_on_centos7_gcc_release() } + expression { ! skipStage() } } agent { dockerfile { filename 'utils/docker/Dockerfile.centos.7' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(qb: quickbuild(), - deps_build:true) + + additionalBuildArgs dockerBuildArgs(qb: quickBuild(), + deps_build: true) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg QUICKBUILD_DEPS="' + - quick_build_deps('centos7') + '"' + - ' --build-arg REPOS="' + pr_repos() + '"' + quickBuildDeps('centos7') + '"' + + ' --build-arg REPOS="' + prRepos() + '"' } } steps { - sconsBuild parallel_build: parallel_build(), + sconsBuild parallel_build: parallelBuild(), scons_exe: 'scons-3', scons_args: "PREFIX=/opt/daos TARGET_TYPE=release", build_deps: "no" @@ -923,23 +461,23 @@ pipeline { stage('Build on CentOS 7 with Clang') { when { beforeAgent true - expression { ! skip_build_on_landing_branch() } + expression { ! skipStage() } } agent { dockerfile { filename 'utils/docker/Dockerfile.centos.7' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(qb: quickbuild(), - deps_build:true) + + additionalBuildArgs dockerBuildArgs(qb: quickBuild(), + deps_build: true) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg QUICKBUILD_DEPS="' + - quick_build_deps('centos7') + '"' + quickBuildDeps('centos7') + '"' } } steps { - sconsBuild parallel_build: parallel_build(), + sconsBuild parallel_build: parallelBuild(), scons_exe: 'scons-3', - scons_args: scons_faults_args() + " PREFIX=/opt/daos TARGET_TYPE=release", + scons_args: sconsFaultsArgs() + " PREFIX=/opt/daos TARGET_TYPE=release", build_deps: "no" } post { @@ -961,20 +499,20 @@ pipeline { stage('Build on CentOS 8') { when { beforeAgent true - expression { ! skip_build_on_centos8_gcc_dev() } + expression { ! skipStage() } } agent { dockerfile { filename 'utils/docker/Dockerfile.centos.8' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(qb: quickbuild(), - deps_build:true) + + additionalBuildArgs dockerBuildArgs(qb: quickBuild(), + deps_build: true) + " -t ${sanitized_JOB_NAME}-centos8 " } } steps { - sconsBuild parallel_build: parallel_build(), - scons_args: scons_faults_args() + " PREFIX=/opt/daos TARGET_TYPE=release", + sconsBuild parallel_build: parallelBuild(), + scons_args: sconsFaultsArgs() + " PREFIX=/opt/daos TARGET_TYPE=release", build_deps: "no", scons_exe: 'scons-3' } @@ -997,19 +535,19 @@ pipeline { stage('Build on Ubuntu 20.04') { when { beforeAgent true - expression { ! skip_build_on_landing_branch() } + expression { ! skipStage() } } agent { dockerfile { filename 'utils/docker/Dockerfile.ubuntu.20.04' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(deps_build:true) + + additionalBuildArgs dockerBuildArgs(deps_build: true) + " -t ${sanitized_JOB_NAME}-ubuntu20.04" } } steps { - sconsBuild parallel_build: parallel_build(), - scons_args: scons_faults_args() + " PREFIX=/opt/daos TARGET_TYPE=release", + sconsBuild parallel_build: parallelBuild(), + scons_args: sconsFaultsArgs() + " PREFIX=/opt/daos TARGET_TYPE=release", build_deps: "no" } post { @@ -1031,19 +569,19 @@ pipeline { stage('Build on Ubuntu 20.04 with Clang') { when { beforeAgent true - expression { ! skip_build_on_ubuntu_clang() } + expression { ! skipStage() } } agent { dockerfile { filename 'utils/docker/Dockerfile.ubuntu.20.04' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(deps_build:true) + + additionalBuildArgs dockerBuildArgs(deps_build: true) + " -t ${sanitized_JOB_NAME}-ubuntu20.04" } } steps { - sconsBuild parallel_build: parallel_build(), - scons_args: scons_faults_args() + " PREFIX=/opt/daos TARGET_TYPE=release", + sconsBuild parallel_build: parallelBuild(), + scons_args: sconsFaultsArgs() + " PREFIX=/opt/daos TARGET_TYPE=release", build_deps: "no" } post { @@ -1065,23 +603,23 @@ pipeline { stage('Build on Leap 15') { when { beforeAgent true - expression { ! skip_build_on_leap15_gcc() } + expression { ! skipStage() } } agent { dockerfile { filename 'utils/docker/Dockerfile.leap.15' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(qb: quickbuild()) + + additionalBuildArgs dockerBuildArgs(qb: quickBuild()) + " -t ${sanitized_JOB_NAME}-leap15 " + ' --build-arg QUICKBUILD_DEPS="' + - quick_build_deps('leap15') + '"' + - ' --build-arg REPOS="' + pr_repos() + '"' + quickBuildDeps('leap15') + '"' + + ' --build-arg REPOS="' + prRepos() + '"' } } steps { - sconsBuild parallel_build: parallel_build(), + sconsBuild parallel_build: parallelBuild(), stash_files: 'ci/test_files_to_stash.txt', - scons_args: scons_faults_args() + scons_args: sconsFaultsArgs() } post { always { @@ -1102,19 +640,19 @@ pipeline { stage('Build on Leap 15 with Clang') { when { beforeAgent true - expression { ! skip_build_on_landing_branch() } + expression { ! skipStage() } } agent { dockerfile { filename 'utils/docker/Dockerfile.leap.15' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(deps_build:true) + + additionalBuildArgs dockerBuildArgs(deps_build: true) + " -t ${sanitized_JOB_NAME}-leap15" } } steps { - sconsBuild parallel_build: parallel_build(), - scons_args: scons_faults_args() + " PREFIX=/opt/daos TARGET_TYPE=release", + sconsBuild parallel_build: parallelBuild(), + scons_args: sconsFaultsArgs() + " PREFIX=/opt/daos TARGET_TYPE=release", build_deps: "no" } post { @@ -1136,20 +674,20 @@ pipeline { stage('Build on Leap 15 with Intel-C and TARGET_PREFIX') { when { beforeAgent true - expression { ! skip_build_on_leap15_icc() } + expression { ! skipStage() } } agent { dockerfile { filename 'utils/docker/Dockerfile.leap.15' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(deps_build:true) + + additionalBuildArgs dockerBuildArgs(deps_build: true) + " -t ${sanitized_JOB_NAME}-leap15" args '-v /opt/intel:/opt/intel' } } steps { - sconsBuild parallel_build: parallel_build(), - scons_args: scons_faults_args() + " PREFIX=/opt/daos TARGET_TYPE=release", + sconsBuild parallel_build: parallelBuild(), + scons_args: sconsFaultsArgs() + " PREFIX=/opt/daos TARGET_TYPE=release", build_deps: "no" } post { @@ -1173,21 +711,21 @@ pipeline { stage('Unit Tests') { when { beforeAgent true - expression { ! skip_unit_testing_stage() } + expression { ! skipStage() } } parallel { stage('Unit Test') { when { beforeAgent true - expression { ! skip_unit_test() } + expression { ! skipStage() } } agent { label 'ci_vm1' } steps { - unitTest timeout_time: 30, - inst_repos: pr_repos(), - inst_rpms: unit_packages() + unitTest timeout_time: 60, + inst_repos: prRepos(), + inst_rpms: unitPackages() } post { always { @@ -1198,16 +736,16 @@ pipeline { stage('NLT') { when { beforeAgent true - expression { ! skip_stage('nlt') } + expression { ! skipStage() } } agent { label 'ci_nlt_1' } steps { unitTest timeout_time: 30, - inst_repos: pr_repos(), + inst_repos: prRepos(), test_script: 'ci/unit/test_nlt.sh', - inst_rpms: unit_packages() + inst_rpms: unitPackages() } post { always { @@ -1237,7 +775,7 @@ pipeline { stage('Unit Test Bullseye') { when { beforeAgent true - expression { ! skip_stage('bullseye', true) } + expression { ! skipStage() } } agent { label 'ci_vm1' @@ -1245,8 +783,8 @@ pipeline { steps { unitTest timeout_time: 60, ignore_failure: true, - inst_repos: pr_repos(), - inst_rpms: unit_packages() + inst_repos: prRepos(), + inst_rpms: unitPackages() } post { always { @@ -1263,7 +801,7 @@ pipeline { stage('Unit Test with memcheck') { when { beforeAgent true - expression { ! skip_stage('unit-test-memcheck') } + expression { ! skipStage() } } agent { label 'ci_vm1' @@ -1271,8 +809,8 @@ pipeline { steps { unitTest timeout_time: 30, ignore_failure: true, - inst_repos: pr_repos(), - inst_rpms: unit_packages() + inst_repos: prRepos(), + inst_rpms: unitPackages() } post { always { @@ -1287,13 +825,13 @@ pipeline { stage('Test') { when { beforeAgent true - expression { ! skip_testing_stage() } + expression { ! skipStage() } } parallel { stage('Coverity on CentOS 7') { when { beforeAgent true - expression { ! skip_coverity() } + expression { ! skipStage() } } agent { dockerfile { @@ -1302,13 +840,13 @@ pipeline { additionalBuildArgs dockerBuildArgs(qb: true) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg QUICKBUILD_DEPS="' + - quick_build_deps('centos7', true) + '"' + - ' --build-arg REPOS="' + pr_repos() + '"' + quickBuildDeps('centos7', true) + '"' + + ' --build-arg REPOS="' + prRepos() + '"' } } steps { sconsBuild coverity: "daos-stack/daos", - parallel_build: parallel_build(), + parallel_build: parallelBuild(), scons_exe: 'scons-3' } post { @@ -1323,14 +861,14 @@ pipeline { stage('Functional on CentOS 7') { when { beforeAgent true - expression { ! skip_ftest('el7') } + expression { ! skipStage() } } agent { label 'ci_vm9' } steps { - functionalTest inst_repos: daos_repos(), - inst_rpms: functional_packages(), + functionalTest inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version), test_function: 'runTestFunctionalV2' } post { @@ -1342,14 +880,14 @@ pipeline { stage('Functional on Leap 15') { when { beforeAgent true - expression { ! skip_ftest('leap15') } + expression { ! skipStage() } } agent { label 'ci_vm9' } steps { - functionalTest inst_repos: daos_repos(), - inst_rpms: functional_packages(), + functionalTest inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version), test_function: 'runTestFunctionalV2' } post { @@ -1361,14 +899,14 @@ pipeline { stage('Functional on Ubuntu 20.04') { when { beforeAgent true - expression { ! skip_ftest('ubuntu20') } + expression { ! skipStage() } } agent { label 'ci_vm9' } steps { - functionalTest inst_repos: daos_repos(), - inst_rpms: functional_packages(), + functionalTest inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version), test_function: 'runTestFunctionalV2' } post { @@ -1380,27 +918,27 @@ pipeline { stage('Test CentOS 7 RPMs') { when { beforeAgent true - expression { ! skip_test_rpms_centos7() } + expression { ! skipStage() } } agent { label 'ci_vm1' } steps { - testRpm inst_repos: daos_repos(), - daos_pkg_version: daos_packages_version() + testRpm inst_repos: daosRepos(), + daos_pkg_version: daosPackagesVersion(next_version) } } // stage('Test CentOS 7 RPMs') stage('Scan CentOS 7 RPMs') { when { beforeAgent true - expression { ! skip_scan_rpms_centos7() } + expression { ! skipStage() } } agent { label 'ci_vm1' } steps { - scanRpms inst_repos: daos_repos(), - daos_pkg_version: daos_packages_version(), + scanRpms inst_repos: daosRepos(), + daos_pkg_version: daosPackagesVersion(next_version), inst_rpms: 'clamav clamav-devel', test_script: 'ci/rpm/scan_daos.sh', junit_files: 'maldetect.xml' @@ -1416,22 +954,21 @@ pipeline { stage('Test Hardware') { when { beforeAgent true - expression { ! skip_testing_stage() } + expression { ! skipStage() } } parallel { - stage('Functional_Hardware_Small') { + stage('Functional Hardware Small') { when { beforeAgent true - expression { ! skip_ftest_hw('small') } + expression { ! skipStage() } } agent { // 2 node cluster with 1 IB/node + 1 test control node - label 'ci_nvme3' + label 'stage_nvme3' } steps { - functionalTest target: hw_distro_target(), - inst_repos: daos_repos(), - inst_rpms: functional_packages(), + functionalTest inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version), test_function: 'runTestFunctionalV2' } post { @@ -1440,19 +977,19 @@ pipeline { } } } // stage('Functional_Hardware_Small') - stage('Functional_Hardware_Medium') { + stage('Functional Hardware Medium') { when { beforeAgent true - expression { ! skip_ftest_hw('medium') } + expression { ! skipStage() } } agent { // 4 node cluster with 2 IB/node + 1 test control node label 'ci_nvme5' } steps { - functionalTest target: hw_distro_target(), - inst_repos: daos_repos(), - inst_rpms: functional_packages(), + functionalTest target: hwDistroTarget(), + inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version), test_function: 'runTestFunctionalV2' } post { @@ -1461,19 +998,19 @@ pipeline { } } } // stage('Functional_Hardware_Medium') - stage('Functional_Hardware_Large') { + stage('Functional Hardware Large') { when { beforeAgent true - expression { ! skip_ftest_hw('large') } + expression { ! skipStage() } } agent { // 8+ node cluster with 1 IB/node + 1 test control node label 'ci_nvme9' } steps { - functionalTest target: hw_distro_target(), - inst_repos: daos_repos(), - inst_rpms: functional_packages(), + functionalTest target: hwDistroTarget(), + inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version), test_function: 'runTestFunctionalV2' } post { @@ -1489,18 +1026,18 @@ pipeline { stage('Bullseye Report') { when { beforeAgent true - expression { ! skip_bullseye_report() } + expression { ! skipStage() } } agent { dockerfile { filename 'utils/docker/Dockerfile.centos.7' label 'docker_runner' - additionalBuildArgs dockerBuildArgs(qb: quickbuild()) + + additionalBuildArgs dockerBuildArgs(qb: quickBuild()) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg BULLSEYE=' + env.BULLSEYE + ' --build-arg QUICKBUILD_DEPS="' + - quick_build_deps('centos7') + '"' + - ' --build-arg REPOS="' + pr_repos() + '"' + quickBuildDeps('centos7') + '"' + + ' --build-arg REPOS="' + prRepos() + '"' } } steps { diff --git a/ci/functional/required_packages.sh b/ci/functional/required_packages.sh new file mode 100755 index 00000000000..c123adb68f4 --- /dev/null +++ b/ci/functional/required_packages.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -eux + +distro="$1" +client_ver="$2" + +pkgs="openmpi3 hwloc ndctl fio \ + patchutils ior-hpc-daos-${client_ver} \ + romio-tests-daos-${client_ver} \ + testmpio \ + mpi4py-tests \ + hdf5-mpich-tests \ + hdf5-openmpi3-tests \ + hdf5-vol-daos-mpich-tests \ + hdf5-vol-daos-openmpi3-tests \ + MACSio-mpich \ + MACSio-openmpi3 \ + mpifileutils-mpich-daos-${client_ver}" + +if [[ $distro = el7* ]] || [[ $distro = centos7* ]]; then + pkgs="--exclude openmpi $pkgs" +fi + +if [[ $distro = ubuntu20* ]]; then + pkgs="openmpi-bin ndctl fio" +fi + +echo "$pkgs" +exit 0 diff --git a/ci/unit/required_packages.sh b/ci/unit/required_packages.sh new file mode 100755 index 00000000000..c1c909dba5e --- /dev/null +++ b/ci/unit/required_packages.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -eux + +distro="$1" +quick_build="${2:-false}" + +pkgs="gotestsum openmpi3 \ + hwloc-devel argobots \ + fuse3-libs fuse3 \ + boost-devel \ + libisa-l-devel libpmem \ + libpmemobj protobuf-c \ + spdk-devel libfabric-devel \ + pmix numactl-devel \ + libipmctl-devel python36-pyxattr \ + python36-tabulate numactl \ + libyaml-devel \ + valgrind-devel patchelf" + +if $quick_build; then + read -r mercury_version < "$distro"-required-mercury-rpm-version + pkgs="$pkgs spdk-tools mercury-$mercury_version \ + boost-devel libisa-l_crypto libfabric-debuginfo \ + argobots-debuginfo protobuf-c-debuginfo" +fi + +echo "$pkgs" +exit 0 diff --git a/utils/docker/Dockerfile.centos.7 b/utils/docker/Dockerfile.centos.7 index 16fae0a1574..8d307a641e5 100644 --- a/utils/docker/Dockerfile.centos.7 +++ b/utils/docker/Dockerfile.centos.7 @@ -123,20 +123,19 @@ ARG REPOS="" ARG REPO_URL="" ARG REPO_EL7="" # Need to clean this up later -RUN if [ -n "$REPO_URL" ]; then \ - cversion=$(cut -d ' ' -f4 /etc/centos-release); \ - version=${cversion%.*} ; \ - MY_REPO="${REPO_URL}repository/centos-${version}-x86_64-group/"; \ - MY_NAME="${MY_REPO#*//}"; \ - MY_NAME="${MY_NAME//\//_}"; \ +RUN if [ -n "$REPO_URL" ]; then \ + version=$(sed -e 's/.* \([0-9\.]*\)\.[0-9]*.*/\1/' /etc/centos-release); \ + MY_REPO="${REPO_URL}repository/centos-${version}-x86_64-group/"; \ + MY_NAME="${MY_REPO#*//}"; \ + MY_NAME="${MY_NAME//\//_}"; \ echo -e "[${MY_NAME}]\n\ name=created from ${MY_REPO}\n\ baseurl=${MY_REPO}\n\ enabled=1\n\ -gpgcheck=False\n" >> /etc/yum.repos.d/local-centos-group.repo; \ - yum --assumeyes install yum-utils; \ - yum-config-manager --assumeyes --quiet \ - --disable base epel extra updates; \ +gpgcheck=False\n" >> /etc/yum.repos.d/local-centos-group.repo; \ + yum --assumeyes install yum-utils; \ + yum-config-manager --assumeyes --quiet \ + --disable base epel extra updates; \ fi RUN if $QUICKBUILD; then \ MY_REPO="${REPO_URL}${REPO_EL7}"; \ @@ -165,7 +164,8 @@ enabled=1\n\ gpgcheck=False\n" >> /etc/yum.repos.d/$repo:$branch:$build_number.repo; \ done; \ echo "Installing: $QUICKBUILD_DEPS"; \ - echo "$QUICKBUILD_DEPS" | tr '\n' '\0' | xargs -0 yum -y install; \ + echo "$QUICKBUILD_DEPS" | sed -e '/^$/d' | tr '\n' '\0' | \ + xargs -0 yum -y install; \ fi ARG BULLSEYE="" diff --git a/utils/docker/Dockerfile.centos.8 b/utils/docker/Dockerfile.centos.8 index 9df8193f0ec..4f6369c01c9 100644 --- a/utils/docker/Dockerfile.centos.8 +++ b/utils/docker/Dockerfile.centos.8 @@ -17,18 +17,17 @@ ARG REPO_URL # This is largely the same as the centos.7 code, however this sets priority for # the new repo, and leaves the default ones enabled as there is some metadata # that is not available locally. -RUN if [ -n "$REPO_URL" ]; then \ - cversion=$(cut -d ' ' -f4 /etc/centos-release); \ - version=${cversion%.*} ; \ - MY_REPO="${REPO_URL}repository/centos-${version}-x86_64-group/"; \ - MY_NAME="${MY_REPO#*//}"; \ - MY_NAME="${MY_NAME//\//_}"; \ +RUN if [ -n "$REPO_URL" ]; then \ + version=$(sed -e 's/.* \([0-9\.]*\)\.[0-9]*.*/\1/' /etc/centos-release); \ + MY_REPO="${REPO_URL}repository/centos-${version}-x86_64-group/"; \ + MY_NAME="${MY_REPO#*//}"; \ + MY_NAME="${MY_NAME//\//_}"; \ echo -e "[${MY_NAME}]\n\ name=created from ${MY_REPO}\n\ baseurl=${MY_REPO}\n\ enabled=1\n\ priority=1\n\ -gpgcheck=False\n" >> /etc/yum.repos.d/local-centos-group.repo; \ +gpgcheck=False\n" >> /etc/yum.repos.d/local-centos-group.repo; \ fi # Install basic tools to build DAOS diff --git a/utils/docker/Dockerfile.leap.15 b/utils/docker/Dockerfile.leap.15 index 4df2b22e90c..7f564a77d96 100644 --- a/utils/docker/Dockerfile.leap.15 +++ b/utils/docker/Dockerfile.leap.15 @@ -147,7 +147,7 @@ RUN if $QUICKBUILD; then $repo:$branch:$build_number; \ done; \ echo "Installing: $QUICKBUILD_DEPS"; \ - echo "$QUICKBUILD_DEPS" | tr '\n' '\0' | xargs -0 zypper --non-interactive install; \ + echo "$QUICKBUILD_DEPS" | sed -e '/^$/d' | tr '\n' '\0' | xargs -0 zypper --non-interactive install; \ fi RUN zypper addlock fuse fuse-libs fuse-devel libraft0 raft-devel mercury mercury-devel