From 2345581a6f8571e0da0d90998dbf79247eda5993 Mon Sep 17 00:00:00 2001 From: prudhvigodithi Date: Fri, 2 Sep 2022 16:46:52 -0700 Subject: [PATCH] Jenkinsfile change for Branching Signed-off-by: prudhvigodithi --- .github/ISSUE_TEMPLATE/release_template.md | 1 + build.gradle | 20 ++-- jenkins/check-for-build.jenkinsfile | 17 +++- .../bwc-test.jenkinsfile | 29 +++++- .../distribution-build.jenkinsfile | 43 +++++++- .../integ-test.jenkinsfile | 29 +++++- jenkins/opensearch/bwc-test.jenkinsfile | 31 +++++- .../opensearch/distribution-build.jenkinsfile | 51 ++++++++-- jenkins/opensearch/integ-test.jenkinsfile | 28 +++++- jenkins/opensearch/perf-test.jenkinsfile | 29 +++++- .../release-notes-check.jenkinsfile | 48 +++++++-- manifests/2.3.0/opensearch-2.3.0-test.yml | 94 ++++++++++++++++++ manifests/2.3.0/opensearch-2.3.0.yml | 98 ++++++++++++++++++- .../opensearch-dashboards-2.3.0-test.yml | 17 ++++ .../2.3.0/opensearch-dashboards-2.3.0.yml | 31 +++++- tests/jenkins/TestOpenSearchBwcTest.groovy | 3 +- .../TestOpenSearchDashboardsBwcTest.groovy | 2 + .../TestOpenSearchDashboardsIntegTest.groovy | 2 +- tests/jenkins/TestOpenSearchIntegTest.groovy | 3 + .../bwc-test.jenkinsfile.txt | 10 ++ .../integ-test.jenkinsfile.txt | 10 ++ .../opensearch/bwc-test.jenkinsfile.txt | 10 ++ .../opensearch/integ-test.jenkinsfile.txt | 9 ++ .../perf-test-with-security.jenkinsfile.txt | 11 +++ .../opensearch/perf-test.jenkinsfile.txt | 9 ++ .../RunPerfTestScriptLibTest.groovy | 1 + 26 files changed, 580 insertions(+), 56 deletions(-) create mode 100644 manifests/2.3.0/opensearch-2.3.0-test.yml create mode 100644 manifests/2.3.0/opensearch-dashboards-2.3.0-test.yml diff --git a/.github/ISSUE_TEMPLATE/release_template.md b/.github/ISSUE_TEMPLATE/release_template.md index 2905537aa3..9205584331 100644 --- a/.github/ISSUE_TEMPLATE/release_template.md +++ b/.github/ISSUE_TEMPLATE/release_template.md @@ -40,6 +40,7 @@ To aid in understanding the state of the release there is a table with status in - [ ] [Create a release issue in every component repo](https://github.com/opensearch-project/opensearch-build/blob/main/meta/README.md#create-a-release-issue) that links back to this issue, update Components section with these links. - [ ] Ensure the label is created in each component repo for this new version, and the next minor release. [Create a version label](https://github.com/opensearch-project/opensearch-plugins/blob/main/META.md#create-or-update-labels-in-all-plugin-repos) - [ ] Ensure that all release issues created above are assigned to an owner in the component team. +- [ ] Increase the build frequency for the this release from once a day (H 1 * * *) to once every hour (H/60 * * * *) in [jenkinsFile](https://github.com/opensearch-project/opensearch-build/blob/main/jenkins/check-for-build.jenkinsfile) ### CI/CD (Feature Freeze) - _Ends __REPLACE_RELEASE-minus-14-days__ diff --git a/build.gradle b/build.gradle index 992f59583c..691e7da4ac 100644 --- a/build.gradle +++ b/build.gradle @@ -31,7 +31,7 @@ dependencies { testImplementation group: 'junit', name: 'junit', version: '4.13.2' implementation group: 'org.codehaus.groovy', name: 'groovy-all', version: '3.0.9', ext: 'pom' implementation group: 'com.cloudbees', name: 'groovy-cps', version: '1.31' - testImplementation group: 'org.yaml', name: 'snakeyaml', version: '1.29' + testImplementation group: 'org.yaml', name: 'snakeyaml', version: '1.31' testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.4.1' testImplementation group: 'com.lesfurets', name:'jenkins-pipeline-unit', version: '1.13' } @@ -56,20 +56,10 @@ sourceSets { srcDirs = ['tests/jenkins'] } } - - jobs { - groovy { - srcDirs 'src/jenkins/jobs' - compileClasspath += main.compileClasspath - } - - compileClasspath += sourceSets.main.output - runtimeClasspath += sourceSets.main.output - } } sharedLibrary { - coreVersion = '2.358' // https://repo.jenkins-ci.org/public/org/jenkins-ci/main/jenkins-core/ + coreVersion = '2.364' // https://repo.jenkins-ci.org/public/org/jenkins-ci/main/jenkins-core/ testHarnessVersion = '1736.vc72c458c5103' // https://mvnrepository.com/artifact/org.jenkins-ci.main/jenkins-test-harness?repo=jenkins-releases pluginDependencies { workflowCpsGlobalLibraryPluginVersion = '570.v21311f4951f8' // https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/workflow/workflow-cps-global-lib/ @@ -79,7 +69,7 @@ sharedLibrary { dependency('org.jenkins-ci.plugins', 'pipeline-input-step', '449.v77f0e8b_845c4') // https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/pipeline-input-step/ dependency('org.jenkins-ci.plugins', 'script-security', '1172.v35f6a_0b_8207e') dependency('org.jenkins-ci.plugins', 'credentials', '1112.vc87b_7a_3597f6') - dependency('org.jenkins-ci.plugins', 'git-client', '3.10.1') + dependency('org.jenkins-ci.plugins', 'git-client', '3.11.1') dependency('org.jenkins-ci.plugins', 'junit', '1.55') dependency('org.jenkins-ci.plugins', 'mailer', '408.vd726a_1130320') // https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/mailer/ } @@ -127,3 +117,7 @@ jacocoTestReport { xml.required = true } } + +tasks.test { + maxParallelForks = Runtime.getRuntime().availableProcessors() +} diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile index e8a2e117eb..c9a83a9e90 100644 --- a/jenkins/check-for-build.jenkinsfile +++ b/jenkins/check-for-build.jenkinsfile @@ -11,10 +11,10 @@ pipeline { } triggers { parameterizedCron ''' - H 1 * * * %INPUT_MANIFEST=2.3.0/opensearch-dashboards-2.3.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards - H 1 * * * %INPUT_MANIFEST=2.3.0/opensearch-2.3.0.yml;TARGET_JOB_NAME=distribution-build-opensearch - H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch - H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards + H/60 * * * * %INPUT_MANIFEST=2.3.0/opensearch-dashboards-2.3.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_BRANCH=main + H/60 * * * * %INPUT_MANIFEST=2.3.0/opensearch-2.3.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_BRANCH=main + H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_BRANCH=main + H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_BRANCH=main ''' } parameters { @@ -28,6 +28,12 @@ pipeline { description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0-test.yml. Currently only applicable for distribution-build-opensearch job', trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) string( name: 'TARGET_JOB_NAME', description: 'Job to trigger if build has changed', @@ -74,7 +80,8 @@ pipeline { } build job: "${TARGET_JOB_NAME}", parameters: [ string(name: 'INPUT_MANIFEST', value: "${INPUT_MANIFEST}"), - string(name: 'TEST_MANIFEST', value: "${TEST_MANIFEST}") + string(name: 'TEST_MANIFEST', value: "${TEST_MANIFEST}"), + string(name: 'BUILD_BRANCH', value: "${BUILD_BRANCH}") ], wait: true echo "Build succeeded, uploading build SHA for that job" diff --git a/jenkins/opensearch-dashboards/bwc-test.jenkinsfile b/jenkins/opensearch-dashboards/bwc-test.jenkinsfile index 84a91034db..61b94d06c1 100644 --- a/jenkins/opensearch-dashboards/bwc-test.jenkinsfile +++ b/jenkins/opensearch-dashboards/bwc-test.jenkinsfile @@ -3,11 +3,13 @@ lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) pipeline { options { timeout(time: 3, unit: 'HOURS') + skipDefaultCheckout() } agent none environment { BUILD_MANIFEST = "build-manifest.yml" DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch-dashboards" + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } parameters { string( @@ -20,6 +22,12 @@ pipeline { description: 'The build manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/2.0.0/98/linux/x64/builds/opensearch-dashboards/manifest.yml.', trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) string( name: 'AGENT_LABEL', description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host.', @@ -27,6 +35,19 @@ pipeline { ) } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_LABEL + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('verify-parameters') { agent { node { @@ -34,6 +55,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { currentBuild.description = TEST_MANIFEST if (AGENT_LABEL == '') { @@ -63,6 +85,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { DOCKER_AGENT = detectTestDockerAgent() } @@ -78,6 +101,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = downloadBuildManifest( url: BUILD_MANIFEST_URL, @@ -99,6 +123,7 @@ pipeline { post { always { script { + unstash(name: 'sourceRepo') uploadTestResults( buildManifestFileName: BUILD_MANIFEST, jobName: JOB_NAME @@ -113,6 +138,7 @@ pipeline { post { success { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get(['bwc-test']) publishNotification( @@ -122,13 +148,13 @@ pipeline { credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) - postCleanup() } } } failure { node(AGENT_LABEL) { + unstash 'sourceRepo' script { def stashed = lib.jenkins.Messages.new(this).get(['bwc-test']) publishNotification( @@ -138,7 +164,6 @@ pipeline { credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) - postCleanup() } } diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index f6daefa585..1a5896f782 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -3,12 +3,14 @@ lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) pipeline { options { timeout(time: 4, unit: 'HOURS') + skipDefaultCheckout() } agent none environment { AGENT_X64 = 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' AGENT_ARM64 = 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host' IMAGE_RPM = 'opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2' // required for rpm to create digest sha256 correctly with rpm 4.12+ + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } parameters { string( @@ -38,6 +40,12 @@ pipeline { defaultValue: "bwc-test-opensearch-dashboards", trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) choice( name: 'BUILD_DOCKER', description: 'Build docker image or not with options.', @@ -50,6 +58,19 @@ pipeline { ) } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_X64 + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('detect docker image + args') { agent { docker { @@ -59,6 +80,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { currentBuild.description = "$INPUT_MANIFEST" dockerAgent = detectDockerAgent() @@ -74,6 +96,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildInfoYaml( componentName: COMPONENT_NAME, @@ -101,6 +124,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = buildAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -177,6 +201,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", @@ -184,7 +209,6 @@ pipeline { "\n${env.ARTIFACT_URL_X64_TAR_INTEG_TEST_RESULT}" + "\n${env.ARTIFACT_URL_X64_TAR_BWC_TEST_RESULT}" ) - postCleanup() } } @@ -203,6 +227,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildArchive( componentName: "${COMPONENT_NAME}", @@ -229,6 +254,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -266,6 +292,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", @@ -293,6 +320,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildArchive( componentName: "${COMPONENT_NAME}", @@ -320,6 +348,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -396,6 +425,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", @@ -403,7 +433,6 @@ pipeline { "\n${env.ARTIFACT_URL_ARM64_TAR_INTEG_TEST_RESULT}" + "\n${env.ARTIFACT_URL_ARM64_TAR_BWC_TEST_RESULT}" ) - postCleanup() } } @@ -424,6 +453,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildArchive( componentName: "${COMPONENT_NAME}", @@ -450,6 +480,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -486,6 +517,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", @@ -512,6 +544,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { uploadIndexFile( indexFilePath: env.INDEX_FILE_PATH @@ -535,6 +568,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { echo "env.ARTIFACT_URL_X64_TAR: ${env.ARTIFACT_URL_X64_TAR}" echo "env.ARTIFACT_URL_ARM64_TAR: ${env.ARTIFACT_URL_ARM64_TAR}" @@ -553,6 +587,7 @@ pipeline { post { always { node(AGENT_X64) { + unstash(name: 'sourceRepo') script { buildInfoYaml( componentName: COMPONENT_NAME, @@ -567,6 +602,7 @@ pipeline { } success { node(AGENT_X64) { + unstash(name: 'sourceRepo') script { if (params.PUBLISH_NOTIFICATION) { def stashed = lib.jenkins.Messages.new(this).get([ @@ -584,13 +620,13 @@ pipeline { manifest: "${INPUT_MANIFEST}" ) } - postCleanup() } } } failure { node(AGENT_X64) { + unstash(name: 'sourceRepo') script { if (params.PUBLISH_NOTIFICATION) { publishNotification( @@ -600,7 +636,6 @@ pipeline { manifest: "${INPUT_MANIFEST}" ) } - postCleanup() } } diff --git a/jenkins/opensearch-dashboards/integ-test.jenkinsfile b/jenkins/opensearch-dashboards/integ-test.jenkinsfile index d09c4461c0..280043412b 100644 --- a/jenkins/opensearch-dashboards/integ-test.jenkinsfile +++ b/jenkins/opensearch-dashboards/integ-test.jenkinsfile @@ -3,11 +3,13 @@ lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) pipeline { options { timeout(time: 3, unit: 'HOURS') + skipDefaultCheckout() } agent none environment { BUILD_MANIFEST = "build-manifest.yml" DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch-dashboards" + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } parameters { string( @@ -25,11 +27,31 @@ pipeline { description: 'The agent label where the tests should be executed. For x64 use Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host,for arm64 use Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host', trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_LABEL + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('verify-parameters') { agent { label AGENT_LABEL } steps { + unstash(name: 'sourceRepo') script { currentBuild.description = BUILD_MANIFEST_URL if (AGENT_LABEL == '') { @@ -59,6 +81,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { DOCKER_AGENT = detectTestDockerAgent() } @@ -74,6 +97,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = downloadBuildManifest( url: BUILD_MANIFEST_URL, @@ -94,6 +118,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { uploadTestResults( buildManifestFileName: BUILD_MANIFEST, @@ -109,6 +134,7 @@ pipeline { post { success { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get(['integ-test']) publishNotification( @@ -118,13 +144,13 @@ pipeline { credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) - postCleanup() } } } failure { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get(['integ-test']) publishNotification( @@ -134,7 +160,6 @@ pipeline { credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) - postCleanup() } } diff --git a/jenkins/opensearch/bwc-test.jenkinsfile b/jenkins/opensearch/bwc-test.jenkinsfile index bceb4d8ea4..02842e5378 100644 --- a/jenkins/opensearch/bwc-test.jenkinsfile +++ b/jenkins/opensearch/bwc-test.jenkinsfile @@ -3,11 +3,13 @@ lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) pipeline { options { timeout(time: 3, unit: 'HOURS') + skipDefaultCheckout() } agent none environment { BUILD_MANIFEST = "build-manifest.yml" DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch" + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } parameters { string( @@ -25,8 +27,27 @@ pipeline { description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host.', trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_LABEL + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('verify-parameters') { agent { node { @@ -34,6 +55,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { currentBuild.description = TEST_MANIFEST if (AGENT_LABEL == '') { @@ -63,6 +85,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { DOCKER_AGENT = detectTestDockerAgent() } @@ -78,6 +101,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = downloadBuildManifest( url: BUILD_MANIFEST_URL, @@ -99,6 +123,7 @@ pipeline { post { always { script { + unstash(name: 'sourceRepo') uploadTestResults( buildManifestFileName: BUILD_MANIFEST, jobName: JOB_NAME @@ -114,6 +139,7 @@ pipeline { success { node(AGENT_LABEL) { script { + unstash(name: 'sourceRepo') def stashed = lib.jenkins.Messages.new(this).get(['bwc-test']) publishNotification( icon: ':white_check_mark:', @@ -122,13 +148,13 @@ pipeline { credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) - postCleanup() } } } failure { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get(['bwc-test']) publishNotification( @@ -138,10 +164,9 @@ pipeline { credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) - postCleanup() } } } } -} +} \ No newline at end of file diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index ad2014549a..6bb8982cfb 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -3,12 +3,14 @@ lib = library(identifier: "jenkins@20211123", retriever: legacySCM(scm)) pipeline { options { timeout(time: 4, unit: 'HOURS') + skipDefaultCheckout() } agent none environment { AGENT_X64 = 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' AGENT_ARM64 = 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host' IMAGE_RPM = 'opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2' // required for rpm to create digest sha256 correctly with rpm 4.12+ + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } parameters { string( @@ -38,6 +40,12 @@ pipeline { defaultValue: "bwc-test", trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) choice( name: 'BUILD_DOCKER', description: 'Build docker image or not with options.', @@ -55,6 +63,19 @@ pipeline { ) } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_X64 + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('detect docker image + args') { agent { docker { @@ -64,6 +85,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { dockerAgent = detectDockerAgent() currentBuild.description = INPUT_MANIFEST @@ -79,6 +101,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildInfoYaml( componentName: COMPONENT_NAME, @@ -109,6 +132,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildManifest( componentName: "${COMPONENT_NAME}", @@ -149,6 +173,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildManifest( componentName: "${COMPONENT_NAME}", @@ -179,6 +204,7 @@ pipeline { jdk dockerAgent.javaVersion } steps { + unstash(name: 'sourceRepo') script { buildManifest( componentName: "${COMPONENT_NAME}", @@ -199,7 +225,7 @@ pipeline { } } } - stage('build-snapshot-windows-x64-zip') { + /*stage('build-snapshot-windows-x64-zip') { agent { node { label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host' @@ -229,7 +255,7 @@ pipeline { postCleanup() } } - } + }*/ stage('build-and-test-x64-tar') { agent { docker { @@ -240,6 +266,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = buildAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -314,6 +341,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", @@ -321,7 +349,6 @@ pipeline { "\n${env.ARTIFACT_URL_X64_TAR_INTEG_TEST_RESULT}" + "\n${env.ARTIFACT_URL_X64_TAR_BWC_TEST_RESULT}" ) - postCleanup() } } @@ -340,6 +367,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildArchive( componentName: "${COMPONENT_NAME}", @@ -366,6 +394,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -403,13 +432,13 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + "\n${env.RPM_VALIDATION_X64_RPM_TEST_RESULT}" ) - postCleanup() } } @@ -427,6 +456,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = buildAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -500,6 +530,7 @@ pipeline { } post { always { + unstash(name: 'sourceRepo') script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", @@ -507,7 +538,6 @@ pipeline { "\n${env.ARTIFACT_URL_ARM64_TAR_INTEG_TEST_RESULT}" + "\n${env.ARTIFACT_URL_ARM64_TAR_BWC_TEST_RESULT}" ) - postCleanup() } } @@ -526,6 +556,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { buildArchive( componentName: "${COMPONENT_NAME}", @@ -552,6 +583,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", @@ -595,7 +627,6 @@ pipeline { lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + "\n${env.RPM_VALIDATION_ARM64_RPM_TEST_RESULT}" ) - postCleanup() } } @@ -615,6 +646,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { uploadIndexFile( indexFilePath: env.INDEX_FILE_PATH @@ -638,6 +670,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { echo "env.ARTIFACT_URL_X64_TAR: ${env.ARTIFACT_URL_X64_TAR}" echo "env.ARTIFACT_URL_ARM64_TAR: ${env.ARTIFACT_URL_ARM64_TAR}" @@ -656,6 +689,7 @@ pipeline { post { always { node(AGENT_X64) { + unstash(name: 'sourceRepo') script { buildInfoYaml( componentName: COMPONENT_NAME, @@ -670,6 +704,7 @@ pipeline { } success { node(AGENT_X64) { + unstash(name: 'sourceRepo') script { if (params.PUBLISH_NOTIFICATION) { def stashed = lib.jenkins.Messages.new(this).get([ @@ -687,13 +722,13 @@ pipeline { manifest: "${INPUT_MANIFEST}" ) } - postCleanup() } } } failure { node(AGENT_X64) { + unstash(name: 'sourceRepo') script { if (params.PUBLISH_NOTIFICATION) { publishNotification( @@ -708,11 +743,9 @@ pipeline { message: buildFailureMessage() ) } - postCleanup() } } } } } - diff --git a/jenkins/opensearch/integ-test.jenkinsfile b/jenkins/opensearch/integ-test.jenkinsfile index 9f92bacb55..6f4efb6603 100644 --- a/jenkins/opensearch/integ-test.jenkinsfile +++ b/jenkins/opensearch/integ-test.jenkinsfile @@ -3,12 +3,14 @@ lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) pipeline { options { timeout(time: 3, unit: 'HOURS') + skipDefaultCheckout() } agent none environment { BUILD_MANIFEST = "build-manifest.yml" DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch" ARTIFACT_BUCKET_NAME = credentials('jenkins-artifact-bucket-name') + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } parameters { string( @@ -26,11 +28,31 @@ pipeline { description: 'The agent label where the tests should be executed. For x64 use Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host,for arm64 use Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host', trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_LABEL + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('verify-parameters') { agent { label AGENT_LABEL } steps { + unstash(name: 'sourceRepo') script { currentBuild.description = BUILD_MANIFEST_URL if (AGENT_LABEL == '') { @@ -60,6 +82,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { DOCKER_AGENT = detectTestDockerAgent() } @@ -70,6 +93,7 @@ pipeline { // Can only be run in runner that is at least 50GB per container agent { label AGENT_LABEL } steps { + unstash(name: 'sourceRepo') script { downloadBuildManifest( @@ -159,6 +183,7 @@ pipeline { success { node(AGENT_LABEL) { script { + unstash(name: 'sourceRepo') def stashed = lib.jenkins.Messages.new(this).get(['integ-test']) publishNotification( icon: ':white_check_mark:', @@ -174,6 +199,7 @@ pipeline { } failure { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get(['integ-test']) publishNotification( @@ -189,4 +215,4 @@ pipeline { } } } -} +} \ No newline at end of file diff --git a/jenkins/opensearch/perf-test.jenkinsfile b/jenkins/opensearch/perf-test.jenkinsfile index b81a71304d..2b992fd11f 100644 --- a/jenkins/opensearch/perf-test.jenkinsfile +++ b/jenkins/opensearch/perf-test.jenkinsfile @@ -4,12 +4,14 @@ pipeline { agent none options { timeout(time: 15, unit: 'HOURS') + skipDefaultCheckout() } environment { AGENT_LABEL = 'Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Perf-Test' AGENT_IMAGE = 'opensearchstaging/ci-runner:ci-runner-centos7-performance-test-v1' BUNDLE_MANIFEST = 'bundle-manifest.yml' JOB_NAME = 'perf-test' + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } triggers { parameterizedCron ''' @@ -47,9 +49,27 @@ pipeline { description: 'Number of times to run a workload before collecting data for Mensor (internal client).', trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) } - stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_LABEL + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash(name: 'sourceRepo') + } + } stage('validate-and-set-parameters') { agent { docker { @@ -59,6 +79,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { if (BUNDLE_MANIFEST_URL == '') { currentBuild.result = 'ABORTED' @@ -100,6 +121,7 @@ pipeline { expression { return env.HAS_SECURITY.toBoolean() } } steps { + unstash(name: 'sourceRepo') script { def bundleManifestObj = downloadBuildManifest( url: BUNDLE_MANIFEST_URL, @@ -125,6 +147,7 @@ pipeline { } post { success { + unstash(name: 'sourceRepo') script { uploadTestResults( buildManifestFileName: BUNDLE_MANIFEST, @@ -151,6 +174,7 @@ pipeline { } } steps { + unstash(name: 'sourceRepo') script { def bundleManifestObj = downloadBuildManifest( url: BUNDLE_MANIFEST_URL, @@ -177,6 +201,7 @@ pipeline { } post { success { + unstash(name: 'sourceRepo') script { uploadTestResults( buildManifestFileName: BUNDLE_MANIFEST, @@ -201,6 +226,7 @@ pipeline { post { success { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get([JOB_NAME]) publishNotification( @@ -215,6 +241,7 @@ pipeline { } failure { node(AGENT_LABEL) { + unstash(name: 'sourceRepo') script { def stashed = lib.jenkins.Messages.new(this).get([JOB_NAME]) publishNotification( diff --git a/jenkins/release-notes-check/release-notes-check.jenkinsfile b/jenkins/release-notes-check/release-notes-check.jenkinsfile index bc4dd84697..d169d1cbf6 100644 --- a/jenkins/release-notes-check/release-notes-check.jenkinsfile +++ b/jenkins/release-notes-check/release-notes-check.jenkinsfile @@ -2,7 +2,8 @@ lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) pipeline { options { - timeout(time: 2, unit: 'HOURS') + timeout(time: 2, unit: 'HOURS') + skipDefaultCheckout true } agent none parameters { @@ -31,11 +32,31 @@ pipeline { description: 'Use only with COMMENT: UPDATE', trim: true ) + string( + name: 'BUILD_BRANCH', + description: "Build repo branch to use.", + defaultValue: "main", + trim: true + ) } environment { AGENT_X64 = 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' + BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git' } stages { + stage('checkout the build repo') { + agent { + docker { + label AGENT_X64 + image 'alpine:3' + alwaysPull true + } + } + steps { + git url: BUILD_REPO, branch: BUILD_BRANCH + stash 'sourceRepo' + } + } stage('detect docker image + args') { agent { docker { @@ -45,19 +66,23 @@ pipeline { } } steps { + unstash 'sourceRepo' script { dockerAgent = detectDockerAgent() currentBuild.description = INPUT_MANIFEST } } - post { - always { - postCleanup() - } - } } stage('Parameters Check') { + agent { + docker { + label AGENT_X64 + image dockerAgent.image + alwaysPull true + } + } steps { + unstash 'sourceRepo' script { currentBuild.description = "Comment:${COMMENT}, Manifest:${INPUT_MANIFEST}" if (GIT_LOG_DATE.isEmpty() || INPUT_MANIFEST.isEmpty()) { @@ -84,6 +109,7 @@ pipeline { } } steps { + unstash 'sourceRepo' script { withCredentials([usernamePassword(credentialsId: "jenkins-github-bot-token", usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) { if (params.COMMENT == "ADD") { @@ -115,10 +141,12 @@ pipeline { } } } - post { - always { - postCleanup() - } + } + } + post { + always { + node(AGENT_X64) { + postCleanup() } } } diff --git a/manifests/2.3.0/opensearch-2.3.0-test.yml b/manifests/2.3.0/opensearch-2.3.0-test.yml new file mode 100644 index 0000000000..fb4cd5bb30 --- /dev/null +++ b/manifests/2.3.0/opensearch-2.3.0-test.yml @@ -0,0 +1,94 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1] + + - name: notifications + working-directory: notifications + integ-test: + test-configs: + - with-security + - without-security + + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + + - name: dashboards-reports + working-directory: reports-scheduler + integ-test: + test-configs: + - without-security + + - name: opensearch-observability + working-directory: opensearch-observability + integ-test: + test-configs: + - without-security + + - name: ml-commons + integ-test: + test-configs: + - with-security + - without-security + + - name: cross-cluster-replication + integ-test: + test-configs: + - with-security + - without-security + + - name: security + integ-test: + test-configs: + - with-security + + - name: geospatial + integ-test: + test-configs: + - without-security diff --git a/manifests/2.3.0/opensearch-2.3.0.yml b/manifests/2.3.0/opensearch-2.3.0.yml index 9693b9d83b..d13e8479aa 100644 --- a/manifests/2.3.0/opensearch-2.3.0.yml +++ b/manifests/2.3.0/opensearch-2.3.0.yml @@ -10,21 +10,29 @@ ci: components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git - ref: 2.x + ref: '2.3' checks: - gradle:publish - gradle:properties:version - name: common-utils repository: https://github.com/opensearch-project/common-utils.git - ref: 2.x + ref: '2.3' platforms: - linux checks: - gradle:publish - gradle:properties:version + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: '2.3' + platforms: + - linux + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version - name: security repository: https://github.com/opensearch-project/security.git - ref: 2.x + ref: '2.3' - name: performance-analyzer repository: https://github.com/opensearch-project/performance-analyzer.git ref: '2.3' @@ -33,3 +41,87 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version + - name: cross-cluster-replication + repository: https://github.com/opensearch-project/cross-cluster-replication.git + ref: '2.3' + platforms: + - linux + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: anomaly-detection + repository: https://github.com/opensearch-project/anomaly-detection.git + ref: '2.3' + platforms: + - linux + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: ml-commons + repository: https://github.com/opensearch-project/ml-commons.git + ref: '2.3' + platforms: + - linux + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: opensearch-ml-plugin + - name: k-NN + repository: https://github.com/opensearch-project/k-NN.git + ref: '2.3' + platforms: + - linux + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: geospatial + repository: https://github.com/opensearch-project/geospatial.git + ref: '2.3' + platforms: + - linux + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: opensearch-observability + repository: https://github.com/opensearch-project/observability.git + ref: '2.3' + platforms: + - linux + working_directory: opensearch-observability + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: dashboards-reports + repository: https://github.com/opensearch-project/dashboards-reports.git + ref: '2.3' + platforms: + - linux + working_directory: reports-scheduler + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: notifications-core + repository: https://github.com/opensearch-project/notifications.git + ref: '2.3' + platforms: + - linux + working_directory: notifications + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: opensearch-notifications-core + - name: notifications + repository: https://github.com/opensearch-project/notifications.git + ref: '2.3' + platforms: + - linux + working_directory: notifications + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: notifications + - name: asynchronous-search + repository: https://github.com/opensearch-project/asynchronous-search.git + ref: '2.3' + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux diff --git a/manifests/2.3.0/opensearch-dashboards-2.3.0-test.yml b/manifests/2.3.0/opensearch-dashboards-2.3.0-test.yml new file mode 100644 index 0000000000..3ad90cc802 --- /dev/null +++ b/manifests/2.3.0/opensearch-dashboards-2.3.0-test.yml @@ -0,0 +1,17 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2 +components: + - name: OpenSearch-Dashboards + bwc-test: + test-configs: + - with-security + - without-security + - name: functionalTestDashboards + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/2.3.0/opensearch-dashboards-2.3.0.yml b/manifests/2.3.0/opensearch-dashboards-2.3.0.yml index 1673213816..7403a23f39 100644 --- a/manifests/2.3.0/opensearch-dashboards-2.3.0.yml +++ b/manifests/2.3.0/opensearch-dashboards-2.3.0.yml @@ -9,4 +9,33 @@ ci: components: - name: OpenSearch-Dashboards repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git - ref: 2.x + ref: '2.3' + - name: functionalTestDashboards + repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git + ref: '2.3' + - name: securityDashboards + repository: https://github.com/opensearch-project/security-dashboards-plugin.git + ref: '2.3' + - name: anomalyDetectionDashboards + repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin + ref: '2.3' + - name: observabilityDashboards + repository: https://github.com/opensearch-project/observability.git + working_directory: dashboards-observability + ref: '2.3' + - name: ganttChartDashboards + repository: https://github.com/opensearch-project/dashboards-visualizations.git + working_directory: gantt-chart + ref: '2.3' + - name: reportsDashboards + repository: https://github.com/opensearch-project/dashboards-reports.git + working_directory: dashboards-reports + ref: '2.3' + - name: customImportMapDashboards + repository: https://github.com/opensearch-project/dashboards-maps.git + working_directory: src/plugins/custom_import_map + ref: '2.3' + - name: notificationsDashboards + repository: https://github.com/opensearch-project/notifications.git + working_directory: dashboards-notifications + ref: '2.3' diff --git a/tests/jenkins/TestOpenSearchBwcTest.groovy b/tests/jenkins/TestOpenSearchBwcTest.groovy index 1df9ab2fb9..0853cb0c01 100644 --- a/tests/jenkins/TestOpenSearchBwcTest.groovy +++ b/tests/jenkins/TestOpenSearchBwcTest.groovy @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ - import jenkins.tests.BuildPipelineTest import org.junit.Before import org.junit.Test @@ -38,6 +37,7 @@ class TestOpenSearchBwcTest extends BuildPipelineTest { binding.setVariable('AGENT_LABEL', agentLabel) binding.setVariable('BUILD_MANIFEST', buildManifest) binding.setVariable('BUILD_ID', "${buildId}") + binding.setVariable('BUILD_BRANCH', "main") def env = binding.getVariable('env') env['DOCKER_AGENT'] = [image:'opensearchstaging/ci-runner:ci-runner-centos7-v1', args:'-e JAVA_HOME=/opt/java/openjdk-11'] @@ -48,6 +48,7 @@ class TestOpenSearchBwcTest extends BuildPipelineTest { }) helper.registerAllowedMethod('findFiles', [Map.class], null) + helper.registerAllowedMethod("git", [Map]) } @Test diff --git a/tests/jenkins/TestOpenSearchDashboardsBwcTest.groovy b/tests/jenkins/TestOpenSearchDashboardsBwcTest.groovy index db1ca6b5be..bcbe0e238a 100644 --- a/tests/jenkins/TestOpenSearchDashboardsBwcTest.groovy +++ b/tests/jenkins/TestOpenSearchDashboardsBwcTest.groovy @@ -38,6 +38,7 @@ class TestOpenSearchDashboardsBwcTest extends BuildPipelineTest { binding.setVariable('AGENT_LABEL', agentLabel) binding.setVariable('BUILD_MANIFEST', buildManifest) binding.setVariable('BUILD_ID', "${buildId}") + binding.setVariable('BUILD_BRANCH', "main") def env = binding.getVariable('env') env['DOCKER_AGENT'] = [image:'opensearchstaging/ci-runner:ci-runner-centos7-v1', args:'-e JAVA_HOME=/opt/java/openjdk-11'] @@ -48,6 +49,7 @@ class TestOpenSearchDashboardsBwcTest extends BuildPipelineTest { }) helper.registerAllowedMethod('findFiles', [Map.class], null) + helper.registerAllowedMethod("git", [Map]) } @Test diff --git a/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy b/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy index 0f1069a2d2..99ddcee740 100644 --- a/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy +++ b/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy @@ -48,7 +48,7 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest { }) helper.registerAllowedMethod('findFiles', [Map.class], null) - helper.registerAllowedMethod('unstash', [String.class], null) + helper.registerAllowedMethod("git", [Map]) } @Test diff --git a/tests/jenkins/TestOpenSearchIntegTest.groovy b/tests/jenkins/TestOpenSearchIntegTest.groovy index e724fd7b7b..4061e630e2 100644 --- a/tests/jenkins/TestOpenSearchIntegTest.groovy +++ b/tests/jenkins/TestOpenSearchIntegTest.groovy @@ -44,6 +44,8 @@ class TestOpenSearchIntegTest extends BuildPipelineTest { binding.setVariable('BUILD_MANIFEST', buildManifest) binding.setVariable('BUILD_ID', "${buildId}") binding.setVariable('ARTIFACT_BUCKET_NAME', bucketName) + binding.setVariable('BUILD_BRANCH', "main") + binding.setVariable('BUILD_BRANCH', "main") def env = binding.getVariable('env') env['DOCKER_AGENT'] = [image:'opensearchstaging/ci-runner:ci-runner-centos7-v1', args:'-e JAVA_HOME=/opt/java/openjdk-11'] @@ -58,6 +60,7 @@ class TestOpenSearchIntegTest extends BuildPipelineTest { helper.registerAllowedMethod('fileExists', [String.class], { args -> return true; }) + helper.registerAllowedMethod("git", [Map]) } @Test diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt index 0c62a6ff2b..709c9d5fb8 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt @@ -3,13 +3,20 @@ bwc-test.library({identifier=jenkins@20211118, retriever=null}) bwc-test.pipeline(groovy.lang.Closure) bwc-test.timeout({time=3, unit=HOURS}) + bwc-test.skipDefaultCheckout() bwc-test.echo(Executing on agent [label:none]) + bwc-test.stage(checkout the build repo, groovy.lang.Closure) + bwc-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + bwc-test.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + bwc-test.stash({name=sourceRepo}) bwc-test.stage(verify-parameters, groovy.lang.Closure) bwc-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) bwc-test.fileExists(manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml) bwc-test.stage(detect docker image + args, groovy.lang.Closure) bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) bwc-test.detectTestDockerAgent() detectTestDockerAgent.legacySCM(groovy.lang.Closure) @@ -19,6 +26,7 @@ detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) bwc-test.stage(bwc-test, groovy.lang.Closure) bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:-e JAVA_HOME=/opt/java/openjdk-11, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) bwc-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/1.2.0/215/linux/x64/dist/opensearch-dashboards/opensearch-dashboards-1.2.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -39,6 +47,7 @@ runBwcTestScript.echo(Paths: opensearch-dashboards=https://ci.opensearch.org/ci/dbc/dummy_job/1.2.0/215/linux/x64/tar) runBwcTestScript.sh(./test.sh bwc-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --test-run-id 487 --paths opensearch-dashboards=https://ci.opensearch.org/ci/dbc/dummy_job/1.2.0/215/linux/x64/tar) bwc-test.script(groovy.lang.Closure) + bwc-test.unstash({name=sourceRepo}) bwc-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) @@ -60,6 +69,7 @@ bwc-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) bwc-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([bwc-test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt index 5d013a9304..aeebc7d546 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt @@ -3,13 +3,20 @@ integ-test.library({identifier=jenkins@20211118, retriever=null}) integ-test.pipeline(groovy.lang.Closure) integ-test.timeout({time=3, unit=HOURS}) + integ-test.skipDefaultCheckout() integ-test.echo(Executing on agent [label:none]) + integ-test.stage(checkout the build repo, groovy.lang.Closure) + integ-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + integ-test.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + integ-test.stash({name=sourceRepo}) integ-test.stage(verify-parameters, groovy.lang.Closure) integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.fileExists(manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml) integ-test.stage(detect docker image + args, groovy.lang.Closure) integ-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.detectTestDockerAgent() detectTestDockerAgent.legacySCM(groovy.lang.Closure) @@ -19,6 +26,7 @@ detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) integ-test.stage(integ-test, groovy.lang.Closure) integ-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:-e JAVA_HOME=/opt/java/openjdk-11, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/1.2.0/215/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-1.2.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -41,6 +49,7 @@ runIntegTestScript.echo(Paths: opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.0/latest/linux/x64/tar opensearch-dashboards=https://ci.opensearch.org/ci/dbc/dummy_job/1.2.0/215/linux/x64/tar) runIntegTestScript.echo(Component: functionalTestDashboards) runIntegTestScript.sh(./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component functionalTestDashboards --test-run-id 487 --paths opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.0/latest/linux/x64/tar opensearch-dashboards=https://ci.opensearch.org/ci/dbc/dummy_job/1.2.0/215/linux/x64/tar) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) @@ -63,6 +72,7 @@ integ-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) integ-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([integ-test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt index 8c8c41d3ac..cccf7f2130 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt @@ -3,13 +3,20 @@ bwc-test.library({identifier=jenkins@20211118, retriever=null}) bwc-test.pipeline(groovy.lang.Closure) bwc-test.timeout({time=3, unit=HOURS}) + bwc-test.skipDefaultCheckout() bwc-test.echo(Executing on agent [label:none]) + bwc-test.stage(checkout the build repo, groovy.lang.Closure) + bwc-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + bwc-test.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + bwc-test.stash({name=sourceRepo}) bwc-test.stage(verify-parameters, groovy.lang.Closure) bwc-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) bwc-test.fileExists(manifests/tests/jenkins/data/opensearch-1.3.0-test.yml) bwc-test.stage(detect docker image + args, groovy.lang.Closure) bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) bwc-test.detectTestDockerAgent() detectTestDockerAgent.legacySCM(groovy.lang.Closure) @@ -19,6 +26,7 @@ detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) bwc-test.stage(bwc-test, groovy.lang.Closure) bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:-e JAVA_HOME=/opt/java/openjdk-11, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + bwc-test.unstash({name=sourceRepo}) bwc-test.script(groovy.lang.Closure) bwc-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-1.3.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -39,6 +47,7 @@ runBwcTestScript.echo(Paths: opensearch=https://ci.opensearch.org/ci/dbc/dummy_job/1.3.0/717/linux/x64/tar) runBwcTestScript.sh(./test.sh bwc-test manifests/tests/jenkins/data/opensearch-1.3.0-test.yml --test-run-id 487 --paths opensearch=https://ci.opensearch.org/ci/dbc/dummy_job/1.3.0/717/linux/x64/tar) bwc-test.script(groovy.lang.Closure) + bwc-test.unstash({name=sourceRepo}) bwc-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) @@ -61,6 +70,7 @@ postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) bwc-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) bwc-test.script(groovy.lang.Closure) + bwc-test.unstash({name=sourceRepo}) Messages.asBoolean() Messages.get([bwc-test]) bwc-test.unstash({name=messages-bwc-test}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt index b737b0a793..8bc6c2506c 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt @@ -4,13 +4,20 @@ integ-test.pipeline(groovy.lang.Closure) integ-test.credentials(jenkins-artifact-bucket-name) integ-test.timeout({time=3, unit=HOURS}) + integ-test.skipDefaultCheckout() integ-test.echo(Executing on agent [label:none]) + integ-test.stage(checkout the build repo, groovy.lang.Closure) + integ-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + integ-test.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + integ-test.stash({name=sourceRepo}) integ-test.stage(verify-parameters, groovy.lang.Closure) integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.fileExists(manifests/tests/jenkins/data/opensearch-1.3.0-test.yml) integ-test.stage(detect docker image + args, groovy.lang.Closure) integ-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.detectTestDockerAgent() detectTestDockerAgent.legacySCM(groovy.lang.Closure) @@ -20,6 +27,7 @@ detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) integ-test.stage(integ-test, groovy.lang.Closure) integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + integ-test.unstash({name=sourceRepo}) integ-test.script(groovy.lang.Closure) integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-1.3.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -86,6 +94,7 @@ postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) integ-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) integ-test.script(groovy.lang.Closure) + integ-test.unstash({name=sourceRepo}) Messages.asBoolean() Messages.get([integ-test]) integ-test.unstash({name=messages-integ-test}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt index 80fe4f9ee1..d15bbca63e 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt @@ -3,6 +3,7 @@ perf-test.library({identifier=jenkins@20211118, retriever=null}) perf-test.pipeline(groovy.lang.Closure) perf-test.timeout({time=15, unit=HOURS}) + perf-test.skipDefaultCheckout() perf-test.echo(Executing on agent [label:none]) perf-test.parameterizedCron( H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.5/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 @@ -14,8 +15,13 @@ H 9 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.2.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 H 9 * * 5 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.2.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 ) + perf-test.stage(checkout the build repo, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + perf-test.stash({name=sourceRepo}) perf-test.stage(validate-and-set-parameters, groovy.lang.Closure) perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -32,6 +38,7 @@ perf-test.stash({includes=messages/*, name=messages-perf-test}) perf-test.stage(test-with-security, groovy.lang.Closure) perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -72,6 +79,7 @@ Performance tests with security for 1236 completed) perf-test.writeFile({file=messages/perf-test.msg, text= Performance tests with security for 1236 completed}) perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=307}) uploadTestResults.legacySCM(groovy.lang.Closure) @@ -95,6 +103,7 @@ Performance tests with security for 1236 completed}) postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) perf-test.stage(test-without-security, groovy.lang.Closure) perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -135,6 +144,7 @@ Performance tests without security for 1236 completed) perf-test.writeFile({file=messages/perf-test.msg, text= Performance tests without security for 1236 completed}) perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=307}) uploadTestResults.legacySCM(groovy.lang.Closure) @@ -157,6 +167,7 @@ Performance tests without security for 1236 completed}) perf-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) perf-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([perf-test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt index f845091810..8ce8de2083 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt @@ -3,6 +3,7 @@ perf-test.library({identifier=jenkins@20211118, retriever=null}) perf-test.pipeline(groovy.lang.Closure) perf-test.timeout({time=15, unit=HOURS}) + perf-test.skipDefaultCheckout() perf-test.echo(Executing on agent [label:none]) perf-test.parameterizedCron( H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.5/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 @@ -14,8 +15,13 @@ H 9 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.2.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 H 9 * * 5 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.2.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 ) + perf-test.stage(checkout the build repo, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + perf-test.stash({name=sourceRepo}) perf-test.stage(validate-and-set-parameters, groovy.lang.Closure) perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -33,6 +39,7 @@ perf-test.echo(Skipping stage test-with-security) perf-test.stage(test-without-security, groovy.lang.Closure) perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -73,6 +80,7 @@ Performance tests without security for 1236 completed) perf-test.writeFile({file=messages/perf-test.msg, text= Performance tests without security for 1236 completed}) perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, jobName=perf-test, buildNumber=307}) uploadTestResults.legacySCM(groovy.lang.Closure) @@ -95,6 +103,7 @@ Performance tests without security for 1236 completed}) perf-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) perf-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) + perf-test.unstash({name=sourceRepo}) perf-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([perf-test]) diff --git a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy index 963da32209..a0b2477663 100644 --- a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy +++ b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy @@ -44,6 +44,7 @@ class RunPerfTestScriptLibTester extends LibFunctionTester { helper.registerAllowedMethod("downloadBuildManifest", [Map], { c -> lib.jenkins.BuildManifest.new(readYaml(file: bundleManifest)) }) + helper.registerAllowedMethod("git", [Map]) helper.registerAllowedMethod('parameterizedCron', [String], null) binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') binding.setVariable('AGENT_IMAGE', 'opensearchstaging/ci-runner:ci-runner-centos7-v1')