diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index 279cffb4d2..c706258837 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -9,8 +9,8 @@ pipeline { agent none environment { AGENT_X64 = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' - AGENT_ARM64 = 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host' - INTEG_TEST_JOB_NAME = 'integ-test-opensearch-dashboards' + AGENT_ARM64 = 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host' + DEFAULT_INTEG_TEST_JOB_NAME = 'integ-test-opensearch-dashboards' } parameters { string( @@ -23,8 +23,34 @@ pipeline { description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-dashboards-2.0.0-test.yml.', trim: true ) + string( + name: 'INTEG_TEST_JOB_NAME', + description: "Name of integration test job that will be triggered, e.g. Playground/integ-test-opensearch-dashboards.", + defaultValue: "integ-test-opensearch-dashboards", + trim: true + ) + booleanParam( + name: 'BUILD_DOCKER', + description: 'Build docker image or not.', + defaultValue: true + ) + booleanParam( + name: 'PUBLISH_NOTIFICATION', + description: 'Publish the status of this build job or not.', + defaultValue: true + ) } stages { + stage('verify-parameters') { + steps { + script { + env.INTEG_TEST_JOB_NAME = INTEG_TEST_JOB_NAME != '' ? + INTEG_TEST_JOB_NAME : + DEFAULT_INTEG_TEST_JOB_NAME + env.PUBLISH_NOTIFICATION = PUBLISH_NOTIFICATION == 'true' + } + } + } stage('detect docker image + args') { agent { docker { @@ -65,22 +91,19 @@ pipeline { echo "artifactUrl (x64): ${artifactUrl}" def integTestResults = - build job: INTEG_TEST_JOB_NAME, + build job: env.INTEG_TEST_JOB_NAME, propagate: false, wait: true, parameters: [ + string(name: 'INPUT_MANIFEST', value: INPUT_MANIFEST), string(name: 'TEST_MANIFEST', value: TEST_MANIFEST), string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl), - string(name: 'AGENT_LABEL', value: AGENT_X64), - string(name: 'CONTAINER_IMAGE', value: dockerAgent.image) + string(name: 'AGENT_LABEL', value: AGENT_X64) ] - String status = integTestResults.getResult() - String icon = status == 'SUCCESS' ? ':white_check_mark:' : ':warning:' - lib.jenkins.Messages.new(this).add( - STAGE_NAME, - lib.jenkins.Messages.new(this).get([STAGE_NAME]) + - "\nInteg Tests (x64): ${icon} ${status} ${integTestResults.getAbsoluteUrl()}" + createTestResultsMessage( + testType: "Integ Tests (x64)", + testResults: integTestResults ) } } @@ -141,22 +164,19 @@ pipeline { echo "artifactUrl (arm64): ${artifactUrl}" def integTestResults = - build job: INTEG_TEST_JOB_NAME, + build job: env.INTEG_TEST_JOB_NAME, propagate: false, wait: true, parameters: [ + string(name: 'INPUT_MANIFEST', value: INPUT_MANIFEST), string(name: 'TEST_MANIFEST', value: TEST_MANIFEST), string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl), - string(name: 'AGENT_LABEL', value: AGENT_ARM64), - string(name: 'CONTAINER_IMAGE', value: dockerAgent.image) + string(name: 'AGENT_LABEL', value: AGENT_ARM64) ] - String status = integTestResults.getResult() - String icon = status == 'SUCCESS' ? ':white_check_mark:' : ':warning:' - lib.jenkins.Messages.new(this).add( - STAGE_NAME, - lib.jenkins.Messages.new(this).get([STAGE_NAME]) + - "\nInteg Tests (arm64): ${icon} ${status} ${integTestResults.getAbsoluteUrl()}" + createTestResultsMessage( + testType: "Integ Tests (arm64)", + testResults: integTestResults ) } } @@ -171,6 +191,10 @@ pipeline { } } stage('docker build') { + when { + beforeAgent true + equals expected: true, actual: BUILD_DOCKER + } steps { node(AGENT_X64) { script { @@ -191,19 +215,20 @@ pipeline { success { node(AGENT_X64) { script { - def stashed = lib.jenkins.Messages.new(this).get([ - 'build-and-test-linux-x64', - 'build-archive-linux-arm64', - 'assemble-archive-and-test-linux-arm64' - ]) + if (env.PUBLISH_NOTIFICATION) { + def stashed = lib.jenkins.Messages.new(this).get([ + 'build-and-test-linux-x64', + 'assemble-archive-and-test-linux-arm64' + ]) - publishNotification( - icon: ':white_check_mark:', - message: 'Successful Build', - extra: stashed, - credentialsId: 'BUILD_NOTICE_WEBHOOK', - manifest: "${INPUT_MANIFEST}" - ) + publishNotification( + icon: ':white_check_mark:', + message: 'Successful Build', + extra: stashed, + credentialsId: 'BUILD_NOTICE_WEBHOOK', + manifest: "${INPUT_MANIFEST}" + ) + } postCleanup() } @@ -212,12 +237,14 @@ pipeline { failure { node(AGENT_X64) { script { - publishNotification( - icon: ':warning:', - message: 'Failed Build', - credentialsId: 'BUILD_NOTICE_WEBHOOK', - manifest: "${INPUT_MANIFEST}" - ) + if (env.PUBLISH_NOTIFICATION) { + publishNotification( + icon: ':warning:', + message: 'Failed Build', + credentialsId: 'BUILD_NOTICE_WEBHOOK', + manifest: "${INPUT_MANIFEST}" + ) + } postCleanup() } diff --git a/jenkins/opensearch-dashboards/integ-test.jenkinsfile b/jenkins/opensearch-dashboards/integ-test.jenkinsfile index 6b9bb440c3..18a8447815 100644 --- a/jenkins/opensearch-dashboards/integ-test.jenkinsfile +++ b/jenkins/opensearch-dashboards/integ-test.jenkinsfile @@ -7,9 +7,13 @@ pipeline { agent none environment { BUILD_MANIFEST = "build-manifest.yml" - BUILD_JOB_NAME = 'distribution-build-opensearch-dashboards' } parameters { + string( + name: 'INPUT_MANIFEST', + description: 'Input manifest under the manifests folder, e.g. 2.0.0/opensearch-dashboards-2.0.0.yml.', + trim: true + ) string( name: 'TEST_MANIFEST', description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-dashboards-2.0.0-test.yml.', @@ -25,11 +29,6 @@ pipeline { description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', trim: true ) - string( - name: 'CONTAINER_IMAGE', - description: 'The container image running on the agent where the tests should be executed, e.g. opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028.', - trim: true - ) } stages { stage('verify-parameters') { @@ -39,10 +38,21 @@ pipeline { currentBuild.result = 'ABORTED' error("Integration Tests failed to start. Missing parameter: AGENT_LABEL.") } - if (CONTAINER_IMAGE == '') { - currentBuild.result = 'ABORTED' - error("Integration Tests failed to start. Missing parameter: CONTAINER_IMAGE.") - } + } + } + } + stage('detect docker image + args') { + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028' + alwaysPull true + } + } + steps { + script { + currentBuild.description = "$INPUT_MANIFEST" + dockerAgent = detectDockerAgent() } } } @@ -50,7 +60,8 @@ pipeline { agent { docker { label AGENT_LABEL - image CONTAINER_IMAGE + image dockerAgent.image + args dockerAgent.args alwaysPull true } } @@ -66,7 +77,7 @@ pipeline { echo "BUILD_ID: ${BUILD_ID}" runIntegTestScript( - jobName: BUILD_JOB_NAME, + jobName: getBuildJobName(), buildManifest: BUILD_MANIFEST, testManifest: "manifests/${TEST_MANIFEST}", buildId: BUILD_ID diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index 602c2dc2fd..6c9bd78490 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -5,7 +5,7 @@ pipeline { environment { AGENT_X64 = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' AGENT_ARM64 = 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host' - INTEG_TEST_JOB_NAME = 'integ-test' + DEFAULT_INTEG_TEST_JOB_NAME = 'integ-test' } parameters { string( @@ -18,8 +18,34 @@ pipeline { description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0-test.yml.', trim: true ) + string( + name: 'INTEG_TEST_JOB_NAME', + description: "Name of integration test job that will be triggered, e.g. Playground/integ-test.", + defaultValue: "integ-test", + trim: true + ) + booleanParam( + name: 'BUILD_DOCKER', + description: 'Build docker image or not.', + defaultValue: true + ) + booleanParam( + name: 'PUBLISH_NOTIFICATION', + description: 'Publish the status of this build job or not.', + defaultValue: true + ) } stages { + stage('verify-parameters') { + steps { + script { + env.INTEG_TEST_JOB_NAME = INTEG_TEST_JOB_NAME != '' ? + INTEG_TEST_JOB_NAME : + DEFAULT_INTEG_TEST_JOB_NAME + env.PUBLISH_NOTIFICATION = PUBLISH_NOTIFICATION == 'true' + } + } + } stage('detect docker image + args') { agent { docker { @@ -123,23 +149,21 @@ pipeline { echo "buildManifestUrl (x64): ${buildManifestUrl}" echo "artifactUrl (x64): ${artifactUrl}" - + def integTestResults = - build job: INTEG_TEST_JOB_NAME, + build job: env.INTEG_TEST_JOB_NAME, propagate: false, wait: true, parameters: [ + string(name: 'INPUT_MANIFEST', value: INPUT_MANIFEST), string(name: 'TEST_MANIFEST', value: TEST_MANIFEST), string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl), string(name: 'AGENT_LABEL', value: AGENT_X64) ] - String status = integTestResults.getResult() - String icon = status == 'SUCCESS' ? ':white_check_mark:' : ':warning:' - lib.jenkins.Messages.new(this).add( - STAGE_NAME, - lib.jenkins.Messages.new(this).get([STAGE_NAME]) + - "\nInteg Tests (x64): ${icon} ${status} ${integTestResults.getAbsoluteUrl()}" + createTestResultsMessage( + testType: "Integ Tests (x64)", + testResults: integTestResults ) } } @@ -171,21 +195,19 @@ pipeline { echo "artifactUrl (arm64): ${artifactUrl}" def integTestResults = - build job: INTEG_TEST_JOB_NAME, + build job: env.INTEG_TEST_JOB_NAME, propagate: false, wait: true, parameters: [ + string(name: 'INPUT_MANIFEST', value: INPUT_MANIFEST), string(name: 'TEST_MANIFEST', value: TEST_MANIFEST), string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl), string(name: 'AGENT_LABEL', value: AGENT_ARM64) ] - String status = integTestResults.getResult() - String icon = status == 'SUCCESS' ? ':white_check_mark:' : ':warning:' - lib.jenkins.Messages.new(this).add( - STAGE_NAME, - lib.jenkins.Messages.new(this).get([STAGE_NAME]) + - "\nInteg Tests (arm64): ${icon} ${status} ${integTestResults.getAbsoluteUrl()}" + createTestResultsMessage( + testType: "Integ Tests (arm64)", + testResults: integTestResults ) } } @@ -198,6 +220,10 @@ pipeline { } } stage('docker build') { + when { + beforeAgent: true + equals expected: true, actual: BUILD_DOCKER + } steps { node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { script { @@ -219,15 +245,17 @@ pipeline { success { node(AGENT_X64) { script { - def stashed = lib.jenkins.Messages.new(this).get(['build-and-test-x64', 'build-and-test-arm64']) + if (env.PUBLISH_NOTIFICATION) { + def stashed = lib.jenkins.Messages.new(this).get(['build-and-test-x64', 'build-and-test-arm64']) - publishNotification( - icon: ':white_check_mark:', - message: 'Successful Build', - extra: stashed, - credentialsId: 'BUILD_NOTICE_WEBHOOK', - manifest: "${INPUT_MANIFEST}" - ) + publishNotification( + icon: ':white_check_mark:', + message: 'Successful Build', + extra: stashed, + credentialsId: 'BUILD_NOTICE_WEBHOOK', + manifest: "${INPUT_MANIFEST}" + ) + } postCleanup() } @@ -236,12 +264,14 @@ pipeline { failure { node(AGENT_X64) { script { - publishNotification( - icon: ':warning:', - message: 'Failed Build', - credentialsId: 'BUILD_NOTICE_WEBHOOK', - manifest: "${INPUT_MANIFEST}" - ) + if (env.PUBLISH_NOTIFICATION) { + publishNotification( + icon: ':warning:', + message: 'Failed Build', + credentialsId: 'BUILD_NOTICE_WEBHOOK', + manifest: "${INPUT_MANIFEST}" + ) + } postCleanup() } diff --git a/jenkins/opensearch/integ-test.jenkinsfile b/jenkins/opensearch/integ-test.jenkinsfile index 6902787b81..ae3ee9f6f0 100644 --- a/jenkins/opensearch/integ-test.jenkinsfile +++ b/jenkins/opensearch/integ-test.jenkinsfile @@ -10,6 +10,11 @@ pipeline { maven "maven-3.8.2" } parameters { + string( + name: 'INPUT_MANIFEST', + description: 'Input manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0.yml.', + trim: true + ) string( name: 'TEST_MANIFEST', description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0-test.yml.', @@ -37,10 +42,28 @@ pipeline { } } } + stage('detect docker image + args') { + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028' + alwaysPull true + } + } + steps { + script { + currentBuild.description = "$INPUT_MANIFEST" + dockerAgent = detectDockerAgent() + } + } + } stage('integ-test') { agent { - node { + docker { label AGENT_LABEL + image dockerAgent.image + args dockerAgent.args + alwaysPull true } } steps { @@ -55,6 +78,7 @@ pipeline { echo "BUILD_ID: ${BUILD_ID}" runIntegTestScript( + jobName: getBuildJobName(), buildManifest: BUILD_MANIFEST, testManifest: "manifests/${TEST_MANIFEST}", buildId: BUILD_ID diff --git a/vars/createTestResultsMessage.groovy b/vars/createTestResultsMessage.groovy new file mode 100644 index 0000000000..0a735d693d --- /dev/null +++ b/vars/createTestResultsMessage.groovy @@ -0,0 +1,14 @@ +def call(Map args = [:]) { + def lib = library(identifier: "jenkins@20211123", retriever: legacySCM(scm)) + + String testType = args.testType + def testResults = args.testResults + String status = testResults.getResult() + String icon = status == 'SUCCESS' ? ':white_check_mark:' : ':warning:' + + lib.jenkins.Messages.new(this).add( + "${STAGE_NAME}", + lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + + "\n${testType}: ${icon} ${status} ${testResults.getAbsoluteUrl()}" + ) +} \ No newline at end of file diff --git a/vars/getBuildJobName.groovy b/vars/getBuildJobName.groovy new file mode 100644 index 0000000000..b6b8c5eb8c --- /dev/null +++ b/vars/getBuildJobName.groovy @@ -0,0 +1,5 @@ +def call(Map args = [:]) { + return currentBuild.upstreamBuilds ? + currentBuild.upstreamBuilds[0].fullProjectName : + 'distribution-build-opensearch' +} \ No newline at end of file