Jenkinsfile change for Branching

Signed-off-by: prudhvigodithi <pgodithi@amazon.com>
opensearch-project · Sep 8, 2022 · 2345581 · 2345581
1 parent e1c5883
commit 2345581
Show file tree

Hide file tree

Showing 26 changed files with 580 additions and 56 deletions.
diff --git a/.github/ISSUE_TEMPLATE/release_template.md b/.github/ISSUE_TEMPLATE/release_template.md
@@ -40,6 +40,7 @@ To aid in understanding the state of the release there is a table with status in
 - [ ] [Create a release issue in every component repo](https://github.com/opensearch-project/opensearch-build/blob/main/meta/README.md#create-a-release-issue) that links back to this issue, update Components section with these links.
 - [ ] Ensure the label is created in each component repo for this new version, and the next minor release. [Create a version label](https://github.com/opensearch-project/opensearch-plugins/blob/main/META.md#create-or-update-labels-in-all-plugin-repos)
 - [ ] Ensure that all release issues created above are assigned to an owner in the component team.
+- [ ] Increase the build frequency for the this release from once a day (H 1 * * *) to once every hour (H/60 * * * *) in [jenkinsFile](https://github.com/opensearch-project/opensearch-build/blob/main/jenkins/check-for-build.jenkinsfile)
 
 ### CI/CD (Feature Freeze) - _Ends __REPLACE_RELEASE-minus-14-days__
 

diff --git a/build.gradle b/build.gradle
@@ -31,7 +31,7 @@ dependencies {
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
     implementation group: 'org.codehaus.groovy', name: 'groovy-all', version: '3.0.9', ext: 'pom'
     implementation group: 'com.cloudbees', name: 'groovy-cps', version: '1.31'
-    testImplementation group: 'org.yaml', name: 'snakeyaml', version: '1.29'
+    testImplementation group: 'org.yaml', name: 'snakeyaml', version: '1.31'
     testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.4.1'
     testImplementation group: 'com.lesfurets', name:'jenkins-pipeline-unit', version: '1.13'
 }
@@ -56,20 +56,10 @@ sourceSets {
             srcDirs = ['tests/jenkins']
         }
     }
-
-    jobs {
-        groovy {
-            srcDirs 'src/jenkins/jobs'
-            compileClasspath += main.compileClasspath
-        }
-
-        compileClasspath += sourceSets.main.output
-        runtimeClasspath += sourceSets.main.output
-    }
 }
 
 sharedLibrary {
-    coreVersion = '2.358' // https://repo.jenkins-ci.org/public/org/jenkins-ci/main/jenkins-core/
+    coreVersion = '2.364' // https://repo.jenkins-ci.org/public/org/jenkins-ci/main/jenkins-core/
     testHarnessVersion = '1736.vc72c458c5103' // https://mvnrepository.com/artifact/org.jenkins-ci.main/jenkins-test-harness?repo=jenkins-releases
     pluginDependencies {
         workflowCpsGlobalLibraryPluginVersion = '570.v21311f4951f8' // https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/workflow/workflow-cps-global-lib/
@@ -79,7 +69,7 @@ sharedLibrary {
         dependency('org.jenkins-ci.plugins', 'pipeline-input-step', '449.v77f0e8b_845c4') // https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/pipeline-input-step/
         dependency('org.jenkins-ci.plugins', 'script-security', '1172.v35f6a_0b_8207e')
         dependency('org.jenkins-ci.plugins', 'credentials', '1112.vc87b_7a_3597f6')
-        dependency('org.jenkins-ci.plugins', 'git-client', '3.10.1')
+        dependency('org.jenkins-ci.plugins', 'git-client', '3.11.1')
         dependency('org.jenkins-ci.plugins', 'junit', '1.55')
         dependency('org.jenkins-ci.plugins', 'mailer', '408.vd726a_1130320') // https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/mailer/
     }
@@ -127,3 +117,7 @@ jacocoTestReport {
         xml.required = true
     }
 }
+
+tasks.test {
+    maxParallelForks = Runtime.getRuntime().availableProcessors()
+}
diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile
@@ -11,10 +11,10 @@ pipeline {
     }
     triggers {
         parameterizedCron '''
-            H 1 * * * %INPUT_MANIFEST=2.3.0/opensearch-dashboards-2.3.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards
-            H 1 * * * %INPUT_MANIFEST=2.3.0/opensearch-2.3.0.yml;TARGET_JOB_NAME=distribution-build-opensearch
-            H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch
-            H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards
+            H/60 * * * * %INPUT_MANIFEST=2.3.0/opensearch-dashboards-2.3.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_BRANCH=main
+            H/60 * * * * %INPUT_MANIFEST=2.3.0/opensearch-2.3.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_BRANCH=main
+            H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_BRANCH=main
+            H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_BRANCH=main
         '''
     }
     parameters {
@@ -28,6 +28,12 @@ pipeline {
             description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0-test.yml. Currently only applicable for distribution-build-opensearch job',
             trim: true
         )
+        string(
+            name: 'BUILD_BRANCH',
+            description: "Build repo branch to use.",
+            defaultValue: "main",
+            trim: true
+        )
         string(
             name: 'TARGET_JOB_NAME',
             description: 'Job to trigger if build has changed',
@@ -74,7 +80,8 @@ pipeline {
                             }
                             build job: "${TARGET_JOB_NAME}", parameters: [
                             string(name: 'INPUT_MANIFEST', value: "${INPUT_MANIFEST}"),
-                            string(name: 'TEST_MANIFEST', value: "${TEST_MANIFEST}")
+                            string(name: 'TEST_MANIFEST', value: "${TEST_MANIFEST}"),
+                            string(name: 'BUILD_BRANCH', value: "${BUILD_BRANCH}")
                             ], wait: true
 
                             echo "Build succeeded, uploading build SHA for that job"

diff --git a/jenkins/opensearch-dashboards/bwc-test.jenkinsfile b/jenkins/opensearch-dashboards/bwc-test.jenkinsfile
@@ -3,11 +3,13 @@ lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm))
 pipeline {
     options {
         timeout(time: 3, unit: 'HOURS')
+        skipDefaultCheckout()
     }
     agent none
     environment {
         BUILD_MANIFEST = "build-manifest.yml"
         DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch-dashboards"
+        BUILD_REPO = 'https://github.com/opensearch-project/opensearch-build.git'
     }
     parameters {
         string(
@@ -20,20 +22,40 @@ pipeline {
             description: 'The build manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/2.0.0/98/linux/x64/builds/opensearch-dashboards/manifest.yml.',
             trim: true
         )
+        string(
+            name: 'BUILD_BRANCH',
+            description: "Build repo branch to use.",
+            defaultValue: "main",
+            trim: true
+        )
         string(
             name: 'AGENT_LABEL',
             description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host.',
             trim: true
         )
     }
     stages {
+        stage('checkout the build repo') {
+            agent {
+                docker {
+                    label AGENT_LABEL
+                    image 'alpine:3'
+                    alwaysPull true
+                }
+            }
+            steps {
+                git url: BUILD_REPO, branch: BUILD_BRANCH
+                stash(name: 'sourceRepo')
+            }
+        }
         stage('verify-parameters') {
             agent {
                 node {
                     label AGENT_LABEL
                 }
             }
             steps {
+                unstash(name: 'sourceRepo')
                 script {
                     currentBuild.description = TEST_MANIFEST
                     if (AGENT_LABEL == '') {
@@ -63,6 +85,7 @@ pipeline {
                 }
             }
             steps {
+                unstash(name: 'sourceRepo')
                 script { 
                     DOCKER_AGENT = detectTestDockerAgent()
                 }
@@ -78,6 +101,7 @@ pipeline {
                 }
             }
             steps {
+                unstash(name: 'sourceRepo')
                 script {
                     def buildManifestObj = downloadBuildManifest(
                         url: BUILD_MANIFEST_URL,
@@ -99,6 +123,7 @@ pipeline {
             post {
                 always {
                     script {
+                        unstash(name: 'sourceRepo')
                         uploadTestResults(
                             buildManifestFileName: BUILD_MANIFEST,
                             jobName: JOB_NAME
@@ -113,6 +138,7 @@ pipeline {
     post {
         success {
             node(AGENT_LABEL) {
+                unstash(name: 'sourceRepo')
                 script {
                     def stashed = lib.jenkins.Messages.new(this).get(['bwc-test'])
                     publishNotification(
@@ -122,13 +148,13 @@ pipeline {
                         credentialsId: 'jenkins-integ-test-webhook',
                         manifest: TEST_MANIFEST,
                     )
-
                     postCleanup()
                 }
             }
         }
         failure {
             node(AGENT_LABEL) {
+                unstash 'sourceRepo'
                 script  {
                     def stashed = lib.jenkins.Messages.new(this).get(['bwc-test'])
                     publishNotification(
@@ -138,7 +164,6 @@ pipeline {
                         credentialsId: 'jenkins-integ-test-webhook',
                         manifest: TEST_MANIFEST,
                     )
-
                     postCleanup()
                 }
             }