upload index.json from artifacts upload (#1622)

* upload index.json from artifacts upload Signed-off-by: Tianle Huang <tianleh@amazon.com>
opensearch-project · Feb 17, 2022 · 1b18900 · 1b18900
1 parent e0a34eb
commit 1b18900
Show file tree

Hide file tree

Showing 10 changed files with 43 additions and 0 deletions.
diff --git a/src/jenkins/BuildManifest.groovy b/src/jenkins/BuildManifest.groovy
@@ -90,6 +90,13 @@ class BuildManifest implements Serializable {
                 this.build.architecture
         ].join("/")
     }
+
+    public String getIndexFileRoot(String jobName) {
+        return [
+                jobName,
+                this.build.version
+        ].join("/")
+    }
 
     public String getArtifactRootUrl(String publicArtifactUrl = 'https://ci.opensearch.org/ci/dbc', String jobName, String buildNumber) {
         return [

diff --git a/tests/jenkins/TestArchiveAssembleUpload.groovy b/tests/jenkins/TestArchiveAssembleUpload.groovy
@@ -30,6 +30,7 @@ class TestArchiveAssembleUpload extends BuildPipelineTest {
         binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role')
 
         helper.registerAllowedMethod("s3Upload", [Map])
+        helper.registerAllowedMethod("writeJSON", [Map])
         helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure ->
             closure.delegate = delegate
             return helper.callClosure(closure)

diff --git a/tests/jenkins/TestAssembleUpload.groovy b/tests/jenkins/TestAssembleUpload.groovy
@@ -29,6 +29,7 @@ class TestAssembleUpload extends BuildPipelineTest {
         binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role')
 
         helper.registerAllowedMethod("s3Upload", [Map])
+        helper.registerAllowedMethod("writeJSON", [Map])
         helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure ->
             closure.delegate = delegate
             return helper.callClosure(closure)

diff --git a/tests/jenkins/TestBuildAssembleUpload.groovy b/tests/jenkins/TestBuildAssembleUpload.groovy
@@ -35,6 +35,7 @@ class TestBuildAssembleUpload extends BuildPipelineTest {
         })
 
         helper.registerAllowedMethod("s3Upload", [Map])
+        helper.registerAllowedMethod("writeJSON", [Map])
         helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure ->
             closure.delegate = delegate
             return helper.callClosure(closure)

diff --git a/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt
@@ -38,6 +38,12 @@
                         uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
                            uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                               uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
+                        BuildManifest.getIndexFileRoot(vars-build)
+                        uploadArtifacts.writeJSON({file=index.json, json={latest=33}})
+                        uploadArtifacts.echo(Uploading index.json to s3://bucket/vars-build/1.1.0)
+                        uploadArtifacts.uploadToS3({sourcePath=index.json, bucket=artifact-bucket, path=vars-build/1.1.0/index.json})
+                           uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                              uploadToS3.s3Upload({file=index.json, bucket=artifact-bucket, path=vars-build/1.1.0/index.json})
                         uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64)
                         uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                            uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/})

diff --git a/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt
@@ -27,6 +27,12 @@
                      uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
                         uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                            uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
+                     BuildManifest.getIndexFileRoot(vars-build)
+                     uploadArtifacts.writeJSON({file=index.json, json={latest=33}})
+                     uploadArtifacts.echo(Uploading index.json to s3://bucket/vars-build/1.1.0)
+                     uploadArtifacts.uploadToS3({sourcePath=index.json, bucket=artifact-bucket, path=vars-build/1.1.0/index.json})
+                        uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                           uploadToS3.s3Upload({file=index.json, bucket=artifact-bucket, path=vars-build/1.1.0/index.json})
                      uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64)
                      uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                         uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/})

diff --git a/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt
@@ -36,6 +36,12 @@
                         uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
                            uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                               uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
+                        BuildManifest.getIndexFileRoot(vars-build)
+                        uploadArtifacts.writeJSON({file=index.json, json={latest=33}})
+                        uploadArtifacts.echo(Uploading index.json to s3://bucket/vars-build/1.1.0)
+                        uploadArtifacts.uploadToS3({sourcePath=index.json, bucket=artifact-bucket, path=vars-build/1.1.0/index.json})
+                           uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                              uploadToS3.s3Upload({file=index.json, bucket=artifact-bucket, path=vars-build/1.1.0/index.json})
                         uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64)
                         uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                            uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/})

diff --git a/tests/jenkins/jobs/BuildManifest_Jenkinsfile b/tests/jenkins/jobs/BuildManifest_Jenkinsfile
@@ -27,6 +27,7 @@ pipeline {
                     echo buildManifest.getArtifactUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1')
                     echo buildManifest.build.getPackageName()
                     echo buildManifest.components.getMinArtifact()
+                    echo buildManifest.getIndexFileRoot("distribution-build-opensearch")
                 }
             }
         }

diff --git a/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt b/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt
@@ -23,3 +23,5 @@
                BuildManifest_Jenkinsfile.echo(opensearch-1.1.0-linux-x64.tar.gz)
                BuildManifest.getMinArtifact()
                BuildManifest_Jenkinsfile.echo(dist/opensearch-min-1.1.0-linux-x64.tar.gz)
+               BuildManifest.getIndexFileRoot(distribution-build-opensearch)
+               BuildManifest_Jenkinsfile.echo(distribution-build-opensearch/1.1.0)
diff --git a/vars/uploadArtifacts.groovy b/vars/uploadArtifacts.groovy
@@ -21,6 +21,18 @@ void call(Map args = [:]) {
             path: "${artifactPath}/dist"
     )
 
+    def indexFilePath = buildManifest.getIndexFileRoot("${JOB_NAME}")
+    def latestBuildData = ['latest': "${BUILD_NUMBER}"]
+    writeJSON file: 'index.json', json: latestBuildData
+
+    echo "Uploading index.json to s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${indexFilePath}"
+
+    uploadToS3(
+            sourcePath: 'index.json',
+            bucket: "${ARTIFACT_BUCKET_NAME}",
+            path: "${indexFilePath}/index.json"
+    )
+
     echo "Uploading to s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${artifactPath}"
 
     withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {