ci: packaging pipeline for the snapshots

.ci/Jenkinsfile

@@ -58,6 +58,7 @@ pipeline {
         dir("${BASE_DIR}"){
           setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true).toString())
           setEnvVar('PACKAGING_CHANGES', isGitRegionMatch(patterns: [ '(^dev-tools/packaging/.*|.ci/Jenkinsfile)' ], shouldMatchAll: false).toString())
+          setEnvVar('BEAT_VERSION', sh(label: 'Get elastic agent version', script: 'make get-version', returnStdout: true)?.trim())
           setEnvVar('K8S_CHANGES', isGitRegionMatch(patterns: [ '(^deploy/kubernetes/.*|^version/docs/version.asciidoc|.ci/Jenkinsfile)' ], shouldMatchAll: false).toString())
           setEnvVar('EXT_WINDOWS_CHANGES', isGitRegionMatch(patterns: [ '.ci/Jenkinsfile' ], shouldMatchAll: false).toString())
           setEnvVar('EXT_M1_CHANGES', isGitRegionMatch(patterns: [ '.ci/Jenkinsfile' ], shouldMatchAll: false).toString())
@@ -71,8 +72,6 @@ pipeline {
         withGithubNotify(context: "Check") {
           withMageEnv(){
            dir("${BASE_DIR}"){
-              setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
-              log(level: 'INFO', text: "env.BEAT_VERSION=${env.BEAT_VERSION}")
               cmd(label: 'check', script: 'make check-ci')
             }
           }
@@ -400,7 +399,7 @@ pipeline {
     }
     stage('e2e tests') {
       when {
-        // Always when running builds on branches/tags
+        // Always when running builds on branches/tags, if so, then it runs sync to support the DRA
         // Enable if e2e related changes.
         beforeAgent true
         anyOf {
@@ -415,9 +414,7 @@ pipeline {
                elasticAgentVersion: "${env.BEAT_VERSION}-SNAPSHOT",
                gitHubCheckName: "e2e-tests",
                gitHubCheckRepo: env.REPO,
-               gitHubCheckSha1: env.GIT_BASE_COMMIT,
-               propagate: true,
-               wait: true)
+               gitHubCheckSha1: env.GIT_BASE_COMMIT)
       }
     }
   }

.ci/packaging.groovy

@@ -0,0 +1,124 @@
+#!/usr/bin/env groovy
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    REPO = 'elastic-agent'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
+    SLACK_CHANNEL = '#elastic-agent'
+    NOTIFY_TO = 'package+beats-contrib@elastic.co'
+    JOB_GCS_BUCKET = credentials('gcs-bucket')
+    JOB_GCS_CREDENTIALS = 'fleet-ci-gcs-plugin'
+    DOCKER_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    DOCKER_REGISTRY = 'docker.elastic.co'
+  }
+  options {
+    timeout(time: 2, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '100', artifactNumToKeepStr: '30', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
+    quietPeriod(10)
+  }
+  triggers {
+    // disable upstream trigger on a PR basis
+    upstream("elastic-agent/elastic-agent-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
+  }
+  stages {
+    stage('Filter build') {
+      agent { label 'ubuntu-20 && immutable' }
+      when {
+        beforeAgent true
+        anyOf {
+          triggeredBy cause: "IssueCommentCause"
+          expression {
+            def ret = isUserTrigger() || isUpstreamTrigger()
+            if(!ret){
+              currentBuild.result = 'NOT_BUILT'
+              currentBuild.description = "The build has been skipped"
+              currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
+              echo("the build has been skipped due the trigger is a branch scan and the allowed ones are manual, GitHub comment, and upstream job")
+            }
+            return ret
+          }
+        }
+      }
+      environment {
+        PATH = "${env.PATH}:${env.WORKSPACE}/bin"
+        HOME = "${env.WORKSPACE}"
+      }
+      stages {
+        stage('Checkout') {
+          options { skipDefaultCheckout() }
+          steps {
+            pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ])
+            deleteDir()
+            gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: false,
+                        shallow: false, reference: "/var/lib/jenkins/.git-references/${REPO}.git")
+            stash allowEmpty: true, name: 'source', useDefaultExcludes: false
+            // set environment variables globally since they are used afterwards but GIT_BASE_COMMIT won't
+            // be available until gitCheckout is executed.
+            setEnvVar('URI_SUFFIX', "commits/${env.REPO}/${env.GIT_BASE_COMMIT}")
+            // JOB_GCS_BUCKET contains the bucket and some folders, let's build the folder structure
+            setEnvVar('PATH_PREFIX', "${JOB_GCS_BUCKET.contains('/') ? JOB_GCS_BUCKET.substring(JOB_GCS_BUCKET.indexOf('/') + 1) + '/' + env.URI_SUFFIX : env.URI_SUFFIX}")
+            //TODO : uncomment
+            //setEnvVar('IS_BRANCH_AVAILABLE', isBranchUnifiedReleaseAvailable(env.BRANCH_NAME))
+            setEnvVar('IS_BRANCH_AVAILABLE', isBranchUnifiedReleaseAvailable('main'))
+          }
+        }
+        stage('Package') {
+          options { skipDefaultCheckout() }
+          steps {
+            echo 'Done as part of the main pipeline'
+          }
+        }
+        stage('DRA') {
+          // The Unified Release process keeps moving branches as soon as a new
+          // minor version is created, therefore old release branches won't be able
+          // to use the release manager as their definition is removed.
+          when {
+            expression { return env.IS_BRANCH_AVAILABLE == "true" }
+          }
+          steps {
+            googleStorageDownload(bucketUri: "gs://${JOB_GCS_BUCKET}/${URI_SUFFIX}/*",
+                                  credentialsId: "${JOB_GCS_CREDENTIALS}",
+                                  localDirectory: "${BASE_DIR}/build/distributions",
+                                  pathPrefix: env.PATH_PREFIX)
+            dir("${BASE_DIR}") {
+              withMageEnv() {
+                sh(label: 'create dependencies file', script: 'make release-manager-dependencies-snapshot')
+              }
+              dockerLogin(secret: env.DOCKER_SECRET, registry: env.DOCKER_REGISTRY)
+              script {
+                getVaultSecret.readSecretWrapper {
+                  sh(label: 'release-manager.sh', script: '.ci/scripts/release-manager.sh')
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult(prComment: false)
+    }
+    failure {
+      echo 'disabled'
+    // notifyStatus(slackStatus: 'danger', subject: "[${env.REPO}@${env.BRANCH_NAME}] DRA failed", body: "Build: (<${env.RUN_DISPLAY_URL}|here>)")
+    }
+  }
+}
+
+def notifyStatus(def args = [:]) {
+  releaseNotification(slackChannel: "${env.SLACK_CHANNEL}",
+                      slackColor: args.slackStatus,
+                      slackCredentialsId: 'jenkins-slack-integration-token',
+                      to: "${env.NOTIFY_TO}",
+                      subject: args.subject,
+                      body: args.body)
+}

.ci/scripts/release-manager.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# This script is executed by the daily release artifact stage.
+# It requires the below environment variables:
+# - BRANCH_NAME
+# - VAULT_ADDR
+# - VAULT_ROLE_ID
+# - VAULT_SECRET_ID
+#
+set -uexo pipefail
+
+source /usr/local/bin/bash_standard_lib.sh
+
+# set required permissions on artifacts and directory
+chmod -R a+r build/distributions/*
+chmod -R a+w build/distributions
+
+# get the current version (without the snapshot)
+VERSION=$(make get-version)
+
+# ensure the latest image has been pulled
+IMAGE=docker.elastic.co/infra/release-manager:latest
+(retry 3 docker pull --quiet "${IMAGE}") || echo "Error pulling ${IMAGE} Docker image, we continue"
+docker images --filter=reference=$IMAGE
+
+# Generate checksum files and upload to GCS
+docker run --rm \
+  --name release-manager \
+  -e VAULT_ADDR \
+  -e VAULT_ROLE_ID \
+  -e VAULT_SECRET_ID \
+  --mount type=bind,readonly=false,src="$PWD",target=/artifacts \
+  "$IMAGE" \
+    cli collect \
+      --project elastic-agent \
+      --branch "main" \
+      --commit "$(git rev-parse HEAD)" \
+      --workflow "snapshot" \
+      --artifact-set main \
+      --version "${VERSION}"

Makefile

@@ -6,6 +6,10 @@ PYTHON_ENV?=$(BUILD_DIR)/python-env
 MAGE_VERSION     ?= v1.13.0
 MAGE_PRESENT     := $(shell mage --version 2> /dev/null | grep $(MAGE_VERSION))
 MAGE_IMPORT_PATH ?= github.com/magefile/mage
+NOTICE_TEMPLATE  ?= NOTICE.txt.tmpl
+NOTICE_FILE      ?= NOTICE.txt
+ELASTIC_AGENT_VERSION=$(shell grep defaultBeatVersion version/version.go | cut -d'=' -f2 | tr -d '" ')
+
 export MAGE_IMPORT_PATH
 
 ## mage : Sets mage
@@ -36,8 +40,8 @@ notice:
 		-includeIndirect \
 		-rules dev-tools/notice/rules.json \
 		-overrides dev-tools/notice/overrides.json \
-		-noticeTemplate dev-tools/notice/NOTICE.txt.tmpl \
-		-noticeOut NOTICE.txt \
+		-noticeTemplate dev-tools/notice/$(NOTICE_TEMPLATE) \
+		-noticeOut $(NOTICE_FILE) \
 		-depsOut ""
 	cat dev-tools/notice/NOTICE.txt.append >> NOTICE.txt
 
@@ -69,7 +73,22 @@ check-no-changes:
 	@git update-index --refresh
 	@git diff-index --exit-code HEAD --
 
-## get-version : Get the libbeat version
+## get-version : Get the Elastic Agent Version
 .PHONY: get-version
 get-version:
-	@mage dumpVariables | grep 'beat_version' | cut -d"=" -f 2 | tr -d " "
+	@echo $(ELASTIC_AGENT_VERSION)
+
+## release-manager-dependencies : Prepares the dependencies file
+.PHONY: release-manager-dependencies
+release-manager-dependencies:
+	@mkdir -p build/distributions
+	@$(MAKE) NOTICE_TEMPLATE=dependencies.csv.tmpl NOTICE_FILE=build/distributions/dependencies.csv notice
+	@cd build/distributions && shasum -a 512 dependencies.csv > dependencies.csv.sha512
+
+.PHONY: release-manager-dependencies-snapshot
+release-manager-dependencies-snapshot: ## - Prepares the dependencies file for a snapshot.
+	@$(MAKE) SNAPSHOT=true release-manager-dependencies
+
+.PHONY: release-manager-dependencies-release
+release-manager-dependencies-release: ## - Prepares the dependencies file for a release.
+	@$(MAKE) release-manager-dependencies