Merge remote-tracking branch 'upstream/master' into feature/download-…

…dependencies-goals

* upstream/master: (46 commits)
  Use badger code from upstream repository (elastic#21705)
  Disable writes sync in persistent cache (elastic#21754)
  Make API address and Shard ID required in Cloud Foundry settings (elastic#21759)
  [CI] Support skip-ci label (elastic#21377)
  Increase recommended memory when deploying in Cloud foundry (elastic#21755)
  typofix for dns timeout configuration option (elastic#21069)
  chore: create CI artifacts for DEV usage (elastic#21645)
  [Ingest Manager] Atomic installed forgotten changelog elastic#21780
  [Ingest Manager] Agent atomic installer (elastic#21745)
  Add missing configuration annotations (elastic#21736)
  [Filebeat] Add check for context.DeadlineExceeded error (elastic#21732)
  Remove kafka partition ISR from Metricbeat docs (elastic#21709)
  Skip flaky test with oauth2 config in httpjson input (elastic#21749)
  Fix for azure retrieve resource by ids (elastic#21711)
  Update obs app links (elastic#21682)
  fix: update fleet test suite name (elastic#21738)
  Remove dot from file.extension value in Auditbeat FIM (elastic#21644)
  Fix leaks with metadata processors (elastic#16349)
  Add istiod metricset (elastic#21519)
  [Ingest Manager] Change Sync/Close call order (elastic#21735)
  ...

.backportrc.json

@@ -1,6 +1,6 @@
 {
   "upstream": "elastic/beats",
-  "branches": [{ "name": "7.9"}, { "name": "7.8"}, { "name": "7.7"}, { "name": "7.x"}],
+  "branches": [ { "name": "7.x", "checked": true }, "7.10", "7.9", "7.8", "7.7"],
   "labels": ["backport"],
   "autoAssign": true,
   "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"

.ci/jobs/apm-beats-update.yml

@@ -18,7 +18,7 @@
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
         discover-tags: true
-        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         disable-pr-notifications: true
         notification-context: 'apm-beats-update'
         repo: 'beats'
@@ -38,6 +38,9 @@
             - regex-name:
                 regex: '7\.[x789]'
                 case-sensitive: true
+            - regex-name:
+                regex: '7\.1\d'
+                case-sensitive: true
             - regex-name:
                 regex: '8\.\d+'
                 case-sensitive: true

.ci/jobs/beats.yml

@@ -17,7 +17,7 @@
         discover-pr-forks-strategy: 'merge-current'
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
-        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         discover-tags: true
         notification-context: "beats-ci"
         repo: 'beats'
@@ -38,6 +38,9 @@
             - regex-name:
                 regex: '7\.[x789]'
                 case-sensitive: true
+            - regex-name:
+                regex: '7\.1\d'
+                case-sensitive: true
             - regex-name:
                 regex: '8\.\d+'
                 case-sensitive: true

.ci/jobs/packaging.yml

@@ -14,7 +14,7 @@
           discover-pr-forks-trust: 'permission'
           discover-pr-origin: 'merge-current'
           discover-tags: true
-          head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+          head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
           disable-pr-notifications: true
           notification-context: 'beats-packaging'
           repo: 'beats'
@@ -34,6 +34,9 @@
               - regex-name:
                   regex: '7\.[x789]'
                   case-sensitive: true
+              - regex-name:
+                  regex: '7\.1\d'
+                  case-sensitive: true
               - regex-name:
                   regex: '8\.\d+'
                   case-sensitive: true

.ci/packaging.groovy

@@ -198,17 +198,30 @@ def tagAndPush(name){
     tagName = "pr-${env.CHANGE_ID}"
   }
 
-  def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}"
-  def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${tagName}"
-  def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}"
   dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
-  retry(3){
-    sh(label:'Change tag and push', script: """
-      docker tag ${oldName} ${newName}
-      docker push ${newName}
-      docker tag ${oldName} ${commitName}
-      docker push ${commitName}
-    """)
+
+  // supported image flavours
+  def variants = ["", "-oss", "-ubi8"]
+  variants.each { variant ->
+    def oldName = "${DOCKER_REGISTRY}/beats/${name}${variant}:${libbetaVer}"
+    def newName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${tagName}"
+    def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${env.GIT_BASE_COMMIT}"
+
+    def iterations = 0
+    retryWithSleep(retries: 3, seconds: 5, backoff: true)
+      iterations++
+      def status = sh(label:'Change tag and push', script: """
+        docker tag ${oldName} ${newName}
+        docker push ${newName}
+        docker tag ${oldName} ${commitName}
+        docker push ${commitName}
+      """, returnStatus: true)
+
+    if ( status > 0 && iterations < 3) {
+      error('tag and push failed, retry')
+    } else if ( status > 0 ) {
+      log(level: 'WARN', text: "${name} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
+    }
   }
 }
 
@@ -217,11 +230,11 @@ def runE2ETestForPackages(){
 
   catchError(buildResult: 'UNSTABLE', message: 'Unable to run e2e tests', stageResult: 'FAILURE') {
     if ("${env.BEATS_FOLDER}" == "filebeat" || "${env.BEATS_FOLDER}" == "x-pack/filebeat") {
-      suite = 'helm,ingest-manager'
+      suite = 'helm,fleet'
     } else if ("${env.BEATS_FOLDER}" == "metricbeat" || "${env.BEATS_FOLDER}" == "x-pack/metricbeat") {
       suite = ''
     } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
-      suite = 'ingest-manager'
+      suite = 'fleet'
     } else {
       echo("Skipping E2E tests for ${env.BEATS_FOLDER}.")
       return
@@ -233,8 +246,12 @@ def runE2ETestForPackages(){
 
 def release(){
   withBeatsEnv(){
-    dir("${env.BEATS_FOLDER}") {
-      sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+    withEnv([
+      "DEV=true"
+    ]) {
+      dir("${env.BEATS_FOLDER}") {
+        sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+      }
     }
     publishPackages("${env.BEATS_FOLDER}")
   }
@@ -245,19 +262,25 @@ def triggerE2ETests(String suite) {
 
   def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
   def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
+
+  def parameters = [
+    booleanParam(name: 'forceSkipGitChecks', value: true),
+    booleanParam(name: 'forceSkipPresubmit', value: true),
+    booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+    string(name: 'runTestsSuites', value: suite),
+    string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
+    string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
+    string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
+  ]
+  if (isPR()) {
+    def version = "pr-${env.CHANGE_ID}"
+    parameters.push(booleanParam(name: 'USE_CI_SNAPSHOTS', value: true))
+    parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}"))
+    parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}"))
+  }
+
   build(job: "${e2eTestsPipeline}",
-    parameters: [
-      booleanParam(name: 'forceSkipGitChecks', value: true),
-      booleanParam(name: 'forceSkipPresubmit', value: true),
-      booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
-      booleanParam(name: 'USE_CI_SNAPSHOTS', value: true),
-      string(name: 'ELASTIC_AGENT_VERSION', value: "pr-${env.CHANGE_ID}"),
-      string(name: 'METRICBEAT_VERSION', value: "pr-${env.CHANGE_ID}"),
-      string(name: 'runTestsSuites', value: suite),
-      string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
-      string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
-      string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
-    ],
+    parameters: parameters,
     propagate: false,
     wait: false
   )