Merge remote-tracking branch 'upstream/master' into feature/windows-7-64

* upstream/master: (364 commits)
  Add vendoring to Google Cloud Functions again (elastic#21070)
  [Elastic Agent] Add fleet.host.id for sending to endpoint. (elastic#21042)
  Do not need Google credentials before using it (elastic#21072)
  [Filebeat][New Module] Zoom webhook module (elastic#20414)
  Add support for GMT timezone offset in decode_cef (elastic#20993)
  Filebeat: Fix random error on harvester close (elastic#21048)
  Add ingress controller dashboards (elastic#21052)
  Fix loggers in composable module. (elastic#21047)
  [Ingest Manager] Increase kibana client timeout to 5 minutes (elastic#21037)
  Add changelog. (elastic#21041)
  [Elastic Agent] Add support for EQL based conditions (elastic#20994)
  Disable Kafka metricsets based on Jolokia (elastic#20989)
  Update apm agent (elastic#21031)
  Add container ECS fields in kubernetes metadata (elastic#20984)
  Sanitize event.host in Metricbeat (elastic#21022)
  Update api-keys.asciidoc - API key prerequisites (elastic#21026)
  [Filebeat][suricata] Map x509 for suricata/eve fileset (elastic#20973)
  [Filebeat][santa] Map x509 fields in santa module (elastic#20976)
  [Filebeat][fortinet] Map x509 ecs fields for fortinet fw fileset (elastic#20983)
  Bump zeek kerberos/ssl/x509 ecs version (elastic#21003)
  ...

.backportrc.json

@@ -1,6 +1,6 @@
 {
   "upstream": "elastic/beats",
-  "branches": [{ "name": "7.8"}, { "name": "7.7"}, { "name": "7.x"}],
+  "branches": [{ "name": "7.9"}, { "name": "7.8"}, { "name": "7.7"}, { "name": "7.x"}],
   "labels": ["backport"],
   "autoAssign": true,
   "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"

.ci/apm-beats-update.groovy

@@ -2,7 +2,7 @@
 @Library('apm@current') _
 
 pipeline {
-  agent none
+  agent { label 'master' }
   environment {
     REPO = 'apm-server'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
@@ -27,69 +27,75 @@ pipeline {
   }
   triggers {
     issueCommentTrigger('(?i).*/run\\s+(?:apm-beats-update\\W+)?.*')
-    upstream("Beats/beats-beats-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
+    upstream("Beats/beats/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
   }
   stages {
     stage('Filter build') {
-      agent { label 'ubuntu && immutable' }
+      agent { label 'ubuntu-18 && immutable' }
       when {
         beforeAgent true
-        expression {
-          def ret = isCommentTrigger() || isUserTrigger()
-          if(!ret){
-            currentBuild.result = 'NOT_BUILT'
-            currentBuild.description = "The build has been skipped"
-            currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
-            echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, and GitHub comment")
+        anyOf {
+          triggeredBy cause: "IssueCommentCause"
+          expression {
+            def ret = isUserTrigger() || isUpstreamTrigger()
+            if(!ret){
+              currentBuild.result = 'NOT_BUILT'
+              currentBuild.description = "The build has been skipped"
+              currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
+              echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
+            }
+            return ret
           }
-          return ret
         }
       }
-      /**
-       Checkout the code and stash it, to use it on other stages.
-      */
-      stage('Checkout') {
-        steps {
-          deleteDir()
-          gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)
-          script {
-            dir("${BEATS_DIR}"){
-              env.GO_VERSION = readFile(".go-version").trim()
-              def regexps =[
-                "^devtools/mage.*",
-                "^libbeat/scripts/Makefile",
-              ]
-              env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps)
-              // Skip all the stages except docs for PR's with asciidoc changes only
-              env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true)
+      stages {
+        /**
+        Checkout the code and stash it, to use it on other stages.
+        */
+        stage('Checkout') {
+          options { skipDefaultCheckout() }
+          steps {
+            deleteDir()
+            gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)
+            script {
+              dir("${BEATS_DIR}"){
+                env.GO_VERSION = readFile(".go-version").trim()
+                def regexps =[
+                  "^devtools/mage.*",
+                  "^libbeat/scripts/Makefile",
+                ]
+                env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps)
+                // Skip all the stages except docs for PR's with asciidoc changes only
+                env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true)
+              }
             }
           }
         }
-      }
-      /**
-      updates beats updates the framework part and go parts of beats.
-      Then build and test.
-      Finally archive the results.
-      */
-      stage('Update Beats') {
-        options { skipDefaultCheckout() }
-        when {
-          beforeAgent true
-          anyOf {
-            branch 'master'
-            branch "\\d+\\.\\d+"
-            branch "v\\d?"
-            tag "v\\d+\\.\\d+\\.\\d+*"
-            allOf {
-              expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
-              changeRequest()
-            }
+        /**
+        updates beats updates the framework part and go parts of beats.
+        Then build and test.
+        Finally archive the results.
+        */
+        stage('Update Beats') {
+          options { skipDefaultCheckout() }
+          when {
+            beforeAgent true
+            anyOf {
+              branch 'master'
+              branch "\\d+\\.\\d+"
+              branch "v\\d?"
+              tag "v\\d+\\.\\d+\\.\\d+*"
+              allOf {
+                expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
+                changeRequest()
+              }
 
+            }
           }
-        }
-        steps {
-          withGithubNotify(context: 'Check Apm Server Beats Update') {
-            beatsUpdate()
+          steps {
+            withGithubNotify(context: 'Check Apm Server Beats Update') {
+              beatsUpdate()
+            }
           }
         }
       }

.ci/jobs/apm-beats-update.yml

@@ -18,12 +18,12 @@
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
         discover-tags: true
-        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
+        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         disable-pr-notifications: true
         notification-context: 'apm-beats-update'
         repo: 'beats'
         repo-owner: 'elastic'
-        credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+        credentials-id: github-app-beats-ci
         ssh-checkout:
           credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
         build-strategies:

.ci/jobs/beats-mbp-2.0.yml

@@ -0,0 +1,59 @@
+---
+- job:
+    name: Beats/beats-mbp-2.0
+    display-name: 'Beats (2.0)'
+    description: 'Beats Main Pipeline 2.0'
+    view: Beats
+    concurrent: true
+    project-type: multibranch
+    prune-dead-branches: true
+    days-to-keep: 30
+    script-path: '.ci/Jenkinsfile'
+    triggers: []
+    wrappers: []
+    scm:
+    - github:
+        branch-discovery: 'no-pr'
+        discover-pr-forks-strategy: 'merge-current'
+        discover-pr-forks-trust: 'permission'
+        discover-pr-origin: 'merge-current'
+        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        discover-tags: true
+        disable-pr-notifications: true
+        notification-context: "beats-ci-2.0"
+        repo: 'beats'
+        repo-owner: 'elastic'
+        credentials-id: github-app-beats-ci
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        build-strategies:
+        - tags:
+            ignore-tags-older-than: -1
+            ignore-tags-newer-than: 365
+        - change-request:
+            ignore-target-only-changes: true
+        - named-branches:
+            - exact-name:
+                name: 'master'
+                case-sensitive: true
+            - regex-name:
+                regex: '7\.[x789]'
+                case-sensitive: true
+            - regex-name:
+                regex: '8\.\d+'
+                case-sensitive: true
+        clean:
+            after: true
+            before: true
+        prune: true
+        shallow-clone: true
+        depth: 3
+        do-not-fetch-tags: true
+        submodule:
+            disable: false
+            recursive: true
+            parent-credentials: true
+            timeout: 100
+        timeout: '15'
+        use-author: true
+        wipe-workspace: true

.ci/jobs/beats-windows-mbp.yml

@@ -15,10 +15,10 @@
           discover-tags: false
           # Run MBP for the master branch and PRs
           head-filter-regex: '(master|PR-.*)'
-          notification-context: 'beats-ci'
+          notification-context: 'beats-ci/windows'
           repo: beats
           repo-owner: elastic
-          credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+          credentials-id: github-app-beats-ci
           ssh-checkout:
             credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
           build-strategies:

.ci/jobs/beats.yml

@@ -1,13 +1,12 @@
 ---
 - job:
     name: Beats/beats
-    display-name: 'Beats (replacement)'
+    display-name: 'Beats'
     description: 'Beats Main Pipeline'
     view: Beats
     concurrent: true
     project-type: multibranch
     prune-dead-branches: true
-    number-to-keep: 10
     days-to-keep: 30
     script-path: 'Jenkinsfile'
     triggers: []
@@ -18,21 +17,18 @@
         discover-pr-forks-strategy: 'merge-current'
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
-        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
+        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         discover-tags: true
         notification-context: "beats-ci"
-        property-strategies:
-          all-branches:
-          - suppress-scm-triggering: true
         repo: 'beats'
         repo-owner: 'elastic'
-        credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+        credentials-id: github-app-beats-ci
         ssh-checkout:
           credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
         build-strategies:
         - tags:
             ignore-tags-older-than: -1
-            ignore-tags-newer-than: -1
+            ignore-tags-newer-than: 365
         - change-request:
             ignore-target-only-changes: true
         - named-branches:

.ci/jobs/packaging.yml

@@ -14,12 +14,12 @@
           discover-pr-forks-trust: 'permission'
           discover-pr-origin: 'merge-current'
           discover-tags: true
-          head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
+          head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
           disable-pr-notifications: true
           notification-context: 'beats-packaging'
           repo: 'beats'
           repo-owner: 'elastic'
-          credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+          credentials-id: github-app-beats-ci
           ssh-checkout:
             credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
           build-strategies:

.ci/packaging.groovy

@@ -26,7 +26,7 @@ pipeline {
   triggers {
     issueCommentTrigger('(?i)^\\/packag[ing|e]$')
     // disable upstream trigger on a PR basis
-    upstream("Beats/beats-beats-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
+    upstream("Beats/beats/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
   }
   parameters {
     booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
@@ -37,15 +37,18 @@ pipeline {
       agent { label 'ubuntu && immutable' }
       when {
         beforeAgent true
-        expression {
-          def ret = isCommentTrigger() || isUserTrigger() || isUpstreamTrigger()
-          if(!ret){
-            currentBuild.result = 'NOT_BUILT'
-            currentBuild.description = "The build has been skipped"
-            currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
-            echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
+        anyOf {
+          triggeredBy cause: "IssueCommentCause"
+          expression {
+            def ret = isUserTrigger() || isUpstreamTrigger()
+            if(!ret){
+              currentBuild.result = 'NOT_BUILT'
+              currentBuild.description = "The build has been skipped"
+              currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
+              echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
+            }
+            return ret
           }
-          return ret
         }
       }
       stages {
@@ -76,7 +79,7 @@ pipeline {
                   'x-pack/dockerlogbeat',
                   'x-pack/filebeat',
                   'x-pack/functionbeat',
-                  // 'x-pack/heartbeat',
+                   'x-pack/heartbeat',
                   // 'x-pack/journalbeat',
                   'x-pack/metricbeat',
                   // 'x-pack/packetbeat',
@@ -158,7 +161,6 @@ def pushCIDockerImages(){
     } else if ("${env.BEATS_FOLDER}" == "filebeat") {
       tagAndPush('filebeat-oss')
     } else if ("${env.BEATS_FOLDER}" == "heartbeat"){
-      tagAndPush('heartbeat')
       tagAndPush('heartbeat-oss')
     } else if ("${env.BEATS_FOLDER}" == "journalbeat"){
       tagAndPush('journalbeat')
@@ -174,6 +176,8 @@ def pushCIDockerImages(){
       tagAndPush('elastic-agent')
     } else if ("${env.BEATS_FOLDER}" == "x-pack/filebeat"){
       tagAndPush('filebeat')
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/heartbeat"){
+        tagAndPush('heartbeat')
     } else if ("${env.BEATS_FOLDER}" == "x-pack/metricbeat"){
       tagAndPush('metricbeat')
     }
@@ -185,8 +189,14 @@ def tagAndPush(name){
   if("${env.SNAPSHOT}" == "true"){
     libbetaVer += "-SNAPSHOT"
   }
+
+  def tagName = "${libbetaVer}"
+  if (isPR()) {
+    tagName = "pr-${env.CHANGE_ID}"
+  }
+
   def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}"
-  def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${libbetaVer}"
+  def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${tagName}"
   def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}"
   dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
   retry(3){
@@ -219,7 +229,12 @@ def withMacOSEnv(Closure body){
 }
 
 def publishPackages(baseDir){
-  googleStorageUpload(bucket: "gs://${JOB_GCS_BUCKET}/snapshots",
+  def bucketUri = "gs://${JOB_GCS_BUCKET}/snapshots"
+  if (isPR()) {
+    bucketUri = "gs://${JOB_GCS_BUCKET}/pull-requests/pr-${env.CHANGE_ID}"
+  }
+  def beatsFolderName = getBeatsName(baseDir)
+  googleStorageUpload(bucket: "${bucketUri}/${beatsFolderName}",
     credentialsId: "${JOB_GCS_CREDENTIALS}",
     pathPrefix: "${baseDir}/build/distributions/",
     pattern: "${baseDir}/build/distributions/**/*",
@@ -228,6 +243,18 @@ def publishPackages(baseDir){
   )
 }
 
+/**
+* There is a specific folder structure in https://staging.elastic.co/ and https://artifacts.elastic.co/downloads/
+* therefore the storage bucket in GCP should follow the same folder structure.
+* This is required by https://github.com/elastic/beats-tester
+* e.g.
+* baseDir=name -> return name
+* baseDir=name1/name2/name3-> return name2
+*/
+def getBeatsName(baseDir) {
+  return baseDir.replace('x-pack/', '')
+}
+
 def withBeatsEnv(Closure body) {
   withMageEnv(){
     withEnv([