Merge remote-tracking branch 'upstream/master' into feature/packaging…

…-arm

* upstream/master: (24 commits)
  Add example input autodsicover config (elastic#24157)
  Empty configuration options generate `<no value>` string for azure-eventhub input (elastic#24156)
  Remove OSS reference for kibana and elasticsearch (elastic#24164)
  Skip flaky TestActions on MacOSx (elastic#23966)
  [Filebeat][AWS] Fix vpcflow pipeline exception: Cannot invoke "Object.getClass()" because "receiver" is null (elastic#24167)
  [Elastic Agent] Fix docker entrypoint for elastic-agent. (elastic#24155)
  [PACKAGING] Push docker images with the architecture in the version (elastic#24121)
  [Agent] Add agent standalone manifests for system module & Pod's log collection (elastic#23938)
  indicator type url is in upper case (elastic#24152)
  [Filebeat] Document netflow internal_networks and set default (elastic#24110)
  [Filebeat] Adding fixes to the TI module (elastic#24133)
  [Enhancement] Add RotateOnStartup feature flag for file output (elastic#19347)
  [Ingest Manager] Fix: Successfully installed and enrolled agent running standalone (elastic#24128)
  Set Elastic licence type for APM server Beats update job (elastic#24122)
  Add logrotation section on Running Filebeat on k8s (elastic#24120)
  [CI] Run if manual UI (elastic#24116)
  [CI] enable x-pack/heartbeat in the CI (elastic#23873)
  chore: comment out the E2E (elastic#24109)
  chore: add-backport-next (elastic#24098)
  Adjust the position of the architecture name in Dockerlogbeat tarball (elastic#24095)
  ...

.backportrc.json

@@ -1,6 +1,6 @@
 {
   "upstream": "elastic/beats",
-  "branches": [ { "name": "7.x", "checked": true }, "7.11" ],
+  "branches": [ { "name": "7.x", "checked": true }, "7.12", "7.11" ],
   "labels": ["backport"],
   "autoAssign": true,
   "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"

.ci/apm-beats-update.groovy

@@ -86,10 +86,9 @@ pipeline {
               branch "v\\d?"
               tag "v\\d+\\.\\d+\\.\\d+*"
               allOf {
-                expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
+                expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() || isUserTrigger() }
                 changeRequest()
               }
-
             }
           }
           steps {
@@ -127,6 +126,7 @@ def beatsUpdate() {
         git config --global --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pr/*"
 
         go mod edit -replace github.com/elastic/beats/v7=\${GOPATH}/src/github.com/elastic/beats-local
+        echo '{"name": "${GOPATH}/src/github.com/elastic/beats-local", "licenceType": "Elastic"}' >> \${GOPATH}/src/github.com/elastic/beats-local/dev-tools/notice/overrides.json
         make update
         git commit -a -m beats-update
 

.ci/packaging.groovy

@@ -152,7 +152,9 @@ pipeline {
                   withGithubNotify(context: "Packaging Linux ${BEATS_FOLDER}") {
                     deleteDir()
                     release()
-                    pushCIDockerImages()
+                    dir("${BASE_DIR}"){
+                      pushCIDockerImages(arch: 'amd64')
+                    }
                   }
                   prepareE2ETestForPackage("${BEATS_FOLDER}")
                 }
@@ -234,7 +236,9 @@ pipeline {
                   withGithubNotify(context: "Packaging linux/arm64 ${BEATS_FOLDER}") {
                     deleteWorkspace()
                     release()
-                    pushCIDockerImages()
+                    dir("${BASE_DIR}"){
+                      pushCIDockerImages(arch: 'arm64')
+                    }
                   }
                 }
                 post {
@@ -270,27 +274,37 @@ pipeline {
   }
 }
 
-def pushCIDockerImages(){
+/**
+* @param arch what architecture
+*/
+def pushCIDockerImages(Map args = [:]) {
+  def arch = args.get('arch', 'amd64')
   catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
     if (env?.BEATS_FOLDER?.endsWith('auditbeat')) {
-      tagAndPush('auditbeat')
+      tagAndPush(beatName: 'auditbeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('filebeat')) {
-      tagAndPush('filebeat')
+      tagAndPush(beatName: 'filebeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('heartbeat')) {
-      tagAndPush('heartbeat')
+      tagAndPush(beatName: 'heartbeat', arch: arch)
     } else if ("${env.BEATS_FOLDER}" == "journalbeat"){
-      tagAndPush('journalbeat')
+      tagAndPush(beatName: 'journalbeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) {
-      tagAndPush('metricbeat')
+      tagAndPush(beatName: 'metricbeat', arch: arch)
     } else if ("${env.BEATS_FOLDER}" == "packetbeat"){
-      tagAndPush('packetbeat')
+      tagAndPush(beatName: 'packetbeat', arch: arch)
     } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
-      tagAndPush('elastic-agent')
+      tagAndPush(beatName: 'elastic-agent', arch: arch)
     }
   }
 }
 
-def tagAndPush(beatName){
+/**
+* @param beatName name of the Beat
+* @param arch what architecture
+*/
+def tagAndPush(Map args = [:]) {
+  def beatName = args.beatName
+  def arch = args.get('arch', 'amd64')
   def libbetaVer = env.BEAT_VERSION
   def aliasVersion = ""
   if("${env.SNAPSHOT}" == "true"){
@@ -307,14 +321,22 @@ def tagAndPush(beatName){
 
   dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
 
+  // supported tags
+  def tags = [tagName, "${env.GIT_BASE_COMMIT}"]
+  if (!isPR() && aliasVersion != "") {
+    tags << aliasVersion
+  }
   // supported image flavours
   def variants = ["", "-oss", "-ubi8"]
   variants.each { variant ->
-    doTagAndPush(beatName, variant, libbetaVer, tagName)
-    doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}")
-
-    if (!isPR() && aliasVersion != "") {
-      doTagAndPush(beatName, variant, libbetaVer, aliasVersion)
+    tags.each { tag ->
+      // TODO:
+      // For backward compatibility let's ensure we tag only for amd64, then E2E can benefit from until
+      // they support the versioning with the architecture
+      if ("${arch}" == "amd64") {
+        doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}")
+      }
+      doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}")
     }
   }
 }
@@ -325,18 +347,19 @@ def tagAndPush(beatName){
 * @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
 * @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
 */
-def doTagAndPush(beatName, variant, sourceTag, targetTag) {
+def doTagAndPush(Map args = [:]) {
+  def beatName = args.beatName
+  def variant = args.variant
+  def sourceTag = args.sourceTag
+  def targetTag = args.targetTag
   def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
   def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"
-
   def iterations = 0
   retryWithSleep(retries: 3, seconds: 5, backoff: true) {
     iterations++
-    def status = sh(label: "Change tag and push ${targetName}", script: """
-      docker tag ${sourceName} ${targetName}
-      docker push ${targetName}
-    """, returnStatus: true)
-
+    def status = sh(label: "Change tag and push ${targetName}",
+                    script: ".ci/scripts/docker-tag-push.sh ${sourceName} ${targetName}",
+                    returnStatus: true)
     if ( status > 0 && iterations < 3) {
       error("tag and push failed for ${beatName}, retry")
     } else if ( status > 0 ) {

.ci/scripts/docker-tag-push.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+MSG="parameter missing."
+SOURCE_IMAGE=${1:?$MSG}
+TARGET_IMAGE=${2:?$MSG}
+
+if docker image inspect "${SOURCE_IMAGE}" &> /dev/null ; then
+    docker tag "${SOURCE_IMAGE}" "${TARGET_IMAGE}"
+    docker push "${TARGET_IMAGE}"
+else
+    echo "docker image ${SOURCE_IMAGE} does not exist"
+fi

CHANGELOG.next.asciidoc

@@ -48,6 +48,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Add fileset to ingest PostgreSQL CSV logs. {pull}23334[23334]
 - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
 - Improve ECS field mappings in panw module.  event.outcome now only contains success/failure per ECS specification. {issue}16025[16025] {pull}17910[17910]
 - Improve ECS categorization field mappings for nginx module. http.request.referrer only populated when nginx sets a value {issue}16174[16174] {pull}17844[17844]
@@ -105,6 +106,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `nodes` to filebeat-kubernetes.yaml ClusterRole. {issue}24051[24051] {pull}24052[24052]
 
 *Heartbeat*
+- Adds negative body match. {pull}20728[20728]
+- Refactor synthetics configuration to new syntax. {pull}23467[23467]
 
 *Journalbeat*
 
@@ -278,6 +281,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change the `event.created` in Netflow events to be the time the event was created by Filebeat
   to be consistent with ECS. {pull}23094[23094]
 - Update `filestream` reader offset when a line is skipped. {pull}23417[23417]
+- Add check for empty values in azure module. {pull}24156[24156]
 
 *Filebeat*
 
@@ -384,6 +388,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix Logstash module handling of logstash.log.log_event.action field. {issue}20709[20709]
 - aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920]
 - Zoom module pipeline failed to ingest some chat_channel events. {pull}23904[23904]
+- Fix Netflow module issue with missing `internal_networks` config parameter. {issue}24094[24094] {pull}24110[24110]
+- Fix aws/vpcflow generating errors for empty logs or unidentified formats. {pull}24167[24167]
 
 *Heartbeat*
 
@@ -569,6 +575,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add keystore support for autodiscover static configurations. {pull]16306[16306]
 - Add TLS support to Kerberos authentication in Elasticsearch. {pull}18607[18607]
 - Add support for multiple sets of hints on autodiscover {pull}18883[18883]
+- Add config option `rotate_on_startup` to file output {issue}19150[19150] {pull}19347[19347]
 - Add a configurable delay between retries when an app metadata cannot be retrieved by `add_cloudfoundry_metadata`. {pull}19181[19181]
 - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562]
 - Add support for DNS over TLS for the dns_processor. {pull}19321[19321]
@@ -993,6 +1000,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Apache: convert status.total_kbytes to status.total_bytes in fleet mode. {pull}23022[23022]
 - Release MSSQL as GA {pull}23146[23146]
 - Enrich events of `state_service` metricset with kubernetes services' metadata. {pull}23730[23730]
+- Add support for Darwin/arm M1. {pull}24019[24019]
 - Check fields are documented in aws metricsets. {pull}23887[23887]
 
 *Packetbeat*

Jenkinsfile

@@ -334,23 +334,26 @@ def uploadPackages(bucketUri, beatsFolder){
 /**
 * Push the docker images for the given beat.
 * @param beatsFolder beats folder
+* @param arch what architecture
 */
-def pushCIDockerImages(beatsFolder){
+def pushCIDockerImages(Map args = [:]) {
+  def arch = args.get('arch', 'amd64')
+  def beatsFolder = args.beatsFolder
   catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
     if (beatsFolder.endsWith('auditbeat')) {
-      tagAndPush('auditbeat')
+      tagAndPush(beatName: 'auditbeat', arch: arch)
     } else if (beatsFolder.endsWith('filebeat')) {
-      tagAndPush('filebeat')
+      tagAndPush(beatName: 'filebeat', arch: arch)
     } else if (beatsFolder.endsWith('heartbeat')) {
-      tagAndPush('heartbeat')
+      tagAndPush(beatName: 'heartbeat', arch: arch)
     } else if ("${beatsFolder}" == "journalbeat"){
-      tagAndPush('journalbeat')
+      tagAndPush(beatName: 'journalbeat', arch: arch)
     } else if (beatsFolder.endsWith('metricbeat')) {
-      tagAndPush('metricbeat')
+      tagAndPush(beatName: 'metricbeat', arch: arch)
     } else if ("${beatsFolder}" == "packetbeat"){
-      tagAndPush('packetbeat')
+      tagAndPush(beatName: 'packetbeat', arch: arch)
     } else if ("${beatsFolder}" == "x-pack/elastic-agent") {
-      tagAndPush('elastic-agent')
+      tagAndPush(beatName: 'elastic-agent', arch: arch)
     }
   }
 }
@@ -359,7 +362,9 @@ def pushCIDockerImages(beatsFolder){
 * Tag and push all the docker images for the given beat.
 * @param beatName name of the Beat
 */
-def tagAndPush(beatName){
+def tagAndPush(Map args = [:]) {
+  def beatName = args.beatName
+  def arch = args.get('arch', 'amd64')
   def libbetaVer = env.VERSION
   if("${env?.SNAPSHOT.trim()}" == "true"){
     aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version
@@ -373,41 +378,40 @@ def tagAndPush(beatName){
     tagName = "pr-${env.CHANGE_ID}"
   }
 
+  // supported tags
+  def tags = [tagName, "${env.GIT_BASE_COMMIT}"]
+  if (!isPR() && aliasVersion != "") {
+    tags << aliasVersion
+  }
   // supported image flavours
   def variants = ["", "-oss", "-ubi8"]
   variants.each { variant ->
-    doTagAndPush(beatName, variant, libbetaVer, tagName)
-    doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}")
-
-    if (!isPR() && aliasVersion != "") {
-      doTagAndPush(beatName, variant, libbetaVer, aliasVersion)
+    tags.each { tag ->
+      doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}")
     }
   }
 }
 
 /**
-* Tag and push the given sourceTag docker image with the tag name targetTag.
 * @param beatName name of the Beat
 * @param variant name of the variant used to build the docker image name
 * @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
 * @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
 */
-def doTagAndPush(beatName, variant, sourceTag, targetTag) {
+def doTagAndPush(Map args = [:]) {
+  def beatName = args.beatName
+  def variant = args.variant
+  def sourceTag = args.sourceTag
+  def targetTag = args.targetTag
   def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
   def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"
 
   def iterations = 0
   retryWithSleep(retries: 3, seconds: 5, backoff: true) {
     iterations++
-    def status = sh(label: "Change tag and push ${targetName}", script: """#!/usr/bin/env bash
-      docker images
-      if docker image inspect "${sourceName}" &> /dev/null ; then
-        docker tag ${sourceName} ${targetName}
-        docker push ${targetName}
-      else
-        echo 'docker image ${sourceName} does not exist'
-      fi
-    """, returnStatus: true)
+    def status = sh(label: "Change tag and push ${targetName}",
+                    script: ".ci/scripts/docker-tag-push.sh ${sourceName} ${targetName}",
+                    returnStatus: true)
     if ( status > 0 && iterations < 3) {
       error("tag and push failed for ${beatName}, retry")
     } else if ( status > 0 ) {
@@ -475,6 +479,7 @@ def target(Map args = [:]) {
   def isMage = args.get('isMage', false)
   def isE2E = args.e2e?.get('enabled', false)
   def isPackaging = args.get('package', false)
+  def dockerArch = args.get('dockerArch', 'amd64')
   withNode(args.label) {
     withGithubNotify(context: "${context}") {
       withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) {
@@ -495,7 +500,7 @@ def target(Map args = [:]) {
         // TODO:
         // push docker images should happen only after the e2e?
         if (isPackaging) {
-          pushCIDockerImages("${directory}")
+          pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch)
         }
       }
     }
@@ -934,7 +939,15 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
       steps.packagingArm(context: args.context, command: args.content.get('packaging-arm'), directory: args.project, label: args.label, isMage: true, id: args.id, e2e: args.content.get('e2e'), package: true)
     }
     if(args?.content?.containsKey('packaging-linux')) {
-      steps.packagingLinux(context: args.context, command: args.content.get('packaging-linux'), directory: args.project, label: args.label, isMage: true, id: args.id, e2e: args.content.get('e2e'), package: true)
+      steps.packagingLinux(context: args.context,
+                           command: args.content.get('packaging-linux'),
+                           directory: args.project,
+                           label: args.label,
+                           isMage: true,
+                           id: args.id,
+                           e2e: args.content.get('e2e'),
+                           package: true,
+                           dockerArch: 'amd64')
     }
     if(args?.content?.containsKey('k8sTest')) {
       steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)

NOTICE.txt

@@ -5891,11 +5891,11 @@ This Agreement is governed by the laws of the State of New York and the intellec
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/ecs
-Version: v1.0.0-beta2.0.20210202203518-638aa2bb5271
+Version: v1.8.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/ecs@v1.0.0-beta2.0.20210202203518-638aa2bb5271/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/ecs@v1.8.0/LICENSE.txt:
 
 
                                  Apache License
@@ -7665,11 +7665,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-sysinfo
-Version: v1.5.0
+Version: v1.6.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.5.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.6.0/LICENSE.txt:
 
 
                                  Apache License
@@ -8299,11 +8299,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-ucfg@v0.8.3/
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/gosigar
-Version: v0.13.0
+Version: v0.14.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/gosigar@v0.13.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/gosigar@v0.14.0/LICENSE:
 
                               Apache License
                         Version 2.0, January 2004