Merge remote-tracking branch 'upstream/master' into feature/bump-stac…

…k-version-after-8-0-creation

* upstream/master: (69 commits)
  Update stale config following (elastic#30082)
  Make include_matches backwards compatible with 7.x config (elastic#30032)
  [Filebeat] Update handling of elasticsearch server logs (elastic#30018)
  Remove SSL3 support from libbeat and its documentation. (elastic#30071)
  Revert "Packaging: rename arm64 suffix to aarch64 in the tar.gz artifacts ONLY (elastic#28813)" (elastic#30083)
  [libbeat] Add script processor to all beats (elastic#29752)
  Add fonts to support more different types of characters for multiple languages (elastic#29861)
  libbeat/reader: Fix messge conversion to beat.Event (elastic#30057)
  probot[stale]: ignore issues with the tag flaky-test (elastic#30065)
  [DOCS] Add redirect for GSuite module (elastic#30034)
  [Automation] Update elastic stack version to 8.1.0-aa69d697 for testing (elastic#30012)
  Remove msitools install for windows build, using the latest docker image with msitools preinstalled (elastic#30040)
  filebeat/generator/fields: fix dropped error (elastic#29943)
  Include the error message with auditd module events (elastic#30009)
  [Metricbeat] gcp: add firestore metricset (elastic#29918)
  probot: update stale dates (elastic#29997)
  Metricbeat enterprise search module: add xpack.enabled support (elastic#29871)
  x-pack/packetbeat: install Npcap at start-up when required (elastic#29112)
  [Filebeat] Fix panic in decode_cef when recovering from invalid data (elastic#30038)
  Correctly fixe how selected packages are defined (elastic#30039)
  ...

.ci/packaging.groovy

@@ -307,8 +307,6 @@ def tagAndPush(Map args = [:]) {
     tagName = "pr-${env.CHANGE_ID}"
   }
 
-  dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
-
   // supported tags
   def tags = [tagName, "${env.GIT_BASE_COMMIT}"]
   if (!isPR() && aliasVersion != "") {
@@ -386,6 +384,7 @@ def release(){
     withEnv([
       "DEV=true"
     ]) {
+      dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
       dir("${env.BEATS_FOLDER}") {
         sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
       }

.ci/scripts/install-tools.bat

@@ -16,21 +16,15 @@ SET PREVIOUS_USERPROFILE=%USERPROFILE%
 SET USERPROFILE=%OLD_USERPROFILE%
 IF NOT EXIST C:\Python38\python.exe (
     REM Install python 3.8
-    choco install python -y -r --no-progress --version 3.8.5
-    IF NOT ERRORLEVEL 0 (
-        exit /b 1
-    )
+    choco install python -y -r --no-progress --version 3.8.5 || exit /b 1
 )
 python --version
 where python
 
-where /q gcc
-IF ERRORLEVEL 1 (
+WHERE /q gcc
+IF %ERRORLEVEL% NEQ 0 (
     REM Install mingw 5.3.0
-    choco install mingw -y -r --no-progress --version 5.3.0
-    IF NOT ERRORLEVEL 0 (
-        exit /b 1
-    )
+    choco install mingw -y -r --no-progress --version 5.3.0 || exit /b 1
 )
 gcc --version
 where gcc

.github/stale.yml

@@ -1,20 +1,21 @@
 # Configuration for probot-stale - https://github.com/probot/stale
 
 # Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 335
+daysUntilStale: 365
 
 # Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
 # Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
-daysUntilClose: 30
+daysUntilClose: 180
 
 # Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
 onlyLabels: []
 
 # Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
-exemptLabels: []
+exemptLabels:
+  - flaky-test
 
 # Set to true to ignore issues in a project (defaults to false)
-exemptProjects: true
+exemptProjects: false
 
 # Set to true to ignore issues in a milestone (defaults to false)
 exemptMilestones: true
@@ -47,7 +48,7 @@ limitPerRun: 30
 
 # Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
 pulls:
-  daysUntilStale: 30
+  daysUntilStale: 60
   daysUntilClose: 30
   markComment: >
     Hi!
@@ -79,4 +80,4 @@ pulls:
 
 # issues:
 #   exemptLabels:
-#     - confirmed
+#     - confirmed

.go-version

@@ -1 +1 @@
-1.17.5
+1.17.6

CHANGELOG-developer.next.asciidoc

@@ -129,6 +129,8 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Introduce `libbeat/beat.Beat.OutputConfigReloader` {pull}28048[28048]
 - Update Go version to 1.17.1. {pull}27543[27543]
 - Whitelist `GCP_*` environment variables in dev tools {pull}28364[28364]
+- Add support for `credentials_json` in `gcp` module, all metricsets {pull}29584[29584]
+- Add gcp firestore metricset. {pull}29918[29918]
 
 ==== Deprecated
 

CHANGELOG.next.asciidoc

@@ -31,6 +31,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add job.name in pods controlled by Jobs {pull}28954[28954]
 - Change Docker base image from CentOS 7 to Ubuntu 20.04 {pull}29681[29681]
 - Enrich kubernetes metadata with node annotations. {pull}29605[29605]
+- Allign kubernetes configuration settings. {pull}29908[29908]
+- Remove legacy support for SSLv3. {pull}30071[30071]
 
 *Auditbeat*
 
@@ -44,10 +46,14 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - With the default configuration the cef and panw modules will no longer send the `host`
 - Add `while_pattern` type to multiline reader. {pull}19662[19662]
 - auditd dataset: Use process.args to store program arguments instead of auditd.log.aNNN fields. {pull}29601[29601]
+- Remove deprecated old awscloudwatch input name. {pull}29844[29844]
 
 *Heartbeat*
+- Fix broken macOS ICMP python e2e test. {pull}29900[29900]
 - Only add monitor.status to browser events when summary. {pull}29460[29460]
 - Also add summary to journeys for which the synthetics runner crashes. {pull}29606[29606]
+- Update size of ICMP packets to adhere to standard min size. {pull}29948[29948]
+- Add fonts to support more different types of characters for multiple languages. {pull}29606[29861]
 
 *Metricbeat*
 
@@ -97,18 +103,24 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - system/socket: Fix process name and arg truncation for long names, paths and args lists. {issue}24667[24667] {pull}29410[29410]
 - system/socket: Fix startup errors on newer 5.x kernels due to missing _do_fork function. {issue}29607[29607] {pull}29744[29744]
 - libbeat/processors/add_process_metadata: Fix memory leak in process cache. {issue}24890[24890] {pull}29717[29717]
+- auditd: Add error.message to events when processing fails. {pull}30009[30009]
 
 *Filebeat*
 
 - aws-s3: Stop trying to increase SQS message visibility after ReceiptHandleIsInvalid errors. {pull}29480[29480]
 - Fix handling of IPv6 addresses in netflow flow events. {issue}19210[19210] {pull}29383[29383]
 - Fix `sophos` KV splitting and syslog header handling {issue}24237[24237] {pull}29331[29331]
 - Undo deletion of endpoint config from cloudtrail fileset in {pull}29415[29415]. {pull}29450[29450]
+- Make Cisco ASA and FTD modules conform to the ECS definition for event.outcome and event.type. {issue}29581[29581] {pull}29698[29698]
 - ibmmq: Fixed `@timestamp` not being populated with correct values. {pull}29773[29773]
+- Fix using log_group_name_prefix in aws-cloudwatch input. {pull}29695[29695]
+- aws-s3: Improve gzip detection to avoid false negatives. {issue}29968[29968]
+- decode_cef: Fix panic when recovering from invalid CEF extensions that contain escape characters. {issue}30010[30010]
 
 *Heartbeat*
 
 - Fix race condition in http monitors using `mode:all` that can cause crashes. {pull}29697[pull]
+- Fix broken ICMP availability check that prevented heartbeat from starting in rare cases. {pull}29413[pull]
 
 *Metricbeat*
 
@@ -118,6 +130,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix gcp metrics metricset apply aligner to all metric_types {pull}29514[29513]
 - Extract correct index property in kibana.stats metricset {pull}29622[29622]
 - Fixed bug with `elasticsearch/cluster_stats` metricset not recording license expiration date correctly. {pull}29711[29711]
+- Fixed GCP GKE Overview dashboard {pull}29913[29913]
 
 *Packetbeat*
 
@@ -148,10 +161,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add FIPS configuration option for all AWS API calls. {pull}28899[28899]
 - Add `default_region` config to AWS common module. {pull}29415[29415]
 - Add support for latest k8s versions v1.23 and v1.22 {pull}29575[29575]
+- Add `script` processor to all beats {issue}29269[29269] {pull}29752[29752]
 - Only connect to Elasticsearch instances with the same version or newer. {pull}29683[29683]
+- Move umask from code to service files. {pull}29708[29708]
 
 *Auditbeat*
 
+- system/process: Prevent hashing files in other mnt namespaces. {issue}25777[25777] {issue}29678[29678] {pull}29786[29786]
 
 *Filebeat*
 
@@ -166,26 +182,34 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Heartbeat*
 
+- More errors are now visible in ES with new logic failing monitors later to ease debugging. {pull}29413[pull]
+
 
 *Metricbeat*
 
 - Preliminary AIX support {pull}27954[27954]
 - Add option to skip older k8s events {pull}29396[29396]
 - Add `add_resource_metadata` configuration to Kubernetes module. {pull}29133[29133]
+- Add `containerd` module with `cpu`, `memory`, `blkio` metricsets. {pull}29247[29247]
 - Add `container.id` and `container.runtime` ECS fields in container metricset. {pull}29560[29560]
 - Add `memory.workingset.limit.pct` field in Kubernetes container/pod metricset. {pull}29547[29547]
 - Add k8s metadata in state_cronjob metricset. {pull}29572[29572]
 - Add `elasticsearch.cluster.id` field to Beat and Kibana modules. {pull}29577[29577]
 - Add `elasticsearch.cluster.id` field to Logstash module. {pull}29625[29625]
+- Add `xpack.enabled` support for Enterprise Search module. {pull}29871[29871]
+- Add gcp firestore metricset. {pull}29918[29918] 
 
 *Packetbeat*
 
+- Add automated OEM Npcap installation handling. {pull}29112[29112]
+
 *Functionbeat*
 
 
 *Winlogbeat*
 
 - Add support for custom XML queries {issue}1054[1054] {pull}29330[29330]
+- Add support for sysmon event ID 26; FileDeleteDetected. {issue}26280[26280] {pull}29957[29957]
 
 *Elastic Log Driver*
 
@@ -213,9 +237,3 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 ==== Known Issue
 
 *Journalbeat*
-
-
-
-
-
-

NOTICE.txt

@@ -16728,6 +16728,43 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+--------------------------------------------------------------------------------
+Dependency : golang.org/x/mod
+Version: v0.5.1
+Licence type (autodetected): BSD-3-Clause
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.5.1/LICENSE:
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/net
 Version: v0.0.0-20211020060615-d418f374d309
@@ -22563,12 +22600,12 @@ For t_cl_generator.cc
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/apoydence/eachers
+Dependency : github.com/poy/eachers
 Version: v0.0.0-20181020210610-23942921fe77
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/apoydence/eachers@v0.0.0-20181020210610-23942921fe77/LICENSE.md:
+Contents of probable licence file $GOMODCACHE/github.com/poy/eachers@v0.0.0-20181020210610-23942921fe77/LICENSE.md:
 
 The MIT License (MIT)
 
@@ -34519,43 +34556,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
 
---------------------------------------------------------------------------------
-Dependency : golang.org/x/mod
-Version: v0.5.1
-Licence type (autodetected): BSD-3-Clause
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.5.1/LICENSE:
-
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/term
 Version: v0.0.0-20210615171337-6886f2dfbf5b
@@ -37624,4 +37624,3 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17.5
+FROM golang:1.17.6
 
 RUN \
     apt-get update \

auditbeat/auditbeat.reference.yml

@@ -414,10 +414,9 @@ output.elasticsearch:
   # Number of workers per Elasticsearch host.
   #worker: 1
 
-  # Optional index name. The default is "auditbeat" plus date
-  # and generates [auditbeat-]YYYY.MM.DD keys.
+  # Optional data stream or index name. The default is "auditbeat-%{[agent.version]}".
   # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
-  #index: "auditbeat-%{[agent.version]}-%{+yyyy.MM.dd}"
+  #index: "auditbeat-%{[agent.version]}"
 
   # Optional ingest pipeline. By default no pipeline will be used.
   #pipeline: ""
@@ -1226,10 +1225,8 @@ setup.template.settings:
 
 # ====================== Index Lifecycle Management (ILM) ======================
 
-# Configure index lifecycle management (ILM). These settings create a write
-# alias and add additional settings to the index template. When ILM is enabled,
-# output.elasticsearch.index is ignored, and the write alias is used to set the
-# index name.
+# Configure index lifecycle management (ILM) to manage the backing indices
+# of your data streams.
 
 # Enable ILM support. Valid values are true, false.
 #setup.ilm.enabled: true

auditbeat/docker-compose.yml

@@ -8,7 +8,7 @@ services:
     environment:
       - ES_HOST=elasticsearch
       - ES_PORT=9200
-      - ES_USER=beats
+      - ES_USER=auditbeat_user
       - ES_PASS=testing
       - KIBANA_HOST=kibana
       - KIBANA_PORT=5601
@@ -32,8 +32,15 @@ services:
     extends:
       file: ../testing/environments/${TESTING_ENVIRONMENT}.yml
       service: elasticsearch
+    healthcheck:
+      test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"]
+      retries: 300
+      interval: 1s
 
   kibana:
     extends:
       file: ../testing/environments/${TESTING_ENVIRONMENT}.yml
       service: kibana
+    healthcheck:
+      test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"]
+      retries: 600

auditbeat/module/auditd/audit_linux.go

@@ -512,7 +512,7 @@ func buildMetricbeatEvent(msgs []*auparse.AuditMessage, config Config) mb.Event
 	auditEvent, err := aucoalesce.CoalesceMessages(msgs)
 	if err != nil {
 		// Add messages on error so that it's possible to debug the problem.
-		out := mb.Event{RootFields: common.MapStr{}}
+		out := mb.Event{RootFields: common.MapStr{}, Error: err}
 		addEventOriginal(msgs, out.RootFields)
 		return out
 	}

auditbeat/module/file_integrity/metricset_test.go

@@ -988,7 +988,7 @@ func getConfig(path ...string) map[string]interface{} {
 }
 
 func skipOnCIForDarwinAMD64(t testing.TB) {
-	if os.Getenv("BUILD_ID") != "" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
+	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
 		t.Skip("Skip test on CI for darwin/amd64")
 	}
 }