Merge remote-tracking branch 'upstream/master' into feature/support-d…

…arwin-arm64

* upstream/master: (295 commits)
  Update urllib to 1.26.5. (elastic#26380)
  Update golang.org/x/crypto (elastic#26448)
  [Filebeat] Update Fortinet Ingest Pipeline (elastic#24816)
  Move parsers outside of filestream input so others can use them as well (elastic#26541)
  [Filebeat] Fix `threatintel.indicator.url.full` field not populating (elastic#26508)
  [Filebeat] Add network direction processor to Zeek and Suricata modules (elastic#24620)
  Logging code cleanup related to Nomad auto-discovery (elastic#26498)
  [Metricbeat] Add Couchbase's Sync Gateway module (elastic#25599)
  Refactor add_cloud_metadata to handle ECS fields easier (elastic#26438)
  [Elastic Agent] Improper casting of int64 (elastic#26520)
  [Elastic Agent] Enable configuring monitoring namespace (elastic#26439)
  [Heartbeat] configure permissions for synthetics config (elastic#26393)
  Osquerybeat: set the raw index name to supress the timestamp suffix (elastic#26545)
  [Heartbeat] add screenshots config to synthetics (elastic#26455)
  [Elastic Agent] Use http2 to connect to Fleet Server. (elastic#26474)
  Remove all docs about  Beats central management (elastic#26399)
  update data.json for gcp billing (elastic#26506)
  Skip x-pack metricbeat tests (elastic#26537)
  [Elastic Agent] Fix issue with FLEET_CA not being used with Fleet Server in container (elastic#26529)
  Add changelog entry for  elastic#26224 (elastic#26531)
  ...

.ci/bump-go-release-version.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Given the Golang release version this script will bump the version.
+#
+# This script is executed by the automation we are putting in place
+# and it requires the git add/commit commands.
+#
+# Parameters:
+#	$1 -> the Golang release version to be bumped. Mandatory.
+#
+set -euo pipefail
+MSG="parameter missing."
+GO_RELEASE_VERSION=${1:?$MSG}
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+
+if [ "${OS}" == "darwin" ] ; then
+	SED="sed -i .bck"
+else
+	SED="sed -i"
+fi
+
+echo "Update go version ${GO_RELEASE_VERSION}"
+echo "${GO_RELEASE_VERSION}" > .go-version
+git add .go-version
+
+find . -maxdepth 3 -name Dockerfile -print0 |
+    while IFS= read -r -d '' line; do
+        ${SED} -E -e "s#(FROM golang):[0-9]+\.[0-9]+\.[0-9]+#\1:${GO_RELEASE_VERSION}#g" "$line"
+        ${SED} -E -e "s#(ARG GO_VERSION)=[0-9]+\.[0-9]+\.[0-9]+#\1=${GO_RELEASE_VERSION}#g" "$line"
+        git add "${line}"
+    done
+
+${SED} -E -e "s#(:go-version:) [0-9]+\.[0-9]+\.[0-9]+#\1 ${GO_RELEASE_VERSION}#g" libbeat/docs/version.asciidoc
+git add libbeat/docs/version.asciidoc
+
+git diff --staged --quiet || git commit -m "[Automation] Update go release version to ${GO_RELEASE_VERSION}"
+git --no-pager log -1
+
+echo "You can now push and create a Pull Request"

.ci/packaging.groovy

@@ -425,11 +425,13 @@ def triggerE2ETests(String suite) {
 
   def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
   def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
+  def beatVersion = "${env.BEAT_VERSION}-SNAPSHOT"
 
   def parameters = [
     booleanParam(name: 'forceSkipGitChecks', value: true),
     booleanParam(name: 'forceSkipPresubmit', value: true),
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+    string(name: 'BEAT_VERSION', value: beatVersion),
     booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
     string(name: 'runTestsSuites', value: suite),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),

.ci/packer_cache.sh

@@ -45,6 +45,8 @@ function dockerPullImages() {
   docker.elastic.co/kibana/kibana:${SNAPSHOT}
   docker.elastic.co/logstash/logstash:${SNAPSHOT}
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm
+  docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armhf
+  docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armel
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-base-arm-debian9
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main

.gitignore

@@ -51,5 +51,5 @@ x-pack/elastic-agent/pkg/agent/operation/tests/scripts/serviceable-1.0-darwin-x8
 *.terraform
 *.tfstate*
 
-# Files generated with the bump elastic stack version automation
-testing/environments/*.bck
+# Files generated with the bump version automations
+*.bck

.go-version

@@ -1 +1 @@
-1.15.9
+1.16.5

.mergify.yml

@@ -1,8 +1,20 @@
 pull_request_rules:
+  - name: forward-port patches to master branch
+    conditions:
+      - merged
+      - label=backport-v8.0.0
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "master"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
   - name: backport patches to 7.x branch
     conditions:
       - merged
-      - base=master
       - label=backport-v7.14.0
     actions:
       backport:
@@ -16,7 +28,6 @@ pull_request_rules:
   - name: backport patches to 7.13 branch
     conditions:
       - merged
-      - base=master
       - label=backport-v7.13.0
     actions:
       backport:
@@ -30,7 +41,6 @@ pull_request_rules:
   - name: backport patches to 7.12 branch
     conditions:
       - merged
-      - base=master
       - label=backport-v7.12.0
     actions:
       backport:
@@ -55,7 +65,17 @@ pull_request_rules:
             git merge upstream/{{base}}
             git push upstream {{head}}
             ```
-  - name: Automatic squash and merge with success checks and the files matching the regex ^testing/environments/snapshot* are modified.
+  - name: automatic approval for mergify pull requests with bump updates
+    conditions:
+      - author=mergify[bot]
+      - check-success=beats-ci/pr-merge
+      - label=automation
+      - files~=^testing/environments/snapshot.*\.yml$
+    actions:
+      review:
+        type: APPROVE
+        message: Automatically approving mergify
+  - name: automatic squash and merge with success checks and the files matching the regex ^testing/environments/snapshot* are modified.
     conditions:
       - check-success=beats-ci/pr-merge
       - label=automation
@@ -64,11 +84,25 @@ pull_request_rules:
       merge:
         method: squash
         strict: smart+fasttrack
-  - name: delete upstream branch after merging changes on testing/environments/snapshot*
+  - name: delete upstream branch after merging changes on testing/environments/snapshot* or it's closed
     conditions:
-      - merged
-      - label=automation
-      - head~=^update-stack-version
-      - files~=^testing/environments/snapshot.*\.yml$
+      - or:
+        - merged
+        - closed
+      - and:
+        - label=automation
+        - head~=^update-stack-version
+        - files~=^testing/environments/snapshot.*\.yml$
+    actions:
+      delete_head_branch:
+  - name: delete upstream branch after merging changes on .go-version or it's closed
+    conditions:
+      - or:
+        - merged
+        - closed
+      - and:
+        - label=automation
+        - head~=^update-go-version
+        - files~=^.go-version$
     actions:
       delete_head_branch:

CHANGELOG-developer.asciidoc

@@ -208,3 +208,4 @@ The list below covers the major changes between 6.3.0 and 7.0.0-alpha2 only.
 - Allow/Merge fields.yml overrides {pull}9188[9188]
 - Filesets can now define multiple ingest pipelines, with the first one considered as the entry point pipeline. {pull}8914[8914]
 - Add `group_measurements_by_instance` option to windows perfmon metricset. {pull}8688[8688]
+- Bump ECS version to 1.10.0. {issue}25734[25734]

CHANGELOG-developer.next.asciidoc

@@ -54,6 +54,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Make implementing `Close` required for `reader.Reader` interfaces. {pull}20455[20455]
 - Remove `NumCPU` as clients should update the CPU count on the fly in case of config changes in a VM. {pull}23154[23154]
 - Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093]
+- Update Darwin build image to a debian 10 base that increases the MacOS SDK and minimum supported version used in build to 10.14. {issue}24193[24193]
 
 ==== Bugfixes
 
@@ -112,3 +113,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.15.8. {pull}23955[23955]
 - Update Go version to 1.15.9. {pull}24442[24442]
 - Update Go version to 1.15.10. {pull}24606[24606]
+- Update Go version to 1.15.12. {pull}25629[25629]
+- Update Go version to 1.16.4. {issue}25346[25346] {pull}25671[25671]
+- Add sorting to array fields for generated data files (*-generated.json) {pull}25320[25320]
+- Update Go version to 1.16.5. {issue}26182[26182] {pull}26186[26186]

CHANGELOG.asciidoc

@@ -3,6 +3,167 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.13.2]]
+=== Beats version 7.13.2
+https://github.com/elastic/beats/compare/v7.13.1...v7.13.2[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix ILM alias creation when write alias exists and initial index does not exist. {pull}26146[26146]
+- Fix ILM setup log reporting that a policy or an alias was created, even though the creation of any resource was disabled. {issue}24046[24046] {pull}24480[24480]
+- Fix ILM alias not being created if `setup.ilm.check_exists: false` and `setup.ilm.overwrite: true` has been configured. {pull}24480[24480]
+- Allow cgroup self-monitoring to see alternate `hostfs` paths. {pull}24334[24334]
+- Fix `make setup` instructions for a new Beat. {pull}24944[24944]
+- Fix out-of-date FreeBSD vagrantbox. {pull}25652[25652]
+- Fix handling of `file_selectors` in aws-s3 input. {pull}25792[25792]
+- Include date separator in the filename prefix of `dateRotator` to make sure nothing gets purged accidentally. {pull}26176[26176]
+
+*Auditbeat*
+
+- auditd: Fix kernel deadlock when netlink congestion causes "no buffer space available" errors. {issue}26031[26031] {pull}26032[26032]
+
+*Filebeat*
+
+- o365: Avoid mapping exception for `Parameters` and `ExtendedProperties` fields of string type. {pull}26164[26164]
+
+[[release-notes-7.13.1]]
+=== Beats version 7.13.1
+https://github.com/elastic/beats/compare/v7.13.0...v7.13.1[View commits]
+
+==== Bugfixes
+
+*Auditbeat*
+
+- Mitigate deadlock is aws-s3 input when SQS visibility timeout is exceeded. {issue}25750[25750]
+- Fix httpjson cursor override with empty values by adding `ignore_empty_value` option. {pull}25802[25802]
+
+*Filebeat*
+
+- Improve inode reuse handling by removing state for removed files more eagerly from the internal state table in the logs inputs. {pull}25756[25756]
+
+[[release-notes-7.13.0]]
+=== Beats version 7.13.0
+https://github.com/elastic/beats/compare/v7.12.1...v7.13.0[View commits]
+
+==== Breaking changes
+
+*Affecting all Beats*
+
+- Use alias to report container image in k8s metadata. {pull}24380[24380]
+- Set `cleanup_timeout` to zero by default in docker and kubernetes autodiscover in all beats except Filebeat where it is kept to 60 seconds. {pull}24681[24681]
+- Update to ECS 1.9.0. {pull}24909[24909]
+
+*Filebeat*
+
+- Changes filebeat httpjson input's append transform to create a list even with only a single value{pull}25074[25074]
+- Deprecated the cyberark module (replaced by cyberarkpas). {issue}25261[25261] {pull}25505[25505]
+
+*Metricbeat*
+
+- Store `cloudfoundry.container.cpu.pct` in decimal form and as `scaled_float`. {pull}24219[24219]
+- Remove `index_stats.created` field from Elasticsearch/index Metricset {pull}25113[25113]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix events being dropped if they contain a floating point value of NaN or Inf. {pull}25051[25051]
+- Fix templates being overwritten if there was an error when check for the template existance. {pull}24332[24332]
+- Add `expand_keys` to the list of permitted config fields for `decode_json_fields` {pull}24862[24862]
+- Fix discovery of short-living and failing pods in Kubernetes autodiscover {issue}22718[22718] {pull}24742[24742]
+- Fix panic when overwriting metadata {pull}24741[24741]
+- Fix role_arn to work with access keys for AWS. {pull}25446[25446]
+- Fix `community_id` processor so that ports greater than 65535 aren't valid. {pull}25409[25409]
+
+*Auditbeat*
+
+- Fix o365 module config when client_secret contains special characters. {issue}25058[25058]
+
+*Filebeat*
+
+- Fix date parsing in GSuite/login fileset. {issue}24694[24694]
+- Improve Cisco ASA/FTD parsing of messages {pull}23766[23766]
+  - Better support for identity FW messages.
+  - Change network.bytes, source.bytes, and destination.bytes to long from integer since value can exceed integer capacity.
+  - Add descriptions for various processors for easier pipeline editing in Kibana UI. 
+- Fix usage of unallowed ECS event.outcome values in Cisco ASA/FTD pipeline. {pull}24744[24744].
+- Fix IPtables Pipeline and Ubiquiti dashboard. {issue}24878[24878] {pull}24928[24928]
+- Strip Azure Eventhub connection string in debug logs. {pulll}25066[25066]
+- Updating Oauth2 flow for m365_defender fileset. {pull}24829[24829]
+- Fix o365 module config when client_secret contains special characters. {issue}25058[25058]
+- Fix s3 input when there is a blank line in the log file. {pull}25357[25357]
+- Remove space from field `sophos.xg.trans_src_ ip`. {issue}25154[25154] {pull}25250[25250]
+- Fix `checkpoint.action_reason` when its a string, not a Long. {issue}25575[25575] {pull}25609[25609]
+- Fix `fortinet.firewall.addr` when its a string, not an IP address. {issue}25585[25585] {pull}25608[25608]
+
+*Metricbeat*
+
+- Sort correctly the keys when accessing JMX through the Jolokia module {pull}25631[25631]
+- Change lookup_fields from metricset.host to service.address {pull}15883[15883]
+- Fix incorrect types of fields GetHits and Ops in NodeInterestingStats for Couchbase module in Metricbeat {issue}21021[21021] {pull}23287[23287]
+- Fix GCP not able to request Cloudfunctions metrics if a region filter was set {pull}24218[24218]
+- Fix type of `uwsgi.status.worker.rss` type. {pull}24468[24468]
+- Accept text/plain type by default for prometheus client scraping. {pull}24622[24622]
+- Use working set bytes to calculate the pod memory limit pct when memory usage is not reported (ie. Windows pods). {pull}25428[25428]
+- Fix copy-paste error in libbeat docs. {pull}25448[25448]
+- Fix azure billing dashboard. {pull}25554[25554]
+
+*Winlogbeat*
+
+- Change `event.code` and `winlog.event_id` from int to keyword. {pull}25176[25176]
+
+==== Added
+
+*Affecting all Beats*
+
+- Add `wineventlog` schema to `decode_xml` processor. {issue}23910[23910] {pull}24726[24726]
+- Add new ECS 1.9 field `cloud.service.name` to `add_cloud_metadata` processor. {pull}24993[24993]
+- Libbeat: report queue capacity, output batch size, and output client count to monitoring. {pull}24700[24700]
+- Add kubernetes.pod.ip field in kubernetes metadata. {pull}25037[25037]
+- Discover changes in Kubernetes namespace metadata as soon as they happen. {pull}25117[25117]
+- Add `decode_xml_wineventlog` processor. {issue}23910[23910] {pull}25115[25115]
+- Add new setting `gc_percent` for tuning the garbage collector limits via configuration file. {pull}25394[25394]
+- Add `unit` and `metric_type` properties to fields.yml for populating field metadata in Elasticsearch templates {pull}25419[25419]
+- Add new option `suffix` to `logging.files` to control how log files are rotated. {pull}25464[25464]
+- Validate that required functionality in Elasticsearch is available upon initial connection. {pull}25351[25351]
+
+*Filebeat*
+
+- Support X-Forwarder-For in IIS logs. {pull}19142[192142]
+- Add support for logs generated by servers configured with `log_statement` and `log_duration` in PostgreSQL module. {pull}24607[24607]
+- Added fifteen new message IDs to Cisco ASA/FTD pipeline. {pull}24744[24744]
+- Added NTP fileset to Zeek module {pull}24224[24224]
+- Add `proxy_url` config for httpjson v2 input. {issue}24615[24615] {pull}24662[24662]
+- Change `okta.target` to `flattened` field type.  {issue}24354[24354] {pull}24636[24636]
+- Added `http.request.id` to `nginx/ingress_controller` and `elasticsearch/audit`. {pull}24994[24994]
+- Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041]
+- New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803]
+- Add `uri_parts` processor to Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules ingest pipelines. {issue}19088[19088] {pull}24699[24699]
+- New module `zookeeper` for Zookeeper service and audit logs {issue}25061[25061] {pull}25128[25128]
+- Add parsing for `haproxy.http.request.raw_request_line` field  {issue}25480[25480] {pull}25482[25482]
+- Mark `filestream` input beta. {pull}25560[25560]
+- Add User Agent Parser for Azure Sign In Logs Ingest Pipeline {pull}23201[23201]
+
+*Heartbeat*
+
+- Handle datastreams for fleet. {pull}24223[24223]
+- Add --sandbox option for browser monitor. {pull}24172[24172]
+- Support additional 'root' fields from synthetics. {pull}24770[24770]
+- Browser zip_url source type. {pull}24714[24714]
+
+*Metricbeat*
+
+- Add support for Consul 1.9. {pull}24123[24123]
+- Add support for defining metrics_filters for prometheus module in hints. {pull}24264[24264]
+- Add support for PostgreSQL 10, 11, 12 and 13. {pull}24402[24402]
+- Add support for SASL/SCRAM authentication to the Kafka module. {pull}24810[24810]
+
+*Winlogbeat*
+
+- Add support for sysmon v13 events 24 and 25. {issue}24217[24217] {pull}24945[24945]
+
 [[release-notes-7.12.1]]
 === Beats version 7.12.1
 https://github.com/elastic/beats/compare/v7.12.0...v7.12.1[View commits]