Merge branch 'master' into anomali_threatstream

.ci/bump-go-release-version.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Given the Golang release version this script will bump the version.
+#
+# This script is executed by the automation we are putting in place
+# and it requires the git add/commit commands.
+#
+# Parameters:
+#	$1 -> the Golang release version to be bumped. Mandatory.
+#
+set -euo pipefail
+MSG="parameter missing."
+GO_RELEASE_VERSION=${1:?$MSG}
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+
+if [ "${OS}" == "darwin" ] ; then
+	SED="sed -i .bck"
+else
+	SED="sed -i"
+fi
+
+echo "Update go version ${GO_RELEASE_VERSION}"
+echo "${GO_RELEASE_VERSION}" > .go-version
+git add .go-version
+
+find . -maxdepth 3 -name Dockerfile -print0 |
+    while IFS= read -r -d '' line; do
+        ${SED} -E -e "s#(FROM golang):[0-9]+\.[0-9]+\.[0-9]+#\1:${GO_RELEASE_VERSION}#g" "$line"
+        ${SED} -E -e "s#(ARG GO_VERSION)=[0-9]+\.[0-9]+\.[0-9]+#\1=${GO_RELEASE_VERSION}#g" "$line"
+        git add "${line}"
+    done
+
+${SED} -E -e "s#(:go-version:) [0-9]+\.[0-9]+\.[0-9]+#\1 ${GO_RELEASE_VERSION}#g" libbeat/docs/version.asciidoc
+git add libbeat/docs/version.asciidoc
+
+git diff --staged --quiet || git commit -m "[Automation] Update go release version to ${GO_RELEASE_VERSION}"
+git --no-pager log -1
+
+echo "You can now push and create a Pull Request"

.ci/packaging.groovy

@@ -212,7 +212,6 @@ pipeline {
                   'x-pack/filebeat',
                   'x-pack/heartbeat',
                   'x-pack/metricbeat',
-                  'x-pack/osquerybeat',
                   'x-pack/packetbeat'
                 )
               }
@@ -426,11 +425,13 @@ def triggerE2ETests(String suite) {
 
   def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
   def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
+  def beatVersion = "${env.BEAT_VERSION}-SNAPSHOT"
 
   def parameters = [
     booleanParam(name: 'forceSkipGitChecks', value: true),
     booleanParam(name: 'forceSkipPresubmit', value: true),
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+    booleanParam(name: 'BEAT_VERSION', value: beatVersion),
     booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
     string(name: 'runTestsSuites', value: suite),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),

.gitignore

@@ -51,5 +51,5 @@ x-pack/elastic-agent/pkg/agent/operation/tests/scripts/serviceable-1.0-darwin-x8
 *.terraform
 *.tfstate*
 
-# Files generated with the bump elastic stack version automation
-testing/environments/*.bck
+# Files generated with the bump version automations
+*.bck

.mergify.yml

@@ -84,11 +84,25 @@ pull_request_rules:
       merge:
         method: squash
         strict: smart+fasttrack
-  - name: delete upstream branch after merging changes on testing/environments/snapshot*
+  - name: delete upstream branch after merging changes on testing/environments/snapshot* or it's closed
     conditions:
-      - merged
-      - label=automation
-      - head~=^update-stack-version
-      - files~=^testing/environments/snapshot.*\.yml$
+      - or:
+        - merged
+        - closed
+      - and:
+        - label=automation
+        - head~=^update-stack-version
+        - files~=^testing/environments/snapshot.*\.yml$
+    actions:
+      delete_head_branch:
+  - name: delete upstream branch after merging changes on .go-version or it's closed
+    conditions:
+      - or:
+        - merged
+        - closed
+      - and:
+        - label=automation
+        - head~=^update-go-version
+        - files~=^.go-version$
     actions:
       delete_head_branch:

CHANGELOG.next.asciidoc

@@ -275,6 +275,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix issue with m365_defender, when parsing incidents that has no alerts attached: {pull}25421[25421]
 - Fix default config template values for paths on oracle module: {pull}26276[26276]
 - Fix bug in aws-s3 input where the end of gzipped log files might have been discarded. {pull}26260[26260]
+- Fix bug in `httpjson` that prevented `first_event` getting updated. {pull}26407[26407]
 
 *Filebeat*
 
@@ -818,10 +819,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Make `filestream` input GA. {pull}26127[26127]
 - Add new `parser` to `filestream` input: `container`. {pull}26115[26115]
 - Add support for ISO8601 timestamps in Zeek fileset {pull}25564[25564]
+- Add possibility to include headers in resulting docs and preserve the original event in http_endpoint input {pull}26279[26279]
 - Add `preserve_original_event` option to `o365audit` input. {pull}26273[26273]
 - Add `log.flags` to events created by the `aws-s3` input. {pull}26267[26267]
 - Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267]
 - RFC 5424 and UNIX socket support in the Syslog input are now GA {pull}26293[26293]
+- Update grok patterns for HA Proxy module {issue}25827[25827] {pull}25835[25835]
 - Added dataset `anomalithreatstream` to the `threatintel` module to ingest indicators from Anomali ThreatStream {pull}26350[26350]
 
 *Heartbeat*
@@ -1012,6 +1015,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Deprecate the MISP module. The Threat Intel module should be used instead. {issue}25240[25240]
 
 *Heartbeat*
 

Jenkinsfile

@@ -520,6 +520,7 @@ def e2e(Map args = [:]) {
     def goVersionForE2E = readFile('.go-version').trim()
     withEnv(["GO_VERSION=${goVersionForE2E}",
               "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}",
+              "BEAT_VERSION=${env.VERSION}-SNAPSHOT",
               "LOG_LEVEL=TRACE"]) {
       def status = 0
       filebeat(output: dockerLogFile){

deploy/kubernetes/Makefile

@@ -1,4 +1,4 @@
-ALL=filebeat metricbeat auditbeat heartbeat elastic-agent-standalone elastic-agent
+ALL=filebeat metricbeat auditbeat heartbeat elastic-agent-standalone
 BEAT_VERSION=$(shell head -n 1 ../../libbeat/docs/version.asciidoc | cut -c 17- )
 
 .PHONY: all $(ALL)