Merge branch 'main' into main

CHANGELOG.asciidoc

@@ -3,6 +3,63 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.6.2]]
+=== Beats version 8.6.2
+https://github.com/elastic/beats/compare/v8.6.1\...v8.6.2[View commits]
+
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix lockfile logic, retry locking {pull}34194[34194]
+- Log errors from the Elastic Agent V2 client errors channel. Avoids blocking when error occurs communicating with the Elastic Agent. {pull}34392[34392]
+- Only log publish event messages in trace log level under elastic-agent. {pull}34391[34391]
+- Fix issue where updating a single Elastic Agent configuration unit results in other units being turned off. {pull}34504[34504]
+
+*Auditbeat*
+
+*Filebeat*
+- [Azure blob storage] Changed logger field name from `container` to `container_name` so that it does not clash
+- [GCS] Added support for more mime types & introduced offset tracking via cursor state. Also added support for
+- [httpsjon] Improved error handling during pagination with chaining & split processor {pull}34127[34127]
+- Fix EOF on single line not producing any event. {issue}30436[30436] {pull}33568[33568]
+- Fix handling of error in states in direct aws-s3 listing input {issue}33513[33513] {pull}33722[33722]
+- Fix `httpjson` input page number initialization and documentation. {pull}33400[33400]
+- Fix reporting of `filebeat.events.active` in log events such that the current value is always reported instead of the difference from the last value. {pull}33597[33597]
+- Fix splitting array of strings/arrays in httpjson input {issue}30345[30345] {pull}33609[33609]
+- Fix Google workspace pagination and document ID generation. {pull}33666[33666]
+- Update mito CEL extension library to v0.0.0-20221207004749-2f0f2875e464 {pull}33974[33974]
+- [azure-eventhub input] Switch the run EPH run mode to non-blocking {pull}34075[34075]
+- Fixing system tests not returning expected content encoding for azure blob storage input. {pull}34412[34412]
+- [Azure Logs] Fix authentication_processing_details parsing in sign-in logs. {issue}34330[34330] {pull}34478[34478]
+
+*Heartbeat*
+- Fix broken zip URL monitors. NOTE: Zip URL Monitors will be removed in version 8.7 and replaced with project monitors. {pull}33723[33723]
+- Fix bug where states.duration_ms was incorrect type. {pull}33563[33563]
+- Fix handling of long UDP messages in UDP input. {issue}33836[33836] {pull}33837[33837]
+- Fix beat capabilities on Docker image. {pull}33584[33584]
+- Fix serialization of state duration to avoid scientific notation. {pull}34280[34280]
+- Enable nodejs engine strict validation when bundling synthetics. {pull}34470[34470]
+
+*Metricbeat*
+
+- Fix metrics split through different events and metadata not matching for aws cloudwatch. {pull}34483[34483]
+- Fix metadata enricher with correct container ids for pods with multiple containers in container metricset. Align `kubernetes.container.id` and `container.id` fields for state_container metricset. {pull}34516[34516]
+
+*Packetbeat*
+
+- Fixed a race condition in Packetbeat that could cause crashes or instability {pull}34514[34514]
+
+
+==== Added
+
+
+*Filebeat*
+
+- Added support for HTTP destination override to Google Cloud Storage input. {pull}34413[34413]
+
+
 [[release-notes-8.6.1]]
 === Beats version 8.6.1
 https://github.com/elastic/beats/compare/v8.6.0\...v8.6.1[View commits]

CHANGELOG.next.asciidoc

@@ -53,6 +53,11 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Log errors from the Elastic Agent V2 client errors channel. Avoids blocking when error occurs communicating with the Elastic Agent. {pull}34392[34392]
 - Only log publish event messages in trace log level under elastic-agent. {pull}34391[34391]
 - Fix issue where updating a single Elastic Agent configuration unit results in other units being turned off. {pull}34504[34504]
+- Fix dropped events when monitor a beat under the agent and send its `Host info` log entry. {pull}34599[34599]
+
+- Fix namespacing on self-monitoring {pull}32336[32336]
+- Fix race condition when stopping runners {pull}32433[32433]
+- Fix concurrent map writes when system/process code called from reporter code {pull}32491[32491]
 
 *Auditbeat*
 
@@ -97,6 +102,10 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Fix beat capabilities on Docker image. {pull}33584[33584]
 - Fix serialization of state duration to avoid scientific notation. {pull}34280[34280]
 - Enable nodejs engine strict validation when bundling synthetics. {pull}34470[34470]
+with the ecs field name `container`. {pull}34403[34403]
+automatic splitting at root level, if root level element is an array. {pull}34155[34155]
+
+*Heartbeat*
 
 
 *Heartbeat*
@@ -139,7 +148,6 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 
 *Packetbeat*
 
-- Fixed a race condition in Packetbeat that could cause crashes or instability {pull}34514[34514]
 
 
 *Winlogbeat*
@@ -280,3 +288,6 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 
 
 
+
+
+

libbeat/cmd/instance/beat.go

@@ -28,6 +28,7 @@ import (
 	"math"
 	"math/big"
 	"math/rand"
+	"net"
 	"os"
 	"os/user"
 	"runtime"
@@ -1108,7 +1109,9 @@ func logSystemInfo(info beat.Info) {
 
 	// Host
 	if host, err := sysinfo.Host(); err == nil {
-		log.Infow("Host info", "host", host.Info())
+		hostInfo := host.Info()
+		hostInfo.IPs = sanitizeIPs(hostInfo.IPs)
+		log.Infow("Host info", "host", hostInfo)
 	}
 
 	// Process
@@ -1221,3 +1224,25 @@ func initPaths(cfg *config.C) error {
 	}
 	return nil
 }
+
+// every IP address received from `Info()` has a netmask suffix
+// which makes every IP address invalid from the validation perspective.
+// If this log entry is ingested to a data stream as it is, the event will be dropped.
+// We must make sure every address is valid and does not have suffixes
+func sanitizeIPs(ips []string) []string {
+	validIPs := make([]string, 0, len(ips))
+	for _, ip := range ips {
+		if ip == "" {
+			continue
+		}
+		trimIndex := strings.LastIndexByte(ip, '/')
+		if trimIndex != -1 {
+			ip = ip[:trimIndex]
+		}
+		if net.ParseIP(ip) == nil {
+			continue
+		}
+		validIPs = append(validIPs, ip)
+	}
+	return validIPs
+}

libbeat/cmd/instance/beat_test.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/gofrs/uuid"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestNewInstance(t *testing.T) {
@@ -155,8 +156,71 @@ func TestMetaJsonWithTimestamp(t *testing.T) {
 		panic(err)
 	}
 	assert.False(t, firstStart.Equal(secondBeat.Info.FirstStart), "Before meta.json is loaded, first start must be different")
-	secondBeat.loadMeta(metaPath)
+	err = secondBeat.loadMeta(metaPath)
+	require.NoError(t, err)
 
 	assert.Equal(t, nil, err, "Unable to load meta file properly")
 	assert.True(t, firstStart.Equal(secondBeat.Info.FirstStart), "Cannot load first start")
 }
+
+func TestSanitizeIPs(t *testing.T) {
+	cases := []struct {
+		name        string
+		ips         []string
+		expectedIPs []string
+	}{
+		{
+			name: "does not change valid IPs",
+			ips: []string{
+				"127.0.0.1",
+				"::1",
+				"fe80::1",
+				"fe80::6ca6:cdff:fe6a:4f59",
+				"192.168.1.101",
+			},
+			expectedIPs: []string{
+				"127.0.0.1",
+				"::1",
+				"fe80::1",
+				"fe80::6ca6:cdff:fe6a:4f59",
+				"192.168.1.101",
+			},
+		},
+		{
+			name: "cuts the masks",
+			ips: []string{
+				"127.0.0.1/8",
+				"::1/128",
+				"fe80::1/64",
+				"fe80::6ca6:cdff:fe6a:4f59/64",
+				"192.168.1.101/24",
+			},
+			expectedIPs: []string{
+				"127.0.0.1",
+				"::1",
+				"fe80::1",
+				"fe80::6ca6:cdff:fe6a:4f59",
+				"192.168.1.101",
+			},
+		},
+		{
+			name: "excludes invalid IPs",
+			ips: []string{
+				"",
+				"fe80::6ca6:cdff:fe6a:4f59",
+				"invalid",
+				"192.168.1.101",
+			},
+			expectedIPs: []string{
+				"fe80::6ca6:cdff:fe6a:4f59",
+				"192.168.1.101",
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.expectedIPs, sanitizeIPs(tc.ips))
+		})
+	}
+}

libbeat/docs/release.asciidoc

@@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read
 <<breaking-changes>> for more detail about changes that affect
 upgrade.
 
+* <<release-notes-8.6.2>>
 * <<release-notes-8.6.1>>
 * <<release-notes-8.6.0>>
 * <<release-notes-8.5.3>>

metricbeat/docs/modules/kubernetes/state_persistentvolumeclaim.asciidoc

@@ -17,4 +17,10 @@ include::../../../module/kubernetes/state_persistentvolumeclaim/_meta/docs.ascii
 For a description of each field in the metricset, see the
 <<exported-fields-kubernetes,exported fields>> section.
 
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/kubernetes/state_persistentvolumeclaim/_meta/data.json[]
+----
 :edit_url!:

metricbeat/module/kubernetes/_meta/test/README.md

@@ -0,0 +1,18 @@
+# Kube State Metrics metrics files
+
+Each KSM metrics files has the **name format `ksm.v<version>.plain`**. **The `<version>` should be compatible with the Kubernetes versions we support**. Check the compatibility in the official repository [here](https://github.com/kubernetes/kube-state-metrics#compatibility-matrix).
+
+**These files are being used in the metricsets that fetch these metrics**: all the `state_*` ones. As of the time of this commit (21.feb.2023), there are two folders that are in use for the `test` file present in each metricset: `test` and `testdata`. Both **these folders require these KSM metrics files** to generate the expected ones. You can check the test file by running `go test -data` to generate the expected files, or simply `go test .` to check against the already present generated files.
+
+> **_NOTE:_**  The expected files inside these two folders are not deleted when running the tests. Remember to delete them if they are from an old version.
+
+**The name of these files needs to end with the suffix `.plain`**. The reason for this is that the `config.yml` file inside each `state_*` metricset `testdata` requires the suffix `.plain` for the metrics files:
+
+```yaml
+type: http
+url: "/metrics"
+suffix: plain
+path: "../_meta/test"
+```
+
+Since all metricsets use a `data.json` file for the documentation - this file contains an example of the metrics fields -, **it is necessary that one of these files has the name `docs.plain`**. This file should have the **same content as one of the metrics file of a KSM version** we support. This means that one of the files is duplicated, for example: `ksm.v2.7.0.plain` has the same content as `docs.plain`. This is not a mistake, as having the `ksm.v2.7.0.plain` tells us the metrics of that specific version.