Merge branch 'httpjson_hex_decode' of https://github.com/darshan-elas…

…tic/beats into httpjson_hex_decode

CHANGELOG-developer.next.asciidoc

@@ -63,6 +63,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Remove `common.MapStr` and use `mapstr.M` from `github.com/elastic/elastic-agent-libs` instead. {pull}31420[31420]
 - Remove `queue.Consumer`. Queues can now be read via a `Get` call directly on the queue object. {pull}31502[31502]
 - The `queue.Batch` API now provides access to individual events instead of an array. {pull}31699[31699]
+- Rename `queue.Batch.ACK()` to `queue.Batch.Done()`. {pull}31903[31903]
 
 ==== Bugfixes
 

CHANGELOG.next.asciidoc

@@ -80,7 +80,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 *Affecting all Beats*
 
 - Add new config option `timestamp.precision` to configure timestamps. {pull}31682[31682]
-
+- Improve performance of disk queue by coalescing writes. {pull}31935[31935]
 
 *Auditbeat*
 
@@ -105,6 +105,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Add template helper function for hashing strings. {issue}31613[31613] {pull}31630[31630]
 - Add extended okta.debug_context.debug_data handling. {pull}31676[31676]
 - Add `auth.oauth2.google.jwt_json` option to `httpjson` input. {pull}31750[31750]
+- Add authentication fields to RabbitMQ module documents. {issue}31159[31159] {pull}31680[31680]
 - Add template helper function for decoding hexadecimal strings. {pull}31886[31886]
 
 *Auditbeat*
@@ -129,6 +130,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Add new Kubernetes module dashboards {pull}31591[31591]
 - system/core: add cpuinfo information for Linux hosts {pull}31643[31643]
 - Enhance Oracle Module: Refactor module to properly use host parsers instead of doing its own parsing of hosts {issue}31611[31611] {pull}31692[#31692]
+- Enhance Oracle Module: New sysmetric metricset {issue}30946[30946] {pull}31462[#31462]
 
 *Packetbeat*
 

dev-tools/mage/build.go

@@ -37,6 +37,7 @@ type BuildArgs struct {
 	InputFiles  []string
 	OutputDir   string
 	CGO         bool
+	BuildMode   string // Controls `go build -buildmode`
 	Static      bool
 	Env         map[string]string
 	LDFlags     []string
@@ -61,7 +62,7 @@ func DefaultBuildArgs() BuildArgs {
 	}
 
 	if positionIndependentCodeSupported() {
-		args.ExtraFlags = append(args.ExtraFlags, "-buildmode", "pie")
+		args.BuildMode = "pie"
 	}
 
 	if DevBuild {
@@ -161,6 +162,9 @@ func Build(params BuildArgs) error {
 		"-o",
 		filepath.Join(params.OutputDir, binaryName),
 	}
+	if params.BuildMode != "" {
+		args = append(args, "-buildmode", params.BuildMode)
+	}
 	args = append(args, params.ExtraFlags...)
 
 	// ldflags

go.mod

@@ -62,7 +62,7 @@ require (
 	github.com/elastic/go-structform v0.0.9
 	github.com/elastic/go-sysinfo v1.8.0
 	github.com/elastic/go-ucfg v0.8.5
-	github.com/elastic/go-windows v1.0.1
+	github.com/elastic/go-windows v1.0.1 // indirect
 	github.com/elastic/gosigar v0.14.2
 	github.com/fatih/color v1.13.0
 	github.com/fearful-symmetry/gorapl v0.0.4
@@ -162,7 +162,7 @@ require (
 	github.com/elastic/bayeux v1.0.5
 	github.com/elastic/elastic-agent-autodiscover v0.1.1
 	github.com/elastic/elastic-agent-libs v0.2.5
-	github.com/elastic/elastic-agent-system-metrics v0.3.1
+	github.com/elastic/elastic-agent-system-metrics v0.4.1
 	github.com/shirou/gopsutil/v3 v3.21.12
 	go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0
 	go.elastic.co/apm/module/apmhttp/v2 v2.0.0

go.sum

@@ -542,8 +542,8 @@ github.com/elastic/elastic-agent-libs v0.0.0-20220303160015-5b4e674da3dd/go.mod
 github.com/elastic/elastic-agent-libs v0.2.2/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s=
 github.com/elastic/elastic-agent-libs v0.2.5 h1:8+sYCW/kkWQe5KegGLMYYT3ELXUwibMcjMh+IMr2LRE=
 github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
-github.com/elastic/elastic-agent-system-metrics v0.3.1 h1:WXdDyIaBr9zIJoyvFNzgZbcFiEnmtaKfazHPJR4DJOM=
-github.com/elastic/elastic-agent-system-metrics v0.3.1/go.mod h1:RIYhJOS7mUeyIthfOSqmmbEILYSzaDWLi5zQ70bQo+o=
+github.com/elastic/elastic-agent-system-metrics v0.4.1 h1:1bKgU0Y2F4PBLSCX2LmJbRd4wWoq5DOvXc9ysuXBVpI=
+github.com/elastic/elastic-agent-system-metrics v0.4.1/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270/go.mod h1:Msl1pdboCbArMF/nSCDUXgQuWTeoMmE/z8607X+k7ng=
 github.com/elastic/go-concert v0.2.0 h1:GAQrhRVXprnNjtvTP9pWJ1d4ToEA4cU5ci7TwTa20xg=

heartbeat/monitors/stdfields/stdfields.go

@@ -39,7 +39,11 @@ type StdMonitorFields struct {
 	Service           ServiceFields      `config:"service"`
 	Origin            string             `config:"origin"`
 	LegacyServiceName string             `config:"service_name"`
-	Enabled           bool               `config:"enabled"`
+	// Used by zip_url and local monitors
+	// kibana originating monitors only run one journey at a time
+	// and just use the `fields` syntax / manually set monitor IDs
+	IsLegacyBrowserSource bool
+	Enabled               bool `config:"enabled"`
 }
 
 func ConfigToStdMonitorFields(config *conf.C) (StdMonitorFields, error) {

heartbeat/monitors/wrappers/wrappers.go

@@ -69,38 +69,60 @@ func WrapBrowser(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.
 		js,
 		addMonitorTimespan(stdMonFields),
 		addServiceName(stdMonFields),
+		addMonitorMeta(stdMonFields, false),
 		addMonitorStatus(true),
 	)
 }
 
 // addMonitorMeta adds the id, name, and type fields to the monitor.
-func addMonitorMeta(sf stdfields.StdMonitorFields, isMulti bool) jobs.JobWrapper {
+func addMonitorMeta(sFields stdfields.StdMonitorFields, hashURLIntoID bool) jobs.JobWrapper {
 	return func(job jobs.Job) jobs.Job {
 		return func(event *beat.Event) ([]jobs.Job, error) {
 			cont, err := job(event)
 
-			id := sf.ID
-			name := sf.Name
+			id := sFields.ID
+			name := sFields.Name
 			// If multiple jobs are listed for this monitor, we can't have a single ID, so we hash the
 			// unique URLs to create unique suffixes for the monitor.
-			if isMulti {
+			if hashURLIntoID {
 				url, err := event.GetValue("url.full")
 				if err != nil {
 					logp.Error(fmt.Errorf("mandatory url.full key missing: %w", err))
 					url = "n/a"
 				}
 				urlHash, _ := hashstructure.Hash(url, nil)
-				id = fmt.Sprintf("%s-%x", sf.ID, urlHash)
+				id = fmt.Sprintf("%s-%x", sFields.ID, urlHash)
 			}
 
 			fields := mapstr.M{
-				"id":   id,
-				"name": name,
-				"type": sf.Type,
+				"type": sFields.Type,
 			}
 
-			if sf.Origin != "" {
-				fields["origin"] = sf.Origin
+			if !sFields.IsLegacyBrowserSource {
+				fields["id"] = id
+				fields["name"] = name
+			} else {
+				// Id and name differs for local
+				// - We use the monitor id and name for inline journeys ignoring the
+				//   autogenerated `inline`journey id and name.
+				// - Monitor id/name is concatenated with the journey id/name for
+				// 	 project monitors
+				journeyId, err := event.GetValue("monitor.id")
+				if err == nil {
+					fields["id"] = fmt.Sprintf("%s-%s", sFields.ID, journeyId)
+				}
+				journeyName, err := event.GetValue("monitor.name")
+				if err == nil {
+					fields["name"] = fmt.Sprintf("%s - %s", sFields.Name, journeyName)
+				}
+				fields["project"] = mapstr.M{
+					"id":   sFields.ID,
+					"name": sFields.Name,
+				}
+			}
+
+			if sFields.Origin != "" {
+				fields["origin"] = sFields.Origin
 			}
 
 			eventext.MergeEventFields(event, mapstr.M{"monitor": fields})

heartbeat/monitors/wrappers/wrappers_test.go

@@ -47,7 +47,7 @@ import (
 
 type testDef struct {
 	name         string
-	stdFields    stdfields.StdMonitorFields
+	sFields      stdfields.StdMonitorFields
 	jobs         []jobs.Job
 	want         []validator.Validator
 	metaWant     []validator.Validator
@@ -70,7 +70,7 @@ var testBrowserMonFields = stdfields.StdMonitorFields{
 
 func testCommonWrap(t *testing.T, tt testDef) {
 	t.Run(tt.name, func(t *testing.T) {
-		wrapped := WrapCommon(tt.jobs, tt.stdFields)
+		wrapped := WrapCommon(tt.jobs, tt.sFields)
 
 		core, observedLogs := observer.New(zapcore.InfoLevel)
 		logger.SetLogger(logp.NewLogger("t", zap.WrapCore(func(in zapcore.Core) zapcore.Core {
@@ -466,6 +466,10 @@ type BrowserMonitor struct {
 	id         string
 	name       string
 	checkGroup string
+	// Used for testing legacy zip_url and local monitors
+	// where the top-level id/name are used to populate monitor.project
+	legacyProjectId   string
+	legacyProjectName string
 }
 
 var inlineMonitorValues = BrowserMonitor{
@@ -482,8 +486,6 @@ func makeInlineBrowserJob(t *testing.T, u string) jobs.Job {
 			"url": URLFields(parsed),
 			"monitor": mapstr.M{
 				"type":        "browser",
-				"id":          inlineMonitorValues.id,
-				"name":        inlineMonitorValues.name,
 				"check_group": inlineMonitorValues.checkGroup,
 			},
 		})
@@ -494,10 +496,12 @@ func makeInlineBrowserJob(t *testing.T, u string) jobs.Job {
 // Browser inline jobs monitor information should not be altered
 // by the wrappers as they are handled separately in synth enricher
 func TestInlineBrowserJob(t *testing.T) {
-	fields := testBrowserMonFields
+	sFields := testBrowserMonFields
+	sFields.ID = inlineMonitorValues.id
+	sFields.Name = inlineMonitorValues.name
 	testCommonWrap(t, testDef{
 		"simple",
-		fields,
+		sFields,
 		[]jobs.Job{makeInlineBrowserJob(t, "http://foo.com")},
 		[]validator.Validator{
 			lookslike.Strict(
@@ -526,17 +530,26 @@ var projectMonitorValues = BrowserMonitor{
 	checkGroup: "journey-1-check-group",
 }
 
-func makeProjectBrowserJob(t *testing.T, u string, summary bool, projectErr error) jobs.Job {
+// Used for testing legacy zip_url / local monitorss
+var legacyProjectMonitorValues = BrowserMonitor{
+	id:                "journey-1",
+	name:              "Journey 1",
+	checkGroup:        "acheckgroup",
+	legacyProjectId:   "my-project",
+	legacyProjectName: "My Project",
+}
+
+func makeProjectBrowserJob(t *testing.T, u string, summary bool, projectErr error, bm BrowserMonitor) jobs.Job {
 	parsed, err := url.Parse(u)
 	require.NoError(t, err)
 	return func(event *beat.Event) (i []jobs.Job, e error) {
 		eventext.MergeEventFields(event, mapstr.M{
 			"url": URLFields(parsed),
 			"monitor": mapstr.M{
 				"type":        "browser",
-				"id":          projectMonitorValues.id,
-				"name":        projectMonitorValues.name,
-				"check_group": projectMonitorValues.checkGroup,
+				"id":          bm.id,
+				"name":        bm.name,
+				"check_group": bm.checkGroup,
 			},
 		})
 		if summary {
@@ -555,14 +568,19 @@ func makeProjectBrowserJob(t *testing.T, u string, summary bool, projectErr erro
 }
 
 func TestProjectBrowserJob(t *testing.T) {
-	fields := testBrowserMonFields
+	sFields := testBrowserMonFields
+	sFields.ID = projectMonitorValues.id
+	sFields.Name = projectMonitorValues.name
+	sFields.Origin = "my-origin"
 	urlStr := "http://foo.com"
 	urlU, _ := url.Parse(urlStr)
+
 	expectedMonFields := lookslike.MustCompile(map[string]interface{}{
 		"monitor": map[string]interface{}{
 			"type":        "browser",
 			"id":          projectMonitorValues.id,
 			"name":        projectMonitorValues.name,
+			"origin":      "my-origin",
 			"check_group": projectMonitorValues.checkGroup,
 			"timespan": mapstr.M{
 				"gte": hbtestllext.IsTime,
@@ -571,10 +589,11 @@ func TestProjectBrowserJob(t *testing.T) {
 		},
 		"url": URLFields(urlU),
 	})
+
 	testCommonWrap(t, testDef{
 		"simple", // has no summary fields!
-		fields,
-		[]jobs.Job{makeProjectBrowserJob(t, urlStr, false, nil)},
+		sFields,
+		[]jobs.Job{makeProjectBrowserJob(t, urlStr, false, nil, projectMonitorValues)},
 		[]validator.Validator{
 			lookslike.Strict(
 				lookslike.Compose(
@@ -586,8 +605,8 @@ func TestProjectBrowserJob(t *testing.T) {
 	})
 	testCommonWrap(t, testDef{
 		"with up summary",
-		fields,
-		[]jobs.Job{makeProjectBrowserJob(t, urlStr, true, nil)},
+		sFields,
+		[]jobs.Job{makeProjectBrowserJob(t, urlStr, true, nil, projectMonitorValues)},
 		[]validator.Validator{
 			lookslike.Strict(
 				lookslike.Compose(
@@ -603,8 +622,8 @@ func TestProjectBrowserJob(t *testing.T) {
 	})
 	testCommonWrap(t, testDef{
 		"with down summary",
-		fields,
-		[]jobs.Job{makeProjectBrowserJob(t, urlStr, true, fmt.Errorf("testerr"))},
+		sFields,
+		[]jobs.Job{makeProjectBrowserJob(t, urlStr, true, fmt.Errorf("testerr"), projectMonitorValues)},
 		[]validator.Validator{
 			lookslike.Strict(
 				lookslike.Compose(
@@ -622,4 +641,41 @@ func TestProjectBrowserJob(t *testing.T) {
 		nil,
 		nil,
 	})
+
+	legacySFields := testBrowserMonFields
+	legacySFields.ID = legacyProjectMonitorValues.legacyProjectId
+	legacySFields.Name = legacyProjectMonitorValues.legacyProjectName
+	legacySFields.IsLegacyBrowserSource = true
+
+	expectedLegacyMonFields := lookslike.MustCompile(map[string]interface{}{
+		"monitor": map[string]interface{}{
+			"type": "browser",
+			"id":   fmt.Sprintf("%s-%s", legacyProjectMonitorValues.legacyProjectId, legacyProjectMonitorValues.id),
+			"name": fmt.Sprintf("%s - %s", legacyProjectMonitorValues.legacyProjectName, legacyProjectMonitorValues.name),
+			"project": mapstr.M{
+				"id":   legacyProjectMonitorValues.legacyProjectId,
+				"name": legacyProjectMonitorValues.legacyProjectName,
+			},
+			"check_group": legacyProjectMonitorValues.checkGroup,
+			"timespan": mapstr.M{
+				"gte": hbtestllext.IsTime,
+				"lt":  hbtestllext.IsTime,
+			},
+		},
+		"url": URLFields(urlU),
+	})
+
+	testCommonWrap(t, testDef{
+		"legacy", // has no summary fields!
+		legacySFields,
+		[]jobs.Job{makeProjectBrowserJob(t, urlStr, false, nil, legacyProjectMonitorValues)},
+		[]validator.Validator{
+			lookslike.Strict(
+				lookslike.Compose(
+					urlValidator(t, urlStr),
+					expectedLegacyMonFields,
+				))},
+		nil,
+		nil,
+	})
 }

libbeat/docs/monitoring/shared-monitor-config.asciidoc

@@ -32,6 +32,10 @@ The default value is `false`.
 The {es} instances that you want to ship your {beatname_uc} metrics to. This
 configuration option contains the following fields:
 
+==== `monitoring.cluster_uuid`
+
+The `monitoring.cluster_uuid` config identifies the {es} cluster under which the monitoring data will appear in the Stack Monitoring UI. 
+
 ===== `api_key`
 
 The detail of the API key to be used to send monitoring information to {es}.

libbeat/outputs/kafka/docs/kafka.asciidoc

@@ -62,11 +62,11 @@ The cluster metadata contain the actual Kafka brokers events are published to.
 
 ===== `version`
 
-Kafka protocol version that {beatname_lc} will request when connecting. Defaults to 1.0.0.
+Kafka protocol version that {beatname_uc} will request when connecting. Defaults to 1.0.0.
 
 Valid values are all kafka releases in between `0.8.2.0` and `2.6.0`.
 
-The protocol version controls the Kafka client features available to {beatname_lc}, it does not prevent {beatname_lc} from connecting to Kafka versions newer than the protocol version.
+The protocol version controls the Kafka client features available to {beatname_uc}; it does not prevent {beatname_uc} from connecting to Kafka versions newer than the protocol version.
 
 See <<kafka-compatibility>> for information on supported versions.