Apache status: adjust field collection in fleet mode (elastic#22821)

* apache status: adjust collected fields in fleet mode

* Fix: mage check

* Update CHANGELOG

* nit-pick

* Address PR comments

(cherry picked from commit 0c1e54c)

CHANGELOG.next.asciidoc

@@ -614,6 +614,18 @@ same journal. {pull}18467[18467]
 - `kibana` module: `stats` metricset no-longer collects usage-related data. {pull}22732[22732]
 - Add more TCP states to Metricbeat system socket_summary. {pull}14347[14347]
 - Add io.ops in fields exported by system.diskio. {pull}22066[22066]
+- Adjust the Apache status fields in the fleet mode. {pull}22821[22821]
+
+*Packetbeat*
+
+- Add an example to packetbeat.yml of using the `forwarded` tag to disable
+  `host` metadata fields when processing network data from network tap or mirror
+  port. {pull}19209[19209]
+- Add ECS fields for x509 certs, event categorization, and related IP info. {pull}19167[19167]
+- Add 100-continue support {issue}15830[15830] {pull}19349[19349]
+- Add initial SIP protocol support {pull}21221[21221]
+- Add support for overriding the published index on a per-protocol/flow basis. {pull}22134[22134]
+- Change build process for x-pack distribution {pull}21979[21979]
 
 *Packetbeat*
 

libbeat/common/fleetmode/fleet_mode.go

@@ -0,0 +1,53 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package fleetmode
+
+import (
+	"flag"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+)
+
+// Enabled checks to see if filebeat/metricbeat is running under Agent
+// The management setting is stored in the main Beat runtime object, but we can't see that from a module
+// So instead we check the CLI flags, since Agent starts filebeat/metricbeat with "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true"
+func Enabled() bool {
+	type management struct {
+		Mode    string `config:"management.mode"`
+		Enabled bool   `config:"management.enabled"`
+	}
+	var managementSettings management
+
+	cfgFlag := flag.Lookup("E")
+	if cfgFlag == nil {
+		return false
+	}
+
+	cfgObject, _ := cfgFlag.Value.(*common.SettingsFlag)
+	cliCfg := cfgObject.Config()
+
+	err := cliCfg.Unpack(&managementSettings)
+	if err != nil {
+		return false
+	}
+
+	if managementSettings.Enabled == true && managementSettings.Mode == "x-pack-fleet" {
+		return true
+	}
+	return false
+}

metricbeat/module/apache/status/status.go

@@ -21,6 +21,7 @@ package status
 import (
 	"github.com/pkg/errors"
 
+	"github.com/elastic/beats/v7/libbeat/common/fleetmode"
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/metricbeat/helper"
 	"github.com/elastic/beats/v7/metricbeat/mb"
@@ -63,6 +64,8 @@ func init() {
 type MetricSet struct {
 	mb.BaseMetricSet
 	http *helper.HTTP
+
+	isFleetMode bool
 }
 
 // New creates new instance of MetricSet.
@@ -74,6 +77,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	return &MetricSet{
 		base,
 		http,
+		fleetmode.Enabled(),
 	}, nil
 }
 
@@ -87,10 +91,25 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error {
 	}
 
 	data, _ := eventMapping(scanner, m.Host())
+	event := mb.Event{
+		MetricSetFields: data,
+	}
 
-	if reported := reporter.Event(mb.Event{MetricSetFields: data}); !reported {
-		m.Logger().Error("error reporting event")
+	if m.isFleetMode {
+		event = adjustFleetEvent(event)
 	}
 
+	if reported := reporter.Event(event); !reported {
+		m.Logger().Error("error reporting event")
+	}
 	return nil
 }
+
+func adjustFleetEvent(event mb.Event) mb.Event {
+	var adjusted mb.Event
+	adjusted.MetricSetFields = event.MetricSetFields.Clone()
+
+	// Remove apache.hostname
+	adjusted.MetricSetFields.Delete("hostname")
+	return adjusted
+}

metricbeat/module/apache/status/status_integration_test.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
 )
@@ -47,6 +48,30 @@ func TestFetch(t *testing.T) {
 	}
 }
 
+func TestFetchFleetMode(t *testing.T) {
+	service := compose.EnsureUp(t, "apache")
+
+	f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host()))
+	f.(*MetricSet).isFleetMode = true // silently simulate running in the fleet mode
+
+	events, errs := mbtest.ReportingFetchV2Error(f)
+	if len(errs) > 0 {
+		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
+	}
+	assert.NotEmpty(t, events)
+	event := events[0]
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check number of fields.
+	if len(event.MetricSetFields) < 11 {
+		t.Fatal("Too few top-level elements in the event")
+	}
+
+	_, err := event.MetricSetFields.GetValue("hostname")
+	assert.Equal(t, common.ErrKeyNotFound, err, "apache.hostname shouldn't be present in the fleet mode")
+}
+
 func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "apache",

metricbeat/module/system/system.go

@@ -21,7 +21,7 @@ import (
 	"flag"
 	"sync"
 
-	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/common/fleetmode"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
@@ -53,34 +53,5 @@ func NewModule(base mb.BaseModule) (mb.Module, error) {
 		initModule()
 	})
 
-	return &Module{BaseModule: base, HostFS: *HostFS, IsAgent: checkMgmtFlags()}, nil
-}
-
-// checkMgmtFlags checks to see if metricbeat is running under Agent
-// The management setting is stored in the main Beat runtime object, but we can't see that from a module
-// So instead we check the CLI flags, since Agent starts metricbeat with "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true"
-func checkMgmtFlags() bool {
-	type management struct {
-		Mode    string `config:"management.mode"`
-		Enabled bool   `config:"management.enabled"`
-	}
-	var managementSettings management
-
-	cfgFlag := flag.Lookup("E")
-	if cfgFlag == nil {
-		return false
-	}
-
-	CfgObject, _ := cfgFlag.Value.(*common.SettingsFlag)
-	cliCfg := CfgObject.Config()
-
-	err := cliCfg.Unpack(&managementSettings)
-	if err != nil {
-		return false
-	}
-
-	if managementSettings.Enabled == true && managementSettings.Mode == "x-pack-fleet" {
-		return true
-	}
-	return false
+	return &Module{BaseModule: base, HostFS: *HostFS, IsAgent: fleetmode.Enabled()}, nil
 }