[Metricbeat] Add dedot to aws ec2 metricset and cloudwatch metricset (#…

…15844) (#15894) * Add dedot to aws ec2 metricset and cloudwatch metricset * Remove dedot config parameter (cherry picked from commit 387bae0)
elastic · Jan 28, 2020 · e0d629c · e0d629c
1 parent 7056e42
commit e0d629c
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 5 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -109,9 +109,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Make `logstash` module more resilient to Logstash unavailability. {issue}15276[15276] {pull}15306[15306]
 - Add username/password in Metricbeat autodiscover hints {pull}15349[15349]
 - Fix CPU count in docker/cpu in cases where no `online_cpus` are reported {pull}15070[15070]
+- Add dedot for tags in ec2 metricset and cloudwatch metricset. {issue}15843[15843] {pull}15844[15844]
 - Use RFC3339 format for timestamps collected using the SQL module. {pull}15847[15847]
 
-
 *Packetbeat*
 
 

diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go
@@ -17,6 +17,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
 	"github.com/pkg/errors"
 
+	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/metricbeat/mb"
 	"github.com/elastic/beats/x-pack/metricbeat/module/aws"
 )
@@ -514,10 +515,11 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes
 						events[identifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID)
 					}
 					events[identifierValue] = insertRootFields(events[identifierValue], output.Values[timestampIdx], labels)
+
+					// By default, replace dot "." using under bar "_" for tag keys and values
 					for _, tag := range tags {
-						events[identifierValue].RootFields.Put("aws.tags."+*tag.Key, *tag.Value)
+						events[identifierValue].RootFields.Put("aws.tags."+common.DeDot(*tag.Key), common.DeDot(*tag.Value))
 					}
-
 				}
 			}
 		}

diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go
@@ -201,8 +201,9 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met
 					}
 				}
 
+				// By default, replace dot "." using under bar "_" for tag keys and values
 				for _, tag := range tags {
-					events[instanceID].ModuleFields.Put("tags."+*tag.Key, *tag.Value)
+					events[instanceID].ModuleFields.Put("tags."+common.DeDot(*tag.Key), common.DeDot(*tag.Value))
 				}
 
 				machineType, err := instanceOutput[instanceID].InstanceType.MarshalValue()

diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_test.go
@@ -73,6 +73,17 @@ func (m *MockEC2Client) DescribeInstancesRequest(input *ec2.DescribeInstancesInp
 	privateDNSName := "ip-5-6-7-8.us-west-1.compute.internal"
 	privateIP := "5.6.7.8"
 
+	tags := []ec2.Tag{
+		{
+			Key:   awssdk.String("app.kubernetes.io/name"),
+			Value: awssdk.String("foo"),
+		},
+		{
+			Key:   awssdk.String("helm.sh/chart"),
+			Value: awssdk.String("foo-chart"),
+		},
+	}
+
 	instance := ec2.Instance{
 		InstanceId:   awssdk.String(instanceID),
 		InstanceType: ec2.InstanceTypeT2Medium,
@@ -95,6 +106,7 @@ func (m *MockEC2Client) DescribeInstancesRequest(input *ec2.DescribeInstancesInp
 		PublicIpAddress:  &publicIP,
 		PrivateDnsName:   &privateDNSName,
 		PrivateIpAddress: &privateIP,
+		Tags:             tags,
 	}
 
 	httpReq, _ := http.NewRequest("", "", nil)
@@ -126,7 +138,7 @@ func TestGetInstanceIDs(t *testing.T) {
 	assert.Equal(t, awssdk.String("us-west-1a"), instancesOutputs[instanceID].Placement.AvailabilityZone)
 }
 
-func TestCreateCloudWatchEvents(t *testing.T) {
+func TestCreateCloudWatchEventsDedotTags(t *testing.T) {
 	expectedEvent := mb.Event{
 		RootFields: common.MapStr{
 			"cloud": common.MapStr{
@@ -156,6 +168,10 @@ func TestCreateCloudWatchEvents(t *testing.T) {
 					"ip":       "5.6.7.8",
 				},
 			},
+			"tags": common.MapStr{
+				"app_kubernetes_io/name": "foo",
+				"helm_sh/chart":          "foo-chart",
+			},
 		},
 	}
 	svcEC2Mock := &MockEC2Client{}
@@ -203,6 +219,7 @@ func TestCreateCloudWatchEvents(t *testing.T) {
 	assert.Equal(t, expectedEvent.RootFields, events[instanceID].RootFields)
 	assert.Equal(t, expectedEvent.MetricSetFields["cpu"], events[instanceID].MetricSetFields["cpu"])
 	assert.Equal(t, expectedEvent.MetricSetFields["instance"], events[instanceID].MetricSetFields["instance"])
+	assert.Equal(t, expectedEvent.MetricSetFields["tags"], events[instanceID].ModuleFields["tags"])
 }
 
 func TestConstructMetricQueries(t *testing.T) {