i5942: http/json module: Allow for modifying json field names with do…

…ts in them in beats (#5970)

CHANGELOG.asciidoc

@@ -40,6 +40,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Rename `golang.heap.system.optained` field to `golang.heap.system.obtained`. {issue}5703[5703]
 - Support haproxy stats gathering using http (additionaly to tcp socket). {pull}5819[5819]
 - De dot keys in jolokia/jmx metricset to prevent collisions. {pull}5957[5957]
+- Support to optionally 'de dot' keys in http/json metricset to prevent collisions. {pull}5957[5957]
 
 *Packetbeat*
 

metricbeat/docs/modules/http.asciidoc

@@ -37,6 +37,7 @@ metricbeat.modules:
   #method: "GET"
   #request.enabled: false
   #response.enabled: false
+  #dedot.enabled: false
 
 - module: http
   metricsets: ["server"]

metricbeat/metricbeat.reference.yml

@@ -219,6 +219,7 @@ metricbeat.modules:
   #method: "GET"
   #request.enabled: false
   #response.enabled: false
+  #dedot.enabled: false
 
 - module: http
   metricsets: ["server"]

metricbeat/module/http/_meta/config.yml

@@ -8,6 +8,7 @@
   #method: "GET"
   #request.enabled: false
   #response.enabled: false
+  #dedot.enabled: false
 
 - module: http
   metricsets: ["server"]

metricbeat/module/http/json/_meta/docs.asciidoc

@@ -136,6 +136,49 @@ Example:
 }
 ----
 
+[float]
+==== dedot.enabled
+With this configuration enabled dots in json field names  are replaced with `_` character,
+
+Example:
+
+[source,json]
+----
+{
+  "@timestamp": "2017-05-01T13:00:24.745Z",
+  "beat": {
+    "hostname": "mbp",
+    "name": "mbp",
+    "version": "6.0.0-alpha1"
+  },
+  "http": {
+    "http_json_namespace": {
+      "date": "05-01-2017",
+      "milliseconds_since_epoch": 1493643625474.000000,
+      "time": "01:00:25 PM"
+    },
+    "response": {
+      "headers": {
+        "Access-Control-Allow-Origin": "*",
+        "Content-Length": "100",
+        "Content-Type": "application/json; charset=ISO-8859-1",
+        "Date": "Mon, 01 May 2017 13:08:38 GMT",
+        "Server": "Google Frontend",
+        "X-Cloud-Trace-Context": "3f532d170112fc5b2a0b94fcbd6493b3"
+      },
+      "code": 200
+    }
+  },
+  "metricset": {
+    "host": "date.jsontest.com",
+    "module": "http",
+    "name": "json",
+    "namespace": "http_json_namespace",
+    "rtt": 238397
+  },
+  "type": "metricsets"
+}
+----
 
 [float]
 === Exposed fields, Dashboards, Indexes, etc.

metricbeat/module/http/json/_meta/test/json_response_dedot.json

@@ -0,0 +1,10 @@
+{
+    "key_without_dot_l1": {
+        "key_with_dot_l2": 1,
+        "key_with_multiple_dots_l2": 2,
+        "key_without_dot_l2": {
+            "key_with_dot_l2": 3,
+            "key_with_multiple_dots_l2": 4
+        }
+    }
+}

metricbeat/module/http/json/_meta/test/json_response_with_dots.json

@@ -0,0 +1,10 @@
+{
+    "key_without_dot_l1": {
+        "key_with_dot_l2": 1,
+        "key.with.multiple.dots_l2": 2,
+        "key_without_dot_l2": {
+            "key_with_dot.l2": 3,
+            "key.with.multiple.dots_l2": 4
+        }
+    }
+}

metricbeat/module/http/json/json.go

@@ -50,6 +50,7 @@ type MetricSet struct {
 	body            string
 	requestEnabled  bool
 	responseEnabled bool
+	deDotEnabled    bool
 }
 
 // New create a new instance of the MetricSet
@@ -64,11 +65,13 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 		Body            string `config:"body"`
 		RequestEnabled  bool   `config:"request.enabled"`
 		ResponseEnabled bool   `config:"response.enabled"`
+		DeDotEnabled    bool   `config:"dedot.enabled"`
 	}{
 		Method:          "GET",
 		Body:            "",
 		RequestEnabled:  false,
 		ResponseEnabled: false,
+		DeDotEnabled:    false,
 	}
 
 	if err := base.Module().UnpackConfig(&config); err != nil {
@@ -87,6 +90,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 		http:            http,
 		requestEnabled:  config.RequestEnabled,
 		responseEnabled: config.ResponseEnabled,
+		deDotEnabled:    config.DeDotEnabled,
 	}, nil
 }
 
@@ -101,6 +105,7 @@ func (m *MetricSet) Fetch() (common.MapStr, error) {
 	defer response.Body.Close()
 
 	var jsonBody map[string]interface{}
+	var event map[string]interface{}
 
 	body, err := ioutil.ReadAll(response.Body)
 	if err != nil {
@@ -112,7 +117,11 @@ func (m *MetricSet) Fetch() (common.MapStr, error) {
 		return nil, err
 	}
 
-	event := jsonBody
+	if m.deDotEnabled {
+		event = replaceDots(jsonBody).(map[string]interface{})
+	} else {
+		event = jsonBody
+	}
 
 	if m.requestEnabled {
 		event[mb.ModuleDataKey] = common.MapStr{
@@ -153,3 +162,16 @@ func (m *MetricSet) getHeaders(header http.Header) map[string]string {
 	}
 	return headers
 }
+
+func replaceDots(data interface{}) interface{} {
+	switch data.(type) {
+	case map[string]interface{}:
+		result := map[string]interface{}{}
+		for key, value := range data.(map[string]interface{}) {
+			result[common.DeDot(key)] = replaceDots(value)
+		}
+		return result
+	default:
+		return data
+	}
+}

metricbeat/module/http/json/json_test.go

@@ -0,0 +1,34 @@
+package json
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+
+	"encoding/json"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEventMapper(t *testing.T) {
+	var actualJSONBody map[string]interface{}
+	var expectedJSONBody map[string]interface{}
+
+	absPath, err := filepath.Abs("./_meta/test")
+	assert.NotNil(t, absPath)
+	assert.Nil(t, err)
+
+	actualJSONResponse, err := ioutil.ReadFile(absPath + "/json_response_with_dots.json")
+	assert.Nil(t, err)
+	err = json.Unmarshal(actualJSONResponse, &actualJSONBody)
+	assert.Nil(t, err)
+
+	dedottedJSONResponse, err := ioutil.ReadFile(absPath + "/json_response_dedot.json")
+	assert.Nil(t, err)
+	err = json.Unmarshal(dedottedJSONResponse, &expectedJSONBody)
+	assert.Nil(t, err)
+
+	actualJSONBody = replaceDots(actualJSONBody).(map[string]interface{})
+
+	assert.Equal(t, expectedJSONBody, actualJSONBody)
+}

metricbeat/modules.d/http.yml.disabled

@@ -8,6 +8,7 @@
   #method: "GET"
   #request.enabled: false
   #response.enabled: false
+  #dedot.enabled: false
 
 - module: http
   metricsets: ["server"]