diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 1edfb0dc5866..04370ccfa6da 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -39,6 +39,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Rename `golang.heap.system.optained` field to `golang.heap.system.obtained`. {issue}5703[5703] - Support haproxy stats gathering using http (additionaly to tcp socket). {pull}5819[5819] - De dot keys in jolokia/jmx metricset to prevent collisions. {pull}5957[5957] +- Support to optionally 'de dot' keys in http/json metricset to prevent collisions. {pull}5957[5957] *Packetbeat* diff --git a/metricbeat/docs/modules/http.asciidoc b/metricbeat/docs/modules/http.asciidoc index 7a37c2f7cb17..bc106639694f 100644 --- a/metricbeat/docs/modules/http.asciidoc +++ b/metricbeat/docs/modules/http.asciidoc @@ -37,6 +37,7 @@ metricbeat.modules: #method: "GET" #request.enabled: false #response.enabled: false + #dedot.enabled: false - module: http metricsets: ["server"] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 525c17c52772..98e5215d51b6 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -219,6 +219,7 @@ metricbeat.modules: #method: "GET" #request.enabled: false #response.enabled: false + #dedot.enabled: false - module: http metricsets: ["server"] diff --git a/metricbeat/module/http/_meta/config.yml b/metricbeat/module/http/_meta/config.yml index 32be789a932a..02c717785e6a 100644 --- a/metricbeat/module/http/_meta/config.yml +++ b/metricbeat/module/http/_meta/config.yml @@ -8,6 +8,7 @@ #method: "GET" #request.enabled: false #response.enabled: false + #dedot.enabled: false - module: http metricsets: ["server"] diff --git a/metricbeat/module/http/json/_meta/docs.asciidoc b/metricbeat/module/http/json/_meta/docs.asciidoc index ac06f61cafb5..5c64c01acc3b 100644 --- a/metricbeat/module/http/json/_meta/docs.asciidoc +++ b/metricbeat/module/http/json/_meta/docs.asciidoc @@ -136,6 +136,49 @@ Example: } ---- +[float] +==== dedot.enabled +With this configuration enabled dots in json field names are replaced with `_` character, + +Example: + +[source,json] +---- +{ + "@timestamp": "2017-05-01T13:00:24.745Z", + "beat": { + "hostname": "mbp", + "name": "mbp", + "version": "6.0.0-alpha1" + }, + "http": { + "http_json_namespace": { + "date": "05-01-2017", + "milliseconds_since_epoch": 1493643625474.000000, + "time": "01:00:25 PM" + }, + "response": { + "headers": { + "Access-Control-Allow-Origin": "*", + "Content-Length": "100", + "Content-Type": "application/json; charset=ISO-8859-1", + "Date": "Mon, 01 May 2017 13:08:38 GMT", + "Server": "Google Frontend", + "X-Cloud-Trace-Context": "3f532d170112fc5b2a0b94fcbd6493b3" + }, + "code": 200 + } + }, + "metricset": { + "host": "date.jsontest.com", + "module": "http", + "name": "json", + "namespace": "http_json_namespace", + "rtt": 238397 + }, + "type": "metricsets" +} +---- [float] === Exposed fields, Dashboards, Indexes, etc. diff --git a/metricbeat/module/http/json/_meta/test/json_response_dedot.json b/metricbeat/module/http/json/_meta/test/json_response_dedot.json new file mode 100644 index 000000000000..768498198940 --- /dev/null +++ b/metricbeat/module/http/json/_meta/test/json_response_dedot.json @@ -0,0 +1,10 @@ +{ + "key_without_dot_l1": { + "key_with_dot_l2": 1, + "key_with_multiple_dots_l2": 2, + "key_without_dot_l2": { + "key_with_dot_l2": 3, + "key_with_multiple_dots_l2": 4 + } + } +} diff --git a/metricbeat/module/http/json/_meta/test/json_response_with_dots.json b/metricbeat/module/http/json/_meta/test/json_response_with_dots.json new file mode 100644 index 000000000000..4e782a461f92 --- /dev/null +++ b/metricbeat/module/http/json/_meta/test/json_response_with_dots.json @@ -0,0 +1,10 @@ +{ + "key_without_dot_l1": { + "key_with_dot_l2": 1, + "key.with.multiple.dots_l2": 2, + "key_without_dot_l2": { + "key_with_dot.l2": 3, + "key.with.multiple.dots_l2": 4 + } + } +} diff --git a/metricbeat/module/http/json/json.go b/metricbeat/module/http/json/json.go index e80936611034..421d06f1f6a0 100644 --- a/metricbeat/module/http/json/json.go +++ b/metricbeat/module/http/json/json.go @@ -50,6 +50,7 @@ type MetricSet struct { body string requestEnabled bool responseEnabled bool + deDotEnabled bool } // New create a new instance of the MetricSet @@ -64,11 +65,13 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { Body string `config:"body"` RequestEnabled bool `config:"request.enabled"` ResponseEnabled bool `config:"response.enabled"` + DeDotEnabled bool `config:"dedot.enabled"` }{ Method: "GET", Body: "", RequestEnabled: false, ResponseEnabled: false, + DeDotEnabled: false, } if err := base.Module().UnpackConfig(&config); err != nil { @@ -87,6 +90,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { http: http, requestEnabled: config.RequestEnabled, responseEnabled: config.ResponseEnabled, + deDotEnabled: config.DeDotEnabled, }, nil } @@ -101,6 +105,7 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { defer response.Body.Close() var jsonBody map[string]interface{} + var event map[string]interface{} body, err := ioutil.ReadAll(response.Body) if err != nil { @@ -112,7 +117,11 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { return nil, err } - event := jsonBody + if m.deDotEnabled { + event = replaceDots(jsonBody).(map[string]interface{}) + } else { + event = jsonBody + } if m.requestEnabled { event[mb.ModuleDataKey] = common.MapStr{ @@ -153,3 +162,16 @@ func (m *MetricSet) getHeaders(header http.Header) map[string]string { } return headers } + +func replaceDots(data interface{}) interface{} { + switch data.(type) { + case map[string]interface{}: + result := map[string]interface{}{} + for key, value := range data.(map[string]interface{}) { + result[common.DeDot(key)] = replaceDots(value) + } + return result + default: + return data + } +} diff --git a/metricbeat/module/http/json/json_test.go b/metricbeat/module/http/json/json_test.go new file mode 100644 index 000000000000..baeb56a889bf --- /dev/null +++ b/metricbeat/module/http/json/json_test.go @@ -0,0 +1,34 @@ +package json + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "encoding/json" + + "github.com/stretchr/testify/assert" +) + +func TestEventMapper(t *testing.T) { + var actualJSONBody map[string]interface{} + var expectedJSONBody map[string]interface{} + + absPath, err := filepath.Abs("./_meta/test") + assert.NotNil(t, absPath) + assert.Nil(t, err) + + actualJSONResponse, err := ioutil.ReadFile(absPath + "/json_response_with_dots.json") + assert.Nil(t, err) + err = json.Unmarshal(actualJSONResponse, &actualJSONBody) + assert.Nil(t, err) + + dedottedJSONResponse, err := ioutil.ReadFile(absPath + "/json_response_dedot.json") + assert.Nil(t, err) + err = json.Unmarshal(dedottedJSONResponse, &expectedJSONBody) + assert.Nil(t, err) + + actualJSONBody = replaceDots(actualJSONBody).(map[string]interface{}) + + assert.Equal(t, expectedJSONBody, actualJSONBody) +} diff --git a/metricbeat/modules.d/http.yml.disabled b/metricbeat/modules.d/http.yml.disabled index 32be789a932a..02c717785e6a 100644 --- a/metricbeat/modules.d/http.yml.disabled +++ b/metricbeat/modules.d/http.yml.disabled @@ -8,6 +8,7 @@ #method: "GET" #request.enabled: false #response.enabled: false + #dedot.enabled: false - module: http metricsets: ["server"]