From 6a831decad7ac72660b120ba7824ff4407c547fb Mon Sep 17 00:00:00 2001 From: Berin Smaldon Date: Thu, 15 Mar 2018 13:29:58 +0000 Subject: [PATCH] adds the lowercase processor for converting field and tag values to lower case --- plugins/processors/all/all.go | 1 + plugins/processors/lowercase/README.md | 32 +++ plugins/processors/lowercase/lowercase.go | 77 +++++++ .../processors/lowercase/lowercase_test.go | 205 ++++++++++++++++++ 4 files changed, 315 insertions(+) create mode 100644 plugins/processors/lowercase/README.md create mode 100644 plugins/processors/lowercase/lowercase.go create mode 100644 plugins/processors/lowercase/lowercase_test.go diff --git a/plugins/processors/all/all.go b/plugins/processors/all/all.go index 1eecbfa7e96d9..a03753ff63f27 100644 --- a/plugins/processors/all/all.go +++ b/plugins/processors/all/all.go @@ -3,4 +3,5 @@ package all import ( _ "github.com/influxdata/telegraf/plugins/processors/override" _ "github.com/influxdata/telegraf/plugins/processors/printer" + _ "github.com/influxdata/telegraf/plugins/processors/lowercase" ) diff --git a/plugins/processors/lowercase/README.md b/plugins/processors/lowercase/README.md new file mode 100644 index 0000000000000..3e25211727407 --- /dev/null +++ b/plugins/processors/lowercase/README.md @@ -0,0 +1,32 @@ +# Lowercase Processor Plugin + +The `lowercase` plugin transforms tag and field values to lower case. If `result_key` parameter is present, it can produce new tags and fields from existing ones. + +### Configuration: + +```toml +[[processors.lowercase]] + namepass = ["uri_stem"] + + # Tag and field conversions defined in a separate sub-tables + [[processors.lowercase.tags]] + ## Tag to change + key = "uri_stem" + + [[processors.lowercase.tags]] + ## Multiple tags or fields may be defined + key = "method" + + [[processors.lowercase.fields]] + key = "cs-host" + result_key = "cs-host_normalised" +``` + +### Tags: + +No tags are applied by this processor. + +### Example Output: +``` +iis_log,method=get,uri_stem=/api/healthcheck cs-host="MIXEDCASE_host",cs-host_normalised="mixedcase_host",referrer="-",ident="-",http_version=1.1,agent="UserAgent",resp_bytes=270i 1519652321000000000 +``` diff --git a/plugins/processors/lowercase/lowercase.go b/plugins/processors/lowercase/lowercase.go new file mode 100644 index 0000000000000..c2f1c8d0e210d --- /dev/null +++ b/plugins/processors/lowercase/lowercase.go @@ -0,0 +1,77 @@ +package lowercase + +import ( + "strings" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/processors" +) + +type Lowercase struct { + Tags []converter + Fields []converter +} + +type converter struct { + Key string + ResultKey string +} + +const sampleConfig = ` + ## Tag and field conversions defined in a separate sub-tables + # [[processors.lowercase.tags]] + # ## Tag to change + # key = "method" + + # [[processors.lowercase.fields]] + # key = "uri_stem" + # result_key = "uri_stem_normalised" +` + +func (r *Lowercase) SampleConfig() string { + return sampleConfig +} + +func (r *Lowercase) Description() string { + return "Transforms tag and field values to lower case" +} + +func (r *Lowercase) Apply(in ...telegraf.Metric) []telegraf.Metric { + for _, metric := range in { + for _, converter := range r.Tags { + if value, ok := metric.Tags()[converter.Key]; ok { + metric.AddTag( + getKey(converter), + strings.ToLower(value), + ) + } + } + + for _, converter := range r.Fields { + if value, ok := metric.Fields()[converter.Key]; ok { + switch value := value.(type) { + case string: + metric.AddField( + getKey(converter), + strings.ToLower(value), + ) + } + } + } + } + + return in +} + +func getKey(c converter) string { + if c.ResultKey != "" { + return c.ResultKey + } + return c.Key +} + +func init() { + processors.Add("lowercase", func() telegraf.Processor { + return &Lowercase{} + }) +} diff --git a/plugins/processors/lowercase/lowercase_test.go b/plugins/processors/lowercase/lowercase_test.go new file mode 100644 index 0000000000000..74aba3f176680 --- /dev/null +++ b/plugins/processors/lowercase/lowercase_test.go @@ -0,0 +1,205 @@ +package lowercase + +import ( + "testing" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/metric" + "github.com/stretchr/testify/assert" +) + +func newM1() telegraf.Metric { + m1, _ := metric.New("IIS_log", + map[string]string{ + "verb": "GET", + "s-computername": "MIXEDCASE_hostname", + }, + map[string]interface{}{ + "request": "/mixed/CASE/paTH/?from=-1D&to=now", + }, + time.Now(), + ) + return m1 +} + +func newM2() telegraf.Metric { + m2, _ := metric.New("IIS_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + "s-computername": "MIXEDCASE_hostname", + }, + map[string]interface{}{ + "request": "/mixed/CASE/paTH/?from=-1D&to=now", + "cs-host": "AAAbbb", + "ignore_number": int64(200), + "ignore_bool": true, + }, + time.Now(), + ) + return m2 +} + +func TestFieldConversions(t *testing.T) { + tests := []struct { + message string + converter converter + expectedFields map[string]interface{} + }{ + { + message: "Should change existing field", + converter: converter{ + Key: "request", + }, + expectedFields: map[string]interface{}{ + "request": "/mixed/case/path/?from=-1d&to=now", + }, + }, + { + message: "Should add new field", + converter: converter{ + Key: "request", + ResultKey: "lowercase_request", + }, + expectedFields: map[string]interface{}{ + "request": "/mixed/CASE/paTH/?from=-1D&to=now", + "lowercase_request": "/mixed/case/path/?from=-1d&to=now", + }, + }, + } + + for _, test := range tests { + lowercase := &Lowercase{} + lowercase.Fields = []converter{ + test.converter, + } + + processed := lowercase.Apply(newM1()) + + expectedTags := map[string]string{ + "verb": "GET", + "s-computername": "MIXEDCASE_hostname", + } + + assert.Equal(t, test.expectedFields, processed[0].Fields(), test.message) + assert.Equal(t, expectedTags, processed[0].Tags(), "Should not change tags") + assert.Equal(t, "IIS_log", processed[0].Name(), "Should not change name") + } +} + +func TestTagConversions(t *testing.T) { + tests := []struct { + message string + converter converter + expectedTags map[string]string + }{ + { + message: "Should change existing tag", + converter: converter{ + Key: "s-computername", + }, + expectedTags: map[string]string{ + "verb": "GET", + "s-computername": "mixedcase_hostname", + }, + }, + { + message: "Should add new tag", + converter: converter{ + Key: "s-computername", + ResultKey: "s-computername_lowercase", + }, + expectedTags: map[string]string{ + "verb": "GET", + "s-computername": "MIXEDCASE_hostname", + "s-computername_lowercase": "mixedcase_hostname", + }, + }, + } + + for _, test := range tests { + lowercase := &Lowercase{} + lowercase.Tags = []converter{ + test.converter, + } + + processed := lowercase.Apply(newM1()) + + expectedFields := map[string]interface{}{ + "request": "/mixed/CASE/paTH/?from=-1D&to=now", + } + + assert.Equal(t, expectedFields, processed[0].Fields(), test.message, "Should not change fields") + assert.Equal(t, test.expectedTags, processed[0].Tags(), test.message) + assert.Equal(t, "IIS_log", processed[0].Name(), "Should not change name") + } +} + +func TestMultipleConversions(t *testing.T) { + lowercase := &Lowercase{} + lowercase.Tags = []converter{ + { + Key: "verb", + }, + { + Key: "s-computername", + }, + } + lowercase.Fields = []converter{ + { + Key: "request", + }, + { + Key: "cs-host", + ResultKey: "cs-host_lowercase", + }, + } + + processed := lowercase.Apply(newM2()) + + expectedFields := map[string]interface{}{ + "request": "/mixed/case/path/?from=-1d&to=now", + "ignore_number": int64(200), + "ignore_bool": true, + "cs-host": "AAAbbb", + "cs-host_lowercase": "aaabbb", + } + expectedTags := map[string]string{ + "verb": "get", + "resp_code": "200", + "s-computername": "mixedcase_hostname", + } + + assert.Equal(t, expectedFields, processed[0].Fields()) + assert.Equal(t, expectedTags, processed[0].Tags()) +} + +func TestNoKey(t *testing.T) { + tests := []struct { + message string + converter converter + expectedFields map[string]interface{} + }{ + { + message: "Should not change anything if there is no field with given key", + converter: converter{ + Key: "not_exists", + }, + expectedFields: map[string]interface{}{ + "request": "/mixed/CASE/paTH/?from=-1D&to=now", + }, + }, + } + + for _, test := range tests { + lowercase := &Lowercase{} + lowercase.Fields = []converter{ + test.converter, + } + + processed := lowercase.Apply(newM1()) + + assert.Equal(t, test.expectedFields, processed[0].Fields(), test.message) + } +}