From 35eaadc6ec7b2de8e01026e367b0b8e3920c5d41 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 25 Feb 2020 21:05:52 -0500 Subject: [PATCH 1/5] Add decode_cef processor benchmarks --- .../processors/decode_cef/decode_cef_test.go | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/x-pack/filebeat/processors/decode_cef/decode_cef_test.go b/x-pack/filebeat/processors/decode_cef/decode_cef_test.go index a23663861bb..dd1308fd678 100644 --- a/x-pack/filebeat/processors/decode_cef/decode_cef_test.go +++ b/x-pack/filebeat/processors/decode_cef/decode_cef_test.go @@ -300,3 +300,38 @@ func assertEqual(t testing.TB, expected, actual interface{}) bool { t.Errorf("Expected and actual are different:\n%s", diff) return false } + +func BenchmarkProcessorRun(b *testing.B) { + dec, err := newDecodeCEF(defaultConfig()) + if err != nil { + b.Fatal(err) + } + + const msg = `CEF:1|Trend Micro|Deep Security Manager|1.2.3|600|User Signed In|3|src=10.52.116.160 suser=admin target=admin msg=User signed in from 2001:db8::5` + b.Run("short_msg", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := dec.Run(&beat.Event{ + Fields: map[string]interface{}{ + "message": msg, + }, + }) + if err != nil { + b.Fatal(err) + } + } + }) + + const longMsg = `CEF:0|CISCO|ASA||305012|Teardown dynamic UDP translation|Low| eventId=56265798504 mrt=1484092683471 proto=UDP categorySignificance=/Informational categoryBehavior=/Access/Stop categoryDeviceGroup=/Firewall catdt=Firewall categoryOutcome=/Success categoryObject=/Host/Application/Service modelConfidence=0 severity=4 relevance=10 assetCriticality=0 priority=4 art=1484096108163 deviceSeverity=6 rt=1484096094000 src=1.2.3.4 sourceZoneID=GqtK3G9YBABCadQ465CqVeW\=\= sourceZoneURI=/All Zones/GTR/GTR/GTR/GTR sourceTranslatedAddress=4.3.2.1 sourceTranslatedZoneID=P84KXXTYDFYYFwwHq40BQcd\=\= sourceTranslatedZoneURI=/All Zones/GTR/GTR Internet Primary spt=5260 sourceTranslatedPort=5260 cs5=dynamic cs6=0:00:00 c6a4=ffff:0:0:0:222:5555:ffff:5555 locality=1 cs1Label=ACL cs2Label=Unit cs3Label=TCP Flags cs4Label=Order cs5Label=Connection Type cs6Label=Duration cn1Label=ICMP Type cn2Label=ICMP Code cn3Label=DurationInSeconds c6a4Label=Agent IPv6 Address ahost=host.gtr.gtr agt=100.222.333.55 av=7.1.7.7602.0 atz=LA/la aid=4p9IZi1kBABCq5RFPFdJWYUw\=\= at=agent_ac dvchost=super dvc=111.111.111.99 deviceZoneID=K-fU33AAOGVdfFpYAT3UdQ\=\= deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 deviceAssetId=5Wa8hHVSDFBCc-t56wI7mTw\=\= dtz=LA/LA deviceInboundInterface=eth0 deviceOutboundInterface=eth1 eventAnnotationStageUpdateTime=1484097686473 eventAnnotationModificationTime=1484097686475 eventAnnotationAuditTrail=1,1484012146095,root,Queued,,,,\\n eventAnnotationVersion=1 eventAnnotationFlags=0 eventAnnotationEndTime=1484096094000 eventAnnotationManagerReceiptTime=1484097686471 originalAgentHostName=host originalAgentAddress=10.2.88.3 originalAgentZoneURI=/All Zones/GR/GR/GR originalAgentVersion=7.3.0.7885.0 originalAgentId=6q0sfHVcBABCcSDFvMpvc1w\=\= originalAgentType=syslog_file _cefVer=0.1 ad.arcSightEventPath=7q0sfHVcBABCcMZVvMSDFc1w\=\=` + b.Run("long_msg", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := dec.Run(&beat.Event{ + Fields: map[string]interface{}{ + "message": longMsg, + }, + }) + if err != nil { + b.Fatal(err) + } + } + }) +} From c779f4931fb0c91cd1d1b8fa69aa0ae7f19ef076 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 25 Feb 2020 21:11:05 -0500 Subject: [PATCH 2/5] Take advantage of string slicing to avoid allocations Change the input of the cef parser to accept a string instead of []byte to avoid unnecessary copies. Data is still copied from the input message when it contains escape sequences. Another minor improvement is to allocate the map for the CEF extensions up front based on the estimated number of extension fields. Results from `go test -run none -bench . -benchtime 5s -benchmem .` before and after: $ benchcmp before.txt after.txt benchmark old ns/op new ns/op delta BenchmarkProcessorRun/short_msg-12 4833 4684 -3.08% BenchmarkProcessorRun/long_msg-12 55724 52493 -5.80% benchmark old allocs new allocs delta BenchmarkProcessorRun/short_msg-12 55 41 -25.45% BenchmarkProcessorRun/long_msg-12 349 219 -37.25% benchmark old bytes new bytes delta BenchmarkProcessorRun/short_msg-12 3728 3424 -8.15% BenchmarkProcessorRun/long_msg-12 26929 21173 -21.37% --- .../filebeat/processors/decode_cef/cef/cef.go | 55 ++++++++++++------- .../filebeat/processors/decode_cef/cef/cef.rl | 26 ++++----- .../processors/decode_cef/cef/cef_test.go | 34 ++++++------ .../decode_cef/cef/cmd/cef2json/cef2json.go | 2 +- .../processors/decode_cef/cef/fuzz/fuzz.go | 2 +- .../processors/decode_cef/cef/parser.go | 42 +++++++------- .../processors/decode_cef/decode_cef.go | 2 +- 7 files changed, 90 insertions(+), 73 deletions(-) diff --git a/x-pack/filebeat/processors/decode_cef/cef/cef.go b/x-pack/filebeat/processors/decode_cef/cef/cef.go index c3cee5ff02d..2ca1c95758a 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/cef.go +++ b/x-pack/filebeat/processors/decode_cef/cef/cef.go @@ -5,7 +5,7 @@ package cef import ( - "bytes" + "strings" "github.com/pkg/errors" "go.uber.org/multierr" @@ -59,7 +59,7 @@ type Event struct { Extensions map[string]*Field `json:"extensions,omitempty"` } -func (e *Event) init() { +func (e *Event) init(data string) { e.Version = -1 e.DeviceVendor = "" e.DeviceProduct = "" @@ -68,13 +68,25 @@ func (e *Event) init() { e.Name = "" e.Severity = "" e.Extensions = nil + + // Estimate length of the extensions. But limit the allocation because + // it's based on user input. This doesn't account for escaped equals. + if n := strings.Count(data, "="); n > 0 { + const maxLen = 50 + if n <= maxLen { + e.Extensions = make(map[string]*Field, n) + } else { + e.Extensions = make(map[string]*Field, maxLen) + } + } } -func (e *Event) pushExtension(key []byte, value []byte) { +func (e *Event) pushExtension(key, value string) { if e.Extensions == nil { e.Extensions = map[string]*Field{} } - e.Extensions[string(key)] = &Field{String: string(value)} + field := &Field{String: value} + e.Extensions[key] = field } // Unpack unpacks a common event format (CEF) message. The data is expected to @@ -99,7 +111,7 @@ func (e *Event) pushExtension(key []byte, value []byte) { // and may contain alphanumeric, underscore (_), period (.), comma (,), and // brackets ([) (]). This is less strict than the CEF specification, but aligns // the key names used in practice. -func (e *Event) Unpack(data []byte, opts ...Option) error { +func (e *Event) Unpack(data string, opts ...Option) error { var settings Settings for _, opt := range opts { opt.Apply(&settings) @@ -137,29 +149,32 @@ func (e *Event) Unpack(data []byte, opts ...Option) error { return multierr.Combine(errs...) } -var ( - backslash = []byte(`\`) - escapedBackslash = []byte(`\\`) +const ( + backslash = `\` + escapedBackslash = `\\` - pipe = []byte(`|`) - escapedPipe = []byte(`\|`) + pipe = `|` + escapedPipe = `\|` - equalsSign = []byte(`=`) - escapedEqualsSign = []byte(`\=`) + equalsSign = `=` + escapedEqualsSign = `\=` +) + +var ( + headerEscapes = strings.NewReplacer(escapedBackslash, backslash, escapedPipe, pipe) + extensionEscapes = strings.NewReplacer(escapedBackslash, backslash, escapedEqualsSign, equalsSign) ) -func replaceHeaderEscapes(b []byte) []byte { - if bytes.IndexByte(b, '\\') != -1 { - b = bytes.ReplaceAll(b, escapedBackslash, backslash) - b = bytes.ReplaceAll(b, escapedPipe, pipe) +func replaceHeaderEscapes(b string) string { + if strings.Index(b, escapedBackslash) != -1 || strings.Index(b, escapedPipe) != -1 { + return headerEscapes.Replace(b) } return b } -func replaceExtensionEscapes(b []byte) []byte { - if bytes.IndexByte(b, '\\') != -1 { - b = bytes.ReplaceAll(b, escapedBackslash, backslash) - b = bytes.ReplaceAll(b, escapedEqualsSign, equalsSign) +func replaceExtensionEscapes(b string) string { + if strings.Index(b, escapedBackslash) != -1 || strings.Index(b, escapedEqualsSign) != -1 { + return extensionEscapes.Replace(b) } return b } diff --git a/x-pack/filebeat/processors/decode_cef/cef/cef.rl b/x-pack/filebeat/processors/decode_cef/cef/cef.rl index 55248ab1534..3ac5af35a40 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/cef.rl +++ b/x-pack/filebeat/processors/decode_cef/cef/cef.rl @@ -16,12 +16,12 @@ import ( }%% // unpack unpacks a CEF message. -func (e *Event) unpack(data []byte) error { +func (e *Event) unpack(data string) error { cs, p, pe, eof := 0, 0, len(data), len(data) mark := 0 // Extension key. - var extKey []byte + var extKey string // Extension value start and end indices. extValueStart, extValueEnd := 0, 0 @@ -30,7 +30,7 @@ func (e *Event) unpack(data []byte) error { // recover from (though the parsing might not be "correct"). var recoveredErrs []error - e.init() + e.init(data) %%{ # Actions to execute while executing state machine. @@ -38,31 +38,31 @@ func (e *Event) unpack(data []byte) error { mark = p } action version { - e.Version, _ = strconv.Atoi(string(data[mark:p])) + e.Version, _ = strconv.Atoi(data[mark:p]) } action device_vendor { - e.DeviceVendor = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceVendor = replaceHeaderEscapes(data[mark:p]) } action device_product { - e.DeviceProduct = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceProduct = replaceHeaderEscapes(data[mark:p]) } action device_version { - e.DeviceVersion = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceVersion = replaceHeaderEscapes(data[mark:p]) } action device_event_class_id { - e.DeviceEventClassID = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceEventClassID = replaceHeaderEscapes(data[mark:p]) } action name { - e.Name = string(replaceHeaderEscapes(data[mark:p])) + e.Name = replaceHeaderEscapes(data[mark:p]) } action severity { - e.Severity = string(data[mark:p]) + e.Severity = data[mark:p] } action extension_key { // A new extension key marks the end of the last extension value. if len(extKey) > 0 && extValueStart <= mark - 1 { e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:mark-1])) - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 } extKey = data[mark:p] } @@ -77,7 +77,7 @@ func (e *Event) unpack(data []byte) error { // Reaching the EOF marks the end of the final extension value. if len(extKey) > 0 && extValueStart <= extValueEnd { e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:extValueEnd])) - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 } } action extension_err { @@ -85,7 +85,7 @@ func (e *Event) unpack(data []byte) error { fhold; fgoto gobble_extension; } action recover_next_extension { - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 // Resume processing at p, the start of the next extension key. p = mark; fgoto extensions; diff --git a/x-pack/filebeat/processors/decode_cef/cef/cef_test.go b/x-pack/filebeat/processors/decode_cef/cef/cef_test.go index a24e38d3aac..01f905a1a06 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/cef_test.go +++ b/x-pack/filebeat/processors/decode_cef/cef/cef_test.go @@ -79,7 +79,7 @@ func TestGenerateFuzzCorpus(t *testing.T) { func TestEventUnpack(t *testing.T) { t.Run("standardMessage", func(t *testing.T) { var e Event - err := e.Unpack([]byte(standardMessage)) + err := e.Unpack(standardMessage) assert.NoError(t, err) assert.Equal(t, 26, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -98,7 +98,7 @@ func TestEventUnpack(t *testing.T) { t.Run("headerOnly", func(t *testing.T) { var e Event - err := e.Unpack([]byte(headerOnly)) + err := e.Unpack(headerOnly) assert.NoError(t, err) assert.Equal(t, 26, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -112,7 +112,7 @@ func TestEventUnpack(t *testing.T) { t.Run("escapedPipeInHeader", func(t *testing.T) { var e Event - err := e.Unpack([]byte(escapedPipeInHeader)) + err := e.Unpack(escapedPipeInHeader) assert.NoError(t, err) assert.Equal(t, 26, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -130,7 +130,7 @@ func TestEventUnpack(t *testing.T) { t.Run("equalsSignInHeader", func(t *testing.T) { var e Event - err := e.Unpack([]byte(equalsSignInHeader)) + err := e.Unpack(equalsSignInHeader) assert.NoError(t, err) assert.Equal(t, 26, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -148,7 +148,7 @@ func TestEventUnpack(t *testing.T) { t.Run("emptyExtensionValue", func(t *testing.T) { var e Event - err := e.Unpack([]byte(emptyExtensionValue)) + err := e.Unpack(emptyExtensionValue) assert.Error(t, err) assert.Equal(t, 26, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -165,7 +165,7 @@ func TestEventUnpack(t *testing.T) { t.Run("emptyDeviceFields", func(t *testing.T) { var e Event - err := e.Unpack([]byte(emptyDeviceFields)) + err := e.Unpack(emptyDeviceFields) assert.NoError(t, err) assert.Equal(t, 0, e.Version) assert.Equal(t, "", e.DeviceVendor) @@ -183,7 +183,7 @@ func TestEventUnpack(t *testing.T) { t.Run("errorEscapedPipeInExtension", func(t *testing.T) { var e Event - err := e.Unpack([]byte(escapedPipeInExtension)) + err := e.Unpack(escapedPipeInExtension) assert.Equal(t, 0, e.Version) assert.Equal(t, "security", e.DeviceVendor) assert.Equal(t, "threatmanager", e.DeviceProduct) @@ -199,7 +199,7 @@ func TestEventUnpack(t *testing.T) { t.Run("leadingWhitespace", func(t *testing.T) { var e Event - err := e.Unpack([]byte(leadingWhitespace)) + err := e.Unpack(leadingWhitespace) assert.NoError(t, err) assert.Equal(t, 0, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -217,7 +217,7 @@ func TestEventUnpack(t *testing.T) { t.Run("pipeInMessage", func(t *testing.T) { var e Event - err := e.Unpack([]byte(pipeInMessage)) + err := e.Unpack(pipeInMessage) assert.NoError(t, err) assert.Equal(t, 0, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -233,7 +233,7 @@ func TestEventUnpack(t *testing.T) { t.Run("errorEqualsInMessage", func(t *testing.T) { var e Event - err := e.Unpack([]byte(equalsInMessage)) + err := e.Unpack(equalsInMessage) assert.Equal(t, 0, e.Version) assert.Equal(t, "security", e.DeviceVendor) assert.Equal(t, "threatmanager", e.DeviceProduct) @@ -249,7 +249,7 @@ func TestEventUnpack(t *testing.T) { t.Run("escapesInExtension", func(t *testing.T) { var e Event - err := e.Unpack([]byte(escapesInExtension)) + err := e.Unpack(escapesInExtension) assert.NoError(t, err) assert.Equal(t, 0, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -266,7 +266,7 @@ func TestEventUnpack(t *testing.T) { t.Run("errorMalformedExtensionEscape", func(t *testing.T) { var e Event - err := e.Unpack([]byte(malformedExtensionEscape)) + err := e.Unpack(malformedExtensionEscape) assert.Equal(t, 0, e.Version) assert.Equal(t, "FooBar", e.DeviceVendor) assert.Equal(t, "Web Gateway", e.DeviceProduct) @@ -296,7 +296,7 @@ func TestEventUnpack(t *testing.T) { t.Run("errorMultipleMalformedExtensionValues", func(t *testing.T) { var e Event - err := e.Unpack([]byte(multipleMalformedExtensionValues)) + err := e.Unpack(multipleMalformedExtensionValues) assert.Equal(t, 0, e.Version) assert.Equal(t, "vendor", e.DeviceVendor) assert.Equal(t, "product", e.DeviceProduct) @@ -319,14 +319,14 @@ func TestEventUnpack(t *testing.T) { t.Run("empty", func(t *testing.T) { var e Event - err := e.Unpack([]byte("CEF:0|||||||a=")) + err := e.Unpack("CEF:0|||||||a=") assert.NoError(t, err) }) } func TestEventUnpackWithFullExtensionNames(t *testing.T) { var e Event - err := e.Unpack([]byte(standardMessage), WithFullExtensionNames()) + err := e.Unpack(standardMessage, WithFullExtensionNames()) assert.NoError(t, err) assert.Equal(t, map[string]*Field{ "sourceAddress": IPField("10.0.0.192"), @@ -337,9 +337,9 @@ func TestEventUnpackWithFullExtensionNames(t *testing.T) { } func BenchmarkEventUnpack(b *testing.B) { - var messages [][]byte + var messages []string for _, m := range testMessages { - messages = append(messages, []byte(m)) + messages = append(messages, m) } b.ResetTimer() diff --git a/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/cef2json.go b/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/cef2json.go index e11b9309eb2..d95beff8d04 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/cef2json.go +++ b/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/cef2json.go @@ -49,7 +49,7 @@ func main() { line = line[begin:] var e cef.Event - if err := e.Unpack(line, opts...); err != nil { + if err := e.Unpack(string(line), opts...); err != nil { log.Println("ERROR:", err, "in:", string(line)) } diff --git a/x-pack/filebeat/processors/decode_cef/cef/fuzz/fuzz.go b/x-pack/filebeat/processors/decode_cef/cef/fuzz/fuzz.go index 427daae679d..9114c12f84f 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/fuzz/fuzz.go +++ b/x-pack/filebeat/processors/decode_cef/cef/fuzz/fuzz.go @@ -11,7 +11,7 @@ import ( // Fuzz is the entry point that go-fuzz uses to fuzz the parser. func Fuzz(data []byte) int { var e cef2.Event - if err := e.Unpack(data); err != nil { + if err := e.Unpack(string(data)); err != nil { return 1 } return 0 diff --git a/x-pack/filebeat/processors/decode_cef/cef/parser.go b/x-pack/filebeat/processors/decode_cef/cef/parser.go index f7ee7802dad..62ac1fb388d 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/parser.go +++ b/x-pack/filebeat/processors/decode_cef/cef/parser.go @@ -34,12 +34,12 @@ const cef_en_main_cef_extensions int = 24 //line cef.rl:16 // unpack unpacks a CEF message. -func (e *Event) unpack(data []byte) error { +func (e *Event) unpack(data string) error { cs, p, pe, eof := 0, 0, len(data), len(data) mark := 0 // Extension key. - var extKey []byte + var extKey string // Extension value start and end indices. extValueStart, extValueEnd := 0, 0 @@ -48,7 +48,7 @@ func (e *Event) unpack(data []byte) error { // recover from (though the parsing might not be "correct"). var recoveredErrs []error - e.init() + e.init(data) //line parser.go:55 { @@ -760,43 +760,43 @@ func (e *Event) unpack(data []byte) error { f1: //line cef.rl:40 - e.Version, _ = strconv.Atoi(string(data[mark:p])) + e.Version, _ = strconv.Atoi(data[mark:p]) goto _again f3: //line cef.rl:43 - e.DeviceVendor = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceVendor = replaceHeaderEscapes(data[mark:p]) goto _again f5: //line cef.rl:46 - e.DeviceProduct = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceProduct = replaceHeaderEscapes(data[mark:p]) goto _again f7: //line cef.rl:49 - e.DeviceVersion = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceVersion = replaceHeaderEscapes(data[mark:p]) goto _again f9: //line cef.rl:52 - e.DeviceEventClassID = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceEventClassID = replaceHeaderEscapes(data[mark:p]) goto _again f11: //line cef.rl:55 - e.Name = string(replaceHeaderEscapes(data[mark:p])) + e.Name = replaceHeaderEscapes(data[mark:p]) goto _again f13: //line cef.rl:58 - e.Severity = string(data[mark:p]) + e.Severity = data[mark:p] goto _again f14: @@ -805,7 +805,7 @@ func (e *Event) unpack(data []byte) error { // A new extension key marks the end of the last extension value. if len(extKey) > 0 && extValueStart <= mark-1 { e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:mark-1])) - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 } extKey = data[mark:p] @@ -829,15 +829,17 @@ func (e *Event) unpack(data []byte) error { recoveredErrs = append(recoveredErrs, fmt.Errorf("malformed value for %s at pos %d", extKey, p+1)) (p)-- cs = 28 + goto _again goto _again f17: //line cef.rl:87 - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 // Resume processing at p, the start of the next extension key. p = mark cs = 24 + goto _again goto _again f2: @@ -847,7 +849,7 @@ func (e *Event) unpack(data []byte) error { //line cef.rl:43 - e.DeviceVendor = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceVendor = replaceHeaderEscapes(data[mark:p]) goto _again f4: @@ -857,7 +859,7 @@ func (e *Event) unpack(data []byte) error { //line cef.rl:46 - e.DeviceProduct = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceProduct = replaceHeaderEscapes(data[mark:p]) goto _again f6: @@ -867,7 +869,7 @@ func (e *Event) unpack(data []byte) error { //line cef.rl:49 - e.DeviceVersion = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceVersion = replaceHeaderEscapes(data[mark:p]) goto _again f8: @@ -877,7 +879,7 @@ func (e *Event) unpack(data []byte) error { //line cef.rl:52 - e.DeviceEventClassID = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceEventClassID = replaceHeaderEscapes(data[mark:p]) goto _again f10: @@ -887,7 +889,7 @@ func (e *Event) unpack(data []byte) error { //line cef.rl:55 - e.Name = string(replaceHeaderEscapes(data[mark:p])) + e.Name = replaceHeaderEscapes(data[mark:p]) goto _again f12: @@ -897,7 +899,7 @@ func (e *Event) unpack(data []byte) error { //line cef.rl:58 - e.Severity = string(data[mark:p]) + e.Severity = data[mark:p] goto _again f23: @@ -950,7 +952,7 @@ func (e *Event) unpack(data []byte) error { // Reaching the EOF marks the end of the final extension value. if len(extKey) > 0 && extValueStart <= extValueEnd { e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:extValueEnd])) - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 } case 16: @@ -972,7 +974,7 @@ func (e *Event) unpack(data []byte) error { // Reaching the EOF marks the end of the final extension value. if len(extKey) > 0 && extValueStart <= extValueEnd { e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:extValueEnd])) - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 } //line parser.go:847 diff --git a/x-pack/filebeat/processors/decode_cef/decode_cef.go b/x-pack/filebeat/processors/decode_cef/decode_cef.go index 45853f5378a..ae6672ac024 100644 --- a/x-pack/filebeat/processors/decode_cef/decode_cef.go +++ b/x-pack/filebeat/processors/decode_cef/decode_cef.go @@ -89,7 +89,7 @@ func (p *processor) Run(event *beat.Event) (*beat.Event, error) { // If the version < 0 after parsing then none of the data is valid so return here. var ce cef.Event - if err = ce.Unpack([]byte(cefData), cef.WithFullExtensionNames()); ce.Version < 0 && err != nil { + if err = ce.Unpack(cefData, cef.WithFullExtensionNames()); ce.Version < 0 && err != nil { if p.IgnoreFailure { return event, nil } From 6f62fff1237e735a7c800911265b4e56690c1581 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 25 Feb 2020 21:17:52 -0500 Subject: [PATCH 3/5] Add changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c5b6f756e96..f258c76bcaa 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -137,6 +137,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307] - Improve ECS categorization field mappings in googlecloud module. {issue}16030[16030] {pull}16500[16500] - Improve ECS field mappings in haproxy module. {issue}16162[16162] {pull}16529[16529] +- Improve the decode_cef processor by reducing the number of memory allocations. {pull}16587[16587] *Heartbeat* From 6b242fbd6f97f73165dd74c47eb7936dd5ccb41d Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 25 Feb 2020 22:01:53 -0500 Subject: [PATCH 4/5] Remove dead generated code --- x-pack/filebeat/processors/decode_cef/cef/parser.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/filebeat/processors/decode_cef/cef/parser.go b/x-pack/filebeat/processors/decode_cef/cef/parser.go index 62ac1fb388d..2ddcb942401 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/parser.go +++ b/x-pack/filebeat/processors/decode_cef/cef/parser.go @@ -830,8 +830,6 @@ func (e *Event) unpack(data string) error { (p)-- cs = 28 goto _again - - goto _again f17: //line cef.rl:87 @@ -840,8 +838,6 @@ func (e *Event) unpack(data string) error { p = mark cs = 24 goto _again - - goto _again f2: //line cef.rl:37 From ae665d1df2d64c05230a528ac89a648f944f6561 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 26 Feb 2020 13:56:22 -0500 Subject: [PATCH 5/5] Update test assertions for empty vs nil extension maps --- x-pack/filebeat/processors/decode_cef/cef/cef_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/processors/decode_cef/cef/cef_test.go b/x-pack/filebeat/processors/decode_cef/cef/cef_test.go index 01f905a1a06..361b105cd52 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/cef_test.go +++ b/x-pack/filebeat/processors/decode_cef/cef/cef_test.go @@ -191,7 +191,7 @@ func TestEventUnpack(t *testing.T) { assert.Equal(t, "100", e.DeviceEventClassID) assert.Equal(t, "trojan successfully stopped", e.Name) assert.Equal(t, "10", e.Severity) - assert.Nil(t, e.Extensions) + assert.Empty(t, e.Extensions) // Pipes in extensions should not be escaped. assert.Error(t, err) @@ -241,7 +241,7 @@ func TestEventUnpack(t *testing.T) { assert.Equal(t, "100", e.DeviceEventClassID) assert.Equal(t, "trojan successfully stopped", e.Name) assert.Equal(t, "10", e.Severity) - assert.Nil(t, e.Extensions) + assert.Empty(t, e.Extensions) // moo contains unescaped equals signs. assert.Error(t, err)