Speed up addition of fields to forwarded EVE-JSON

processing/forward_handler.go

@@ -5,13 +5,13 @@ package processing
 
 import (
 	"crypto/tls"
+	"fmt"
 	"net"
 	"sync"
 	"time"
 
 	"github.com/DCSO/fever/types"
 	"github.com/DCSO/fever/util"
-	"github.com/buger/jsonparser"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -29,7 +29,7 @@ type ForwardHandler struct {
 	Logger              *log.Entry
 	DoRDNS              bool
 	RDNSHandler         *RDNSHandler
-	AddedFields         map[string]string
+	AddedFields         string
 	ContextCollector    *ContextCollector
 	StenosisIface       string
 	StenosisConnector   *StenosisConnector
@@ -210,19 +210,16 @@ func (fh *ForwardHandler) Consume(e *types.Entry) error {
 				return err
 			}
 		}
-		for k, v := range fh.AddedFields {
-			val, err := util.EscapeJSON(v)
-			if err != nil {
-				fh.Logger.Warningf("cannot escape value: %s", v)
-				continue
-			}
-			newJSON, err := jsonparser.Set([]byte(e.JSONLine), val, k)
-			if err != nil {
-				fh.Logger.Warningf("cannot set %s: %s", k, v)
-				continue
-			} else {
-				e.JSONLine = string(newJSON)
-			}
+		// Replace the final brace `}` in the JSON with the prepared string to
+		// add the 'added fields' defined in the config. I the length of this
+		// string is 1 then there are no added fields, only a final brace '}'.
+		// In this case we don't even need to modify the JSON string at all.
+		if len(fh.AddedFields) > 1 {
+			j := e.JSONLine
+			l := len(j)
+			j = j[:l-1]
+			j += fh.AddedFields
+			e.JSONLine = j
 		}
 		// if we use Stenosis, the Stenosis connector will take ownership of
 		// alerts
@@ -263,8 +260,33 @@ func (fh *ForwardHandler) EnableRDNS(expiryPeriod time.Duration) {
 
 // AddFields enables the addition of a custom set of top-level fields to the
 // forwarded JSON.
-func (fh *ForwardHandler) AddFields(fields map[string]string) {
-	fh.AddedFields = fields
+func (fh *ForwardHandler) AddFields(fields map[string]string) error {
+	j := ""
+	// We preprocess the JSON to be able to only use fast string operations
+	// later. This code progressively builds a JSON snippet by adding JSON
+	// key-value pairs for each added field, e.g. `, "foo":"bar"`.
+	for k, v := range fields {
+		// Escape the fields to make sure we do not mess up the JSON when
+		// encountering weird symbols in field names or values.
+		kval, err := util.EscapeJSON(k)
+		if err != nil {
+			fh.Logger.Warningf("cannot escape value: %s", v)
+			return err
+		}
+		vval, err := util.EscapeJSON(v)
+		if err != nil {
+			fh.Logger.Warningf("cannot escape value: %s", v)
+			return err
+		}
+		j += fmt.Sprintf(",%s:%s", kval, vval)
+	}
+	// We finish the list of key-value pairs with a final brace:
+	// `, "foo":"bar"}`. This string can now just replace the final brace in a
+	// given JSON string. If there were no added fields, we just leave the
+	// output at the final brace.
+	j += "}"
+	fh.AddedFields = j
+	return nil
 }
 
 // EnableStenosis ...

types/entry.go

@@ -14,33 +14,33 @@ type DNSAnswer struct {
 
 // Entry is a collection of data that needs to be parsed FAST from the entry
 type Entry struct {
-	SrcIP         string
-	SrcHosts      []string
-	SrcPort       int64
-	DestIP        string
-	DestHosts     []string
-	DestPort      int64
-	Timestamp     string
-	EventType     string
-	Proto         string
-	HTTPHost      string
-	HTTPUrl       string
-	HTTPMethod    string
-	JSONLine      string
-	DNSVersion    int64
-	DNSRRName     string
-	DNSRRType     string
-	DNSRCode      string
-	DNSRData      string
-	DNSType       string
-	DNSAnswers    []DNSAnswer
-	TLSSNI        string
-	BytesToClient int64
-	BytesToServer int64
-	PktsToClient  int64
-	PktsToServer  int64
-	FlowID        string
-	Iface         string
-	AppProto      string
+	SrcIP          string
+	SrcHosts       []string
+	SrcPort        int64
+	DestIP         string
+	DestHosts      []string
+	DestPort       int64
+	Timestamp      string
+	EventType      string
+	Proto          string
+	HTTPHost       string
+	HTTPUrl        string
+	HTTPMethod     string
+	JSONLine       string
+	DNSVersion     int64
+	DNSRRName      string
+	DNSRRType      string
+	DNSRCode       string
+	DNSRData       string
+	DNSType        string
+	DNSAnswers     []DNSAnswer
+	TLSSNI         string
+	BytesToClient  int64
+	BytesToServer  int64
+	PktsToClient   int64
+	PktsToServer   int64
+	FlowID         string
+	Iface          string
+	AppProto       string
 	TLSFingerprint string
 }

util/util.go

@@ -274,7 +274,7 @@ func GetSensorID() (string, error) {
 // RndStringFromRunes returns a string of length n
 // with randomly picked runes from fromRunes slice
 func RndStringFromRunes(fromRunes []rune, n int) string {
-    result := make([]rune, n)
+	result := make([]rune, n)
 	numRunes := len(fromRunes)
 	for i := range result {
 		result[i] = fromRunes[rand.Intn(numRunes)]
@@ -284,7 +284,7 @@ func RndStringFromRunes(fromRunes []rune, n int) string {
 
 // RndStringFromBytes
 func RndStringFromBytes(fromBytes []byte, n int) string {
-    result := make([]byte, n)
+	result := make([]byte, n)
 	numBytes := len(fromBytes)
 	for i := range result {
 		result[i] = fromBytes[rand.Intn(numBytes)]
@@ -294,25 +294,24 @@ func RndStringFromBytes(fromBytes []byte, n int) string {
 
 // RndStringFromAlpha
 func RndStringFromAlpha(n int) string {
-    return RndStringFromBytes([]byte("abcdefghijklmnopqrstuvwxyz"), n)
+	return RndStringFromBytes([]byte("abcdefghijklmnopqrstuvwxyz"), n)
 }
 
 // RndHexString returns a Hex string of length n
 func RndHexString(n int) string {
-    return RndStringFromBytes([]byte("0123456789abcdef"), n)
+	return RndStringFromBytes([]byte("0123456789abcdef"), n)
 }
 
 // RndTLSFingerprint returns a random string in
 // the form of a TLS fingerprint
 func RndTLSFingerprint() string {
-    nums := make([]string, 20)
-	for i := 0 ; i < 20 ; i++ {
+	nums := make([]string, 20)
+	for i := 0; i < 20; i++ {
 		nums[i] = RndHexString(2)
 	}
 	return strings.Join(nums, ":")
 }
 
-
 // MakeTLSConfig returns a TLS configuration suitable for an endpoint with private
 // key stored in keyFile and corresponding certificate stored in certFile. rcas
 // defines a list of root CA filenames.