custom time, msg, level fields are now appended to the default list

This is so that the tool works more often than not at parsing logs. If
this cause problems, we might want to allow forcing a specific set of
strings (prior behavior) somehow.

handler.go

@@ -16,10 +16,10 @@ var DefaultOptions = func() *HandlerOptions {
 	opts := &HandlerOptions{
 		TimeFields:    []string{"time", "ts", "@timestamp", "timestamp", "Timestamp"},
 		MessageFields: []string{"message", "msg", "Body"},
-		LevelFields:   []string{"level", "lvl", "loglevel", "severity", "SeverityLevel"},
+		LevelFields:   []string{"level", "lvl", "loglevel", "severity", "SeverityText"},
 	}
 	return opts
-}()
+}
 
 type HandlerOptions struct {
 	TimeFields    []string
@@ -30,15 +30,37 @@ type HandlerOptions struct {
 var _ = HandlerOptionsFrom(config.DefaultConfig) // ensure it's valid
 
 func HandlerOptionsFrom(cfg config.Config) *HandlerOptions {
-	opts := DefaultOptions
+	opts := DefaultOptions()
 	if cfg.TimeFields != nil {
-		opts.TimeFields = *cfg.TimeFields
+		opts.TimeFields = appendUnique(opts.TimeFields, *cfg.TimeFields)
 	}
 	if cfg.MessageFields != nil {
-		opts.MessageFields = *cfg.MessageFields
+		opts.MessageFields = appendUnique(opts.MessageFields, *cfg.MessageFields)
 	}
 	if cfg.LevelFields != nil {
-		opts.LevelFields = *cfg.LevelFields
+		opts.LevelFields = appendUnique(opts.LevelFields, *cfg.LevelFields)
 	}
 	return opts
 }
+
+func appendUnique(a []string, b []string) []string {
+	// init with `len(b)` because usually `a` will be
+	// nil at first, but `b` wont be
+	seen := make(map[string]struct{}, len(b))
+	out := make([]string, 0, len(b))
+	for _, aa := range a {
+		if _, ok := seen[aa]; ok {
+			continue
+		}
+		seen[aa] = struct{}{}
+		out = append(out, aa)
+	}
+	for _, bb := range b {
+		if _, ok := seen[bb]; ok {
+			continue
+		}
+		seen[bb] = struct{}{}
+		out = append(out, bb)
+	}
+	return out
+}

internal/pkg/config/config.go

@@ -102,14 +102,23 @@ func (cfg Config) populateEmpty(other *Config) *Config {
 		out.Skip = other.Skip
 		out.Keep = other.Keep
 	}
-	if out.TimeFields == nil && other.TimeFields != nil {
-		out.TimeFields = other.TimeFields
+	if other.TimeFields != nil {
+		if out.TimeFields == nil {
+			out.TimeFields = ptr(make([]string, 0, len(*other.TimeFields)))
+		}
+		*out.TimeFields = append(*out.TimeFields, *other.TimeFields...)
 	}
 	if out.MessageFields == nil && other.MessageFields != nil {
-		out.MessageFields = other.MessageFields
+		if out.MessageFields == nil {
+			out.MessageFields = ptr(make([]string, 0, len(*other.MessageFields)))
+		}
+		*out.MessageFields = append(*out.MessageFields, *other.MessageFields...)
 	}
 	if out.LevelFields == nil && other.LevelFields != nil {
-		out.LevelFields = other.LevelFields
+		if out.LevelFields == nil {
+			out.LevelFields = ptr(make([]string, 0, len(*other.LevelFields)))
+		}
+		*out.LevelFields = append(*out.LevelFields, *other.LevelFields...)
 	}
 	if out.SortLongest == nil && other.SortLongest != nil {
 		out.SortLongest = other.SortLongest

json_handler.go

@@ -110,10 +110,6 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) bool {
 		return false
 	}
 
-	if h.Opts == nil {
-		h.Opts = DefaultOptions
-	}
-
 	searchJSON(raw, h.Opts.TimeFields, func(field string, value interface{}) bool {
 		var ok bool
 		h.Time, ok = tryParseTime(value)

json_handler_test.go

@@ -20,9 +20,9 @@ func TestJSONHandler_UnmarshalJSON_ParsesFields(t *testing.T) {
 
 	raw := []byte(fmt.Sprintf(`{ "message": %q, "level": %q, "time": %q }`, msg, level, tm))
 
-	opts := *humanlog.DefaultOptions
+	opts := humanlog.DefaultOptions()
 
-	h := humanlog.JSONHandler{Opts: &opts}
+	h := humanlog.JSONHandler{Opts: opts}
 	ev := new(model.Structured)
 	if !h.TryHandle(raw, ev) {
 		t.Fatalf("failed to parse log level")
@@ -52,12 +52,12 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomFields(t *testing.T) {
 
 	raw := []byte(fmt.Sprintf(`{ "mymessage": %q, "mylevel": %q, "mytime": %q }`, msg, level, tm))
 
-	opts := *humanlog.DefaultOptions
+	opts := humanlog.DefaultOptions()
 	opts.LevelFields = []string{"mylevel"}
 	opts.MessageFields = []string{"mymessage"}
 	opts.TimeFields = []string{"mytime"}
 
-	h := humanlog.JSONHandler{Opts: &opts}
+	h := humanlog.JSONHandler{Opts: opts}
 
 	ev := new(model.Structured)
 	if !h.TryHandle(raw, ev) {
@@ -87,12 +87,12 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomNestedFields(t *testing.T) {
 
 	raw := []byte(fmt.Sprintf(`{ "data": { "message": %q, "level": %q, "time": %q }}`, msg, level, tm))
 
-	opts := *humanlog.DefaultOptions
+	opts := humanlog.DefaultOptions()
 	opts.LevelFields = []string{"data.level"}
 	opts.MessageFields = []string{"data.message"}
 	opts.TimeFields = []string{"data.time"}
 
-	h := humanlog.JSONHandler{Opts: &opts}
+	h := humanlog.JSONHandler{Opts: opts}
 	ev := new(model.Structured)
 	if !h.TryHandle(raw, ev) {
 		t.Fatalf("failed to handle log")
@@ -130,12 +130,12 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomMultiNestedFields(t *testing.T) {
 	  }
 	}`, msg, level, tm))
 
-	opts := *humanlog.DefaultOptions
+	opts := humanlog.DefaultOptions()
 	opts.LevelFields = []string{"data.l2.level"}
 	opts.MessageFields = []string{"data.l2.message"}
 	opts.TimeFields = []string{"data.l2.time"}
 
-	h := humanlog.JSONHandler{Opts: &opts}
+	h := humanlog.JSONHandler{Opts: opts}
 	ev := new(model.Structured)
 	if !h.TryHandle(raw, ev) {
 		t.Fatalf("failed to handle log")

scanner_test.go

@@ -18,7 +18,7 @@ func TestScannerLongLine(t *testing.T) {
 		{Raw: []byte(data), Structured: &model.Structured{Msg: strings.Repeat("a", 1023*1024)}},
 	}
 	sink := bufsink.NewSizedBufferedSink(100, nil)
-	err := Scan(ctx, src, sink, DefaultOptions)
+	err := Scan(ctx, src, sink, DefaultOptions())
 	require.NoError(t, err, "got %#v", err)
 	require.Equal(t, want, sink.Buffered)
 }

zap_development_handler_test.go

@@ -199,7 +199,7 @@ func Test_tryZapDevPrefix(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			ev := new(model.Structured)
-			m := tryZapDevPrefix(test.logLine, ev, &JSONHandler{})
+			m := tryZapDevPrefix(test.logLine, ev, &JSONHandler{Opts: DefaultOptions()})
 
 			if m != test.wantMatch {
 				t.Error("expected the prefix to match, it did not")
@@ -429,7 +429,7 @@ func Test_tryZapDevDCPrefix(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			ev := new(model.Structured)
-			m := tryZapDevDCPrefix(test.logLine, ev, &JSONHandler{})
+			m := tryZapDevDCPrefix(test.logLine, ev, &JSONHandler{Opts: DefaultOptions()})
 
 			if m != test.wantMatch {
 				t.Error("expected the prefix to match, it did not")