fix: Retain original values in logfmt pattern tokenizer (#13535)

grafana · Jul 17, 2024 · 5fa9c4b · 5fa9c4b
1 parent dbff69a
commit 5fa9c4b
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 1 deletion.
diff --git a/pkg/pattern/drain/line_tokenizer.go b/pkg/pattern/drain/line_tokenizer.go
@@ -233,7 +233,7 @@ func (t *logfmtTokenizer) Join(tokens []string, _ interface{}) string {
 	buf := bytes.NewBuffer(make([]byte, 0, 1024))
 	enc := gologfmt.NewEncoder(buf)
 	for i := 0; i < len(tokens); i += 2 {
-		k, v := tokens[i], tokens[i+1]
+		k, v := tokens[i], unsafeBytes(tokens[i+1])
 		if err := enc.EncodeKeyval(k, v); err != nil {
 			return ""
 		}

diff --git a/pkg/pattern/drain/line_tokenizer_test.go b/pkg/pattern/drain/line_tokenizer_test.go
@@ -209,6 +209,10 @@ func TestLogFmtTokenizer(t *testing.T) {
 			line: `ts=2024-05-30T12:50:36.648377186Z caller=scheduler_processor.go:143 level=warn msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"error reading server preface: EOF\"" addr=10.0.151.101:9095`,
 			want: []string{"ts", param, "caller", "scheduler_processor.go:143", "level", "warn", "msg", "error contacting scheduler", "err", "rpc error: code = Unavailable desc = connection error: desc = \"error reading server preface: EOF\"", "addr", "10.0.151.101:9095"},
 		},
+		{
+			line: `logger=sqlstore.metrics traceID=c933fefbe893411d3be8e1648d6bcf37 t=2024-07-10T16:00:15.564896897Z level=debug msg="query finished" status=success elapsedtime=1.324305ms <REDACTED> error=null`,
+			want: []string{"logger", "sqlstore.metrics", "traceID", "<_>", "t", "<_>", "level", "debug", "msg", "query finished", "status", "success", "elapsedtime", "1.324305ms", "<REDACTED>", "", "error", "null"},
+		},
 	}
 
 	tokenizer := newLogfmtTokenizer(param)
@@ -258,6 +262,10 @@ func TestLogFmtTokenizerJoin(t *testing.T) {
 			want:   `ts=2024-05-30T12:50:36.648377186Z caller=scheduler_processor.go:143 level=warn msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"error reading server preface: EOF\"" addr=10.0.151.101:9095`,
 			tokens: []string{"ts", "2024-05-30T12:50:36.648377186Z", "caller", "scheduler_processor.go:143", "level", "warn", "msg", "error contacting scheduler", "err", "rpc error: code = Unavailable desc = connection error: desc = \"error reading server preface: EOF\"", "addr", "10.0.151.101:9095"},
 		},
+		{
+			want:   `error=null`,
+			tokens: []string{"error", "null"},
+		},
 	}
 
 	tokenizer := newLogfmtTokenizer("")