From 5fa9c4bd56ec996fa2cf018b4b74b992fedca26a Mon Sep 17 00:00:00 2001 From: benclive Date: Wed, 17 Jul 2024 14:15:04 +0100 Subject: [PATCH] fix: Retain original values in logfmt pattern tokenizer (#13535) --- pkg/pattern/drain/line_tokenizer.go | 2 +- pkg/pattern/drain/line_tokenizer_test.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/pattern/drain/line_tokenizer.go b/pkg/pattern/drain/line_tokenizer.go index 04b998b8dba5..533f1f92931c 100644 --- a/pkg/pattern/drain/line_tokenizer.go +++ b/pkg/pattern/drain/line_tokenizer.go @@ -233,7 +233,7 @@ func (t *logfmtTokenizer) Join(tokens []string, _ interface{}) string { buf := bytes.NewBuffer(make([]byte, 0, 1024)) enc := gologfmt.NewEncoder(buf) for i := 0; i < len(tokens); i += 2 { - k, v := tokens[i], tokens[i+1] + k, v := tokens[i], unsafeBytes(tokens[i+1]) if err := enc.EncodeKeyval(k, v); err != nil { return "" } diff --git a/pkg/pattern/drain/line_tokenizer_test.go b/pkg/pattern/drain/line_tokenizer_test.go index 106d5d44b487..200d1a8f510e 100644 --- a/pkg/pattern/drain/line_tokenizer_test.go +++ b/pkg/pattern/drain/line_tokenizer_test.go @@ -209,6 +209,10 @@ func TestLogFmtTokenizer(t *testing.T) { line: `ts=2024-05-30T12:50:36.648377186Z caller=scheduler_processor.go:143 level=warn msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"error reading server preface: EOF\"" addr=10.0.151.101:9095`, want: []string{"ts", param, "caller", "scheduler_processor.go:143", "level", "warn", "msg", "error contacting scheduler", "err", "rpc error: code = Unavailable desc = connection error: desc = \"error reading server preface: EOF\"", "addr", "10.0.151.101:9095"}, }, + { + line: `logger=sqlstore.metrics traceID=c933fefbe893411d3be8e1648d6bcf37 t=2024-07-10T16:00:15.564896897Z level=debug msg="query finished" status=success elapsedtime=1.324305ms error=null`, + want: []string{"logger", "sqlstore.metrics", "traceID", "<_>", "t", "<_>", "level", "debug", "msg", "query finished", "status", "success", "elapsedtime", "1.324305ms", "", "", "error", "null"}, + }, } tokenizer := newLogfmtTokenizer(param) @@ -258,6 +262,10 @@ func TestLogFmtTokenizerJoin(t *testing.T) { want: `ts=2024-05-30T12:50:36.648377186Z caller=scheduler_processor.go:143 level=warn msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"error reading server preface: EOF\"" addr=10.0.151.101:9095`, tokens: []string{"ts", "2024-05-30T12:50:36.648377186Z", "caller", "scheduler_processor.go:143", "level", "warn", "msg", "error contacting scheduler", "err", "rpc error: code = Unavailable desc = connection error: desc = \"error reading server preface: EOF\"", "addr", "10.0.151.101:9095"}, }, + { + want: `error=null`, + tokens: []string{"error", "null"}, + }, } tokenizer := newLogfmtTokenizer("")