From 6838ddf886a4e9592efa1442f255b7a77a1f0ff7 Mon Sep 17 00:00:00 2001 From: Antoine Grondin Date: Tue, 10 Dec 2019 16:47:23 +0900 Subject: [PATCH] convert to logfmt, cleanup unmarshaling a bit the code is still super messy and not DRY at all. but it's a bit better now. hopefully nothing was broken, and the exhaustive 0 test suite will catch any regression. yolo --- go.mod | 1 + go.sum | 2 + json_handler.go | 82 ++++++++++--------- logrus_handler.go => logfmt_handler.go | 108 +++++++++++++++++-------- scanner.go | 14 ++-- 5 files changed, 130 insertions(+), 77 deletions(-) rename logrus_handler.go => logfmt_handler.go (63%) diff --git a/go.mod b/go.mod index c50d59f..594f15a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 github.com/fatih/color v1.7.1-0.20180516100307-2d684516a886 + github.com/go-logfmt/logfmt v0.4.0 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 github.com/mattn/go-colorable v0.1.0 github.com/mattn/go-isatty v0.0.4 // indirect diff --git a/go.sum b/go.sum index 68c884c..8359a61 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/fatih/color v1.7.1-0.20180516100307-2d684516a886 h1:NAFoy+QgUpERgK3y1xiVh5HcOvSeZHpXTTo5qnvnuK4= github.com/fatih/color v1.7.1-0.20180516100307-2d684516a886/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= diff --git a/json_handler.go b/json_handler.go index 4294b9c..9c02db9 100644 --- a/json_handler.go +++ b/json_handler.go @@ -29,9 +29,24 @@ type JSONHandler struct { last map[string]string } +func checkEachUntilFound(fieldList []string, found func(string) bool) bool { + for _, field := range fieldList { + if found(field) { + return true + } + } + return false +} + // supportedTimeFields enumerates supported timestamp field names var supportedTimeFields = []string{"time", "ts", "@timestamp"} +// supportedMessageFields enumarates supported Message field names +var supportedMessageFields = []string{"message", "msg"} + +// supportedLevelFields enumarates supported level field names +var supportedLevelFields = []string{"level", "lvl"} + func (h *JSONHandler) clear() { h.Level = "" h.Time = time.Time{} @@ -48,7 +63,7 @@ func (h *JSONHandler) TryHandle(d []byte) bool { var ok bool for _, field := range supportedTimeFields { - ok = bytes.Contains(d, []byte(`"`+field+`":`)) + ok = bytes.Contains(d, []byte(field)) if ok { break } @@ -58,8 +73,7 @@ func (h *JSONHandler) TryHandle(d []byte) bool { return false } - err := h.UnmarshalJSON(d) - if err != nil { + if !h.UnmarshalJSON(d) { h.clear() return false } @@ -67,52 +81,46 @@ func (h *JSONHandler) TryHandle(d []byte) bool { } // UnmarshalJSON sets the fields of the handler. -func (h *JSONHandler) UnmarshalJSON(data []byte) error { +func (h *JSONHandler) UnmarshalJSON(data []byte) bool { raw := make(map[string]interface{}) err := json.Unmarshal(data, &raw) if err != nil { - return err + return false } - var time interface{} - var ok bool - - for _, field := range supportedTimeFields { - time, ok = raw[field] + checkEachUntilFound(supportedLevelFields, func(field string) bool { + time, ok := tryParseTime(raw[field]) if ok { + h.Time = time delete(raw, field) - break } - } + return ok + }) - if ok { - h.Time, ok = tryParseTime(time) - if !ok { - return fmt.Errorf("field time is not a known timestamp: %v", time) + checkEachUntilFound(supportedMessageFields, func(field string) bool { + msg, ok := raw[field].(string) + if ok { + h.Message = msg + delete(raw, field) } - } + return ok + }) - if h.Message, ok = raw["msg"].(string); ok { - delete(raw, "msg") - } else if h.Message, ok = raw["message"].(string); ok { - delete(raw, "message") - } - - h.Level, ok = raw["level"].(string) - if !ok { - h.Level, ok = raw["lvl"].(string) - delete(raw, "lvl") + checkEachUntilFound(supportedLevelFields, func(field string) bool { + lvl, ok := raw[field] if !ok { - // bunyan uses numerical log levels - level, ok := raw["level"].(float64) - if ok { - h.Level = convertBunyanLogLevel(level) - delete(raw, "level") - } else { - h.Level = "???" - } + return false } - } + if strLvl, ok := lvl.(string); ok { + h.Level = strLvl + } else if flLvl, ok := lvl.(float64); ok { + h.Level = convertBunyanLogLevel(flLvl) + } else { + h.Level = "???" + } + delete(raw, field) + return true + }) if h.Fields == nil { h.Fields = make(map[string]string) @@ -134,7 +142,7 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) error { } } - return nil + return true } // Prettify the output in a logrus like fashion. diff --git a/logrus_handler.go b/logfmt_handler.go similarity index 63% rename from logrus_handler.go rename to logfmt_handler.go index 26dfc33..1669311 100644 --- a/logrus_handler.go +++ b/logfmt_handler.go @@ -10,10 +10,11 @@ import ( "time" "github.com/fatih/color" + "github.com/go-logfmt/logfmt" ) -// LogrusHandler can handle logs emmited by logrus.TextFormatter loggers. -type LogrusHandler struct { +// LogfmtHandler can handle logs emmited by logrus.TextFormatter loggers. +type LogfmtHandler struct { buf *bytes.Buffer out *tabwriter.Writer truncKV int @@ -28,52 +29,95 @@ type LogrusHandler struct { last map[string]string } -func (h *LogrusHandler) clear() { +func (h *LogfmtHandler) clear() { h.Level = "" h.Time = time.Time{} h.Message = "" h.last = h.Fields h.Fields = make(map[string]string) - h.buf.Reset() + if h.buf != nil { + h.buf.Reset() + } } // CanHandle tells if this line can be handled by this handler. -func (h *LogrusHandler) CanHandle(d []byte) bool { - if !(bytes.Contains(d, []byte(`level=`)) || bytes.Contains(d, []byte(`lvl=`))) { - return false +func (h *LogfmtHandler) TryHandle(d []byte) bool { + var ok bool + for _, field := range supportedTimeFields { + ok = bytes.Contains(d, []byte(field)) + if ok { + break + } } - if !(bytes.Contains(d, []byte(`time=`)) || bytes.Contains(d, []byte(`ts=`))) { + + if !ok { return false } - if !(bytes.Contains(d, []byte(`message=`)) || bytes.Contains(d, []byte(`msg=`))) { + + err := h.UnmarshalLogfmt(d) + if err != nil { + h.clear() return false } return true } // HandleLogfmt sets the fields of the handler. -func (h *LogrusHandler) visit(key, val []byte) bool { - switch { - case bytes.Equal(key, []byte("level")): - h.setLevel(val) - case bytes.Equal(key, []byte("lvl")): - h.setLevel(val) - case bytes.Equal(key, []byte("msg")): - h.setMessage(val) - case bytes.Equal(key, []byte("message")): - h.setMessage(val) - case bytes.Equal(key, []byte("time")): - h.setTime(val) - case bytes.Equal(key, []byte("ts")): - h.setTime(val) - default: - h.setField(key, val) +func (h *LogfmtHandler) UnmarshalLogfmt(data []byte) error { + + dec := logfmt.NewDecoder(bytes.NewReader(data)) + for dec.ScanRecord() { + next_kv: + for dec.ScanKeyval() { + key := dec.Key() + val := dec.Value() + if h.Time.IsZero() { + foundTime := checkEachUntilFound(supportedLevelFields, func(field string) bool { + time, ok := tryParseTime(string(val)) + if ok { + h.Time = time + } + return ok + }) + if foundTime { + continue next_kv + } + } + + if len(h.Message) == 0 { + foundMessage := checkEachUntilFound(supportedMessageFields, func(field string) bool { + if !bytes.Equal(key, []byte(field)) { + return false + } + h.Message = string(val) + return true + }) + if foundMessage { + continue next_kv + } + } + + if len(h.Level) == 0 { + foundLevel := checkEachUntilFound(supportedLevelFields, func(field string) bool { + if !bytes.Equal(key, []byte(field)) { + return false + } + h.Level = string(val) + return true + }) + if foundLevel { + continue next_kv + } + } + + h.setField(key, val) + } } - return true + return dec.Err() } // Prettify the output in a logrus like fashion. -func (h *LogrusHandler) Prettify(skipUnchanged bool) []byte { +func (h *LogfmtHandler) Prettify(skipUnchanged bool) []byte { defer h.clear() if h.out == nil { if h.Opts == nil { @@ -137,9 +181,9 @@ func (h *LogrusHandler) Prettify(skipUnchanged bool) []byte { return h.buf.Bytes() } -func (h *LogrusHandler) setLevel(val []byte) { h.Level = string(val) } -func (h *LogrusHandler) setMessage(val []byte) { h.Message = string(val) } -func (h *LogrusHandler) setTime(val []byte) (parsed bool) { +func (h *LogfmtHandler) setLevel(val []byte) { h.Level = string(val) } +func (h *LogfmtHandler) setMessage(val []byte) { h.Message = string(val) } +func (h *LogfmtHandler) setTime(val []byte) (parsed bool) { valStr := string(val) if valFloat, err := strconv.ParseFloat(valStr, 64); err == nil { h.Time, parsed = tryParseTime(valFloat) @@ -149,14 +193,14 @@ func (h *LogrusHandler) setTime(val []byte) (parsed bool) { return } -func (h *LogrusHandler) setField(key, val []byte) { +func (h *LogfmtHandler) setField(key, val []byte) { if h.Fields == nil { h.Fields = make(map[string]string) } h.Fields[string(key)] = string(val) } -func (h *LogrusHandler) joinKVs(skipUnchanged bool, sep string) []string { +func (h *LogfmtHandler) joinKVs(skipUnchanged bool, sep string) []string { kv := make([]string, 0, len(h.Fields)) for k, v := range h.Fields { diff --git a/scanner.go b/scanner.go index 597ca44..14bcde2 100644 --- a/scanner.go +++ b/scanner.go @@ -4,8 +4,6 @@ import ( "bufio" "bytes" "io" - - "github.com/aybabtme/humanlog/parser/logfmt" ) var ( @@ -21,10 +19,10 @@ func Scanner(src io.Reader, dst io.Writer, opts *HandlerOptions) error { var line uint64 - var lastLogrus bool + var lastLogfmt bool var lastJSON bool - logrusEntry := LogrusHandler{Opts: opts} + logfmtEntry := LogfmtHandler{Opts: opts} jsonEntry := JSONHandler{Opts: opts} for in.Scan() { @@ -40,12 +38,12 @@ func Scanner(src io.Reader, dst io.Writer, opts *HandlerOptions) error { dst.Write(jsonEntry.Prettify(opts.SkipUnchanged && lastJSON)) lastJSON = true - case logrusEntry.CanHandle(lineData) && logfmt.Parse(lineData, true, true, logrusEntry.visit): - dst.Write(logrusEntry.Prettify(opts.SkipUnchanged && lastLogrus)) - lastLogrus = true + case logfmtEntry.TryHandle(lineData): + dst.Write(logfmtEntry.Prettify(opts.SkipUnchanged && lastLogfmt)) + lastLogfmt = true default: - lastLogrus = false + lastLogfmt = false lastJSON = false dst.Write(lineData) }