From 711a37244fea5791eb5a02cec8841bbeacaa2d92 Mon Sep 17 00:00:00 2001 From: Mathew Heard Date: Sun, 12 Jun 2022 13:45:44 +1000 Subject: [PATCH 1/4] Extend json label extraction test --- pkg/logql/syntax/parser_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/logql/syntax/parser_test.go b/pkg/logql/syntax/parser_test.go index 0b3f9ca4ec6b1..d4c9e9f09e88d 100644 --- a/pkg/logql/syntax/parser_test.go +++ b/pkg/logql/syntax/parser_test.go @@ -2944,13 +2944,14 @@ func TestParse(t *testing.T) { }, }, { - in: `{app="foo"} | json response_code, api_key="request.headers[\"X-API-KEY\"]"`, + in: `{app="foo"} | json response_code, api_key="request.headers[\"X-API-KEY\"]", layer7_something_specific="layer7_something_specific"`, exp: &PipelineExpr{ Left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), MultiStages: MultiStageExpr{ newJSONExpressionParser([]log.JSONExpression{ log.NewJSONExpr("response_code", `response_code`), log.NewJSONExpr("api_key", `request.headers["X-API-KEY"]`), + log.NewJSONExpr("layer7_something_specific", `layer7_something_specific`), }), }, }, From 3cff1d0bb5fca425479af6be7b73b8256b1ec707 Mon Sep 17 00:00:00 2001 From: Mathew Heard Date: Sun, 12 Jun 2022 14:03:50 +1000 Subject: [PATCH 2/4] Add json label extraction to aggregation --- pkg/logql/syntax/parser_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pkg/logql/syntax/parser_test.go b/pkg/logql/syntax/parser_test.go index d4c9e9f09e88d..45c29935a9269 100644 --- a/pkg/logql/syntax/parser_test.go +++ b/pkg/logql/syntax/parser_test.go @@ -2956,6 +2956,23 @@ func TestParse(t *testing.T) { }, }, }, + { + in: `count_over_time({ foo ="bar" } | json layer7_something_specific="layer7_something_specific" [12m])`, + exp: &RangeAggregationExpr{ + Left: &LogRange{ + Left: &PipelineExpr{ + MultiStages: MultiStageExpr{ + newJSONExpressionParser([]log.JSONExpression{ + log.NewJSONExpr("layer7_something_specific", `layer7_something_specific`), + }), + }, + Left: &MatchersExpr{Mts: []*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}}, + }, + Interval: 12 * time.Minute, + }, + Operation: "count_over_time", + }, + }, } { t.Run(tc.in, func(t *testing.T) { ast, err := ParseExpr(tc.in) From 6cdbaec3b35fbda6631de8ade8bdcc8d90ea4809 Mon Sep 17 00:00:00 2001 From: Mathew Heard Date: Sun, 12 Jun 2022 14:54:42 +1000 Subject: [PATCH 3/4] fix lexeer issue, it's value to have numbers inside the field (just not at the start) Also simplified logic --- pkg/logql/log/jsonexpr/lexer.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/logql/log/jsonexpr/lexer.go b/pkg/logql/log/jsonexpr/lexer.go index fd30730d1e11b..86df0e5926a3e 100644 --- a/pkg/logql/log/jsonexpr/lexer.go +++ b/pkg/logql/log/jsonexpr/lexer.go @@ -68,7 +68,7 @@ func (sc *Scanner) lex(lval *JSONExprSymType) int { return RSB case r == '.': return DOT - case isIdentifier(r): + case isStartIdentifier(r): sc.unread() lval.field = sc.scanField() return FIELD @@ -83,21 +83,20 @@ func (sc *Scanner) lex(lval *JSONExprSymType) int { } } -func isIdentifier(r rune) bool { +func isStartIdentifier(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || r == '_' } +func isIdentifier(r rune) bool { + return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' +} + func (sc *Scanner) scanField() string { var str []rune for { r := sc.read() - if !isIdentifier(r) { - sc.unread() - break - } - - if r == '.' || isEndOfInput(r) { + if !isIdentifier(r) || isEndOfInput(r) { sc.unread() break } From d8588d4aab41c15ed8ef2b912e2bacdb2bc5fa70 Mon Sep 17 00:00:00 2001 From: Mathew Heard Date: Mon, 13 Jun 2022 22:33:07 +1000 Subject: [PATCH 4/4] integrate feedback from reviews --- CHANGELOG.md | 1 + pkg/logql/log/jsonexpr/jsonexpr_test.go | 6 ++++++ pkg/logql/log/jsonexpr/lexer.go | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad7fa3f7db70e..e642929c59580 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Main +* [6372](https://github.com/grafana/loki/pull/6372) **splitice**: Add support for numbers in JSON fields * [6136](https://github.com/grafana/loki/pull/6136) **periklis**: Add support for alertmanager header authorization * [6102](https://github.com/grafana/loki/pull/6102) **timchenko-a**: Add multi-tenancy support to lambda-promtail * [5971](https://github.com/grafana/loki/pull/5971) **kavirajk**: Record statistics about metadata queries such as labels and series queries in `metrics.go` as well diff --git a/pkg/logql/log/jsonexpr/jsonexpr_test.go b/pkg/logql/log/jsonexpr/jsonexpr_test.go index 8357372aa8ee8..a132cc7ba112d 100644 --- a/pkg/logql/log/jsonexpr/jsonexpr_test.go +++ b/pkg/logql/log/jsonexpr/jsonexpr_test.go @@ -119,6 +119,12 @@ func TestJSONExpressionParser(t *testing.T) { nil, fmt.Errorf("syntax error: unexpected $end, expecting RSB"), }, + { + "identifier with number", + `utf8`, + []interface{}{"utf8"}, + nil, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/logql/log/jsonexpr/lexer.go b/pkg/logql/log/jsonexpr/lexer.go index 86df0e5926a3e..f3ba6dcd9536b 100644 --- a/pkg/logql/log/jsonexpr/lexer.go +++ b/pkg/logql/log/jsonexpr/lexer.go @@ -88,7 +88,7 @@ func isStartIdentifier(r rune) bool { } func isIdentifier(r rune) bool { - return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' + return isStartIdentifier(r) || (r >= '0' && r <= '9') } func (sc *Scanner) scanField() string {