-
Notifications
You must be signed in to change notification settings - Fork 3.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(logql): Support drop labels in logql pipeline (#7975)
- Loading branch information
Showing
11 changed files
with
1,048 additions
and
527 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| package log | ||
|
|
||
| import ( | ||
| "github.com/grafana/loki/pkg/logqlmodel" | ||
| "github.com/prometheus/prometheus/model/labels" | ||
| ) | ||
|
|
||
| type DropLabels struct { | ||
| dropLabels []DropLabel | ||
| } | ||
|
|
||
| type DropLabel struct { | ||
| Matcher *labels.Matcher | ||
| Name string | ||
| } | ||
|
|
||
| func NewDropLabel(matcher *labels.Matcher, name string) DropLabel { | ||
| return DropLabel{ | ||
| Matcher: matcher, | ||
| Name: name, | ||
| } | ||
| } | ||
|
|
||
| func NewDropLabels(dl []DropLabel) *DropLabels { | ||
| return &DropLabels{dropLabels: dl} | ||
| } | ||
|
|
||
| func (dl *DropLabels) Process(ts int64, line []byte, lbls *LabelsBuilder) ([]byte, bool) { | ||
| for _, dropLabel := range dl.dropLabels { | ||
| if dropLabel.Matcher != nil { | ||
| dropLabelMatches(dropLabel.Matcher, lbls) | ||
| continue | ||
| } | ||
| name := dropLabel.Name | ||
| dropLabelNames(name, lbls) | ||
| } | ||
| return line, true | ||
| } | ||
|
|
||
| func (dl *DropLabels) RequiredLabelNames() []string { return []string{} } | ||
|
|
||
| func isErrorLabel(name string) bool { | ||
| return name == logqlmodel.ErrorLabel | ||
| } | ||
|
|
||
| func isErrorDetailsLabel(name string) bool { | ||
| return name == logqlmodel.ErrorDetailsLabel | ||
| } | ||
|
|
||
| func dropLabelNames(name string, lbls *LabelsBuilder) { | ||
| if isErrorLabel(name) { | ||
| lbls.ResetError() | ||
| return | ||
| } | ||
| if isErrorDetailsLabel(name) { | ||
| lbls.ResetErrorDetails() | ||
| return | ||
| } | ||
| if _, ok := lbls.Get(name); ok { | ||
| lbls.Del(name) | ||
| } | ||
| } | ||
|
|
||
| func dropLabelMatches(matcher *labels.Matcher, lbls *LabelsBuilder) { | ||
| var value string | ||
| name := matcher.Name | ||
| if isErrorLabel(name) { | ||
| value = lbls.GetErr() | ||
| if matcher.Matches(value) { | ||
| lbls.ResetError() | ||
| } | ||
| return | ||
| } | ||
| if isErrorDetailsLabel(name) { | ||
| value = lbls.GetErrorDetails() | ||
| if matcher.Matches(value) { | ||
| lbls.ResetErrorDetails() | ||
| } | ||
| return | ||
| } | ||
| value, _ = lbls.Get(name) | ||
| if matcher.Matches(value) { | ||
| lbls.Del(name) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,160 @@ | ||
| package log | ||
|
|
||
| import ( | ||
| "sort" | ||
| "testing" | ||
|
|
||
| "github.com/prometheus/prometheus/model/labels" | ||
| "github.com/stretchr/testify/require" | ||
|
|
||
| "github.com/grafana/loki/pkg/logqlmodel" | ||
| ) | ||
|
|
||
| func Test_DropLabels(t *testing.T) { | ||
| tests := []struct { | ||
| Name string | ||
| dropLabels []DropLabel | ||
| err string | ||
| errDetails string | ||
| lbs labels.Labels | ||
| want labels.Labels | ||
| }{ | ||
| { | ||
| "drop by name", | ||
| []DropLabel{ | ||
| { | ||
| nil, | ||
| "app", | ||
| }, | ||
| { | ||
| nil, | ||
| "namespace", | ||
| }, | ||
| }, | ||
| "", | ||
| "", | ||
| labels.Labels{ | ||
| {Name: "app", Value: "foo"}, | ||
| {Name: "namespace", Value: "prod"}, | ||
| {Name: "pod_uuid", Value: "foo"}, | ||
| }, | ||
| labels.Labels{ | ||
| {Name: "pod_uuid", Value: "foo"}, | ||
| }, | ||
| }, | ||
| { | ||
| "drop by __error__", | ||
| []DropLabel{ | ||
| { | ||
| labels.MustNewMatcher(labels.MatchEqual, logqlmodel.ErrorLabel, errJSON), | ||
| "", | ||
| }, | ||
| { | ||
| nil, | ||
| "__error_details__", | ||
| }, | ||
| }, | ||
| errJSON, | ||
| "json error", | ||
| labels.Labels{ | ||
| {Name: "app", Value: "foo"}, | ||
| {Name: "namespace", Value: "prod"}, | ||
| {Name: "pod_uuid", Value: "foo"}, | ||
| }, | ||
| labels.Labels{ | ||
| {Name: "app", Value: "foo"}, | ||
| {Name: "namespace", Value: "prod"}, | ||
| {Name: "pod_uuid", Value: "foo"}, | ||
| }, | ||
| }, | ||
| { | ||
| "drop with wrong __error__ value", | ||
| []DropLabel{ | ||
| { | ||
| labels.MustNewMatcher(labels.MatchEqual, logqlmodel.ErrorLabel, errLogfmt), | ||
| "", | ||
| }, | ||
| }, | ||
| errJSON, | ||
| "json error", | ||
| labels.Labels{ | ||
| {Name: "app", Value: "foo"}, | ||
| {Name: "namespace", Value: "prod"}, | ||
| {Name: "pod_uuid", Value: "foo"}, | ||
| }, | ||
| labels.Labels{ | ||
| {Name: "app", Value: "foo"}, | ||
| {Name: "namespace", Value: "prod"}, | ||
| {Name: "pod_uuid", Value: "foo"}, | ||
| {Name: logqlmodel.ErrorLabel, Value: errJSON}, | ||
| {Name: logqlmodel.ErrorDetailsLabel, Value: "json error"}, | ||
| }, | ||
| }, | ||
| { | ||
| "drop by __error_details__", | ||
| []DropLabel{ | ||
| { | ||
| labels.MustNewMatcher(labels.MatchRegexp, logqlmodel.ErrorDetailsLabel, "expecting json.*"), | ||
| "", | ||
| }, | ||
| { | ||
| nil, | ||
| "__error__", | ||
| }, | ||
| }, | ||
| errJSON, | ||
| "expecting json object but it is not", | ||
| labels.Labels{ | ||
| {Name: "app", Value: "foo"}, | ||
| {Name: "namespace", Value: "prod"}, | ||
| {Name: "pod_uuid", Value: "foo"}, | ||
| }, | ||
| labels.Labels{ | ||
| {Name: "app", Value: "foo"}, | ||
| {Name: "namespace", Value: "prod"}, | ||
| {Name: "pod_uuid", Value: "foo"}, | ||
| }, | ||
| }, | ||
| { | ||
| "drop labels with names and matcher", | ||
| []DropLabel{ | ||
| { | ||
| labels.MustNewMatcher(labels.MatchEqual, logqlmodel.ErrorLabel, errJSON), | ||
| "", | ||
| }, | ||
| { | ||
| nil, | ||
| "__error_details__", | ||
| }, | ||
| { | ||
| nil, | ||
| "app", | ||
| }, | ||
| { | ||
| nil, | ||
| "namespace", | ||
| }, | ||
| }, | ||
| errJSON, | ||
| "json error", | ||
| labels.Labels{ | ||
| {Name: "app", Value: "foo"}, | ||
| {Name: "namespace", Value: "prod"}, | ||
| {Name: "pod_uuid", Value: "foo"}, | ||
| }, | ||
| labels.Labels{ | ||
| {Name: "pod_uuid", Value: "foo"}, | ||
| }, | ||
| }, | ||
| } | ||
| for _, tt := range tests { | ||
| dropLabels := NewDropLabels(tt.dropLabels) | ||
| lbls := NewBaseLabelsBuilder().ForLabels(tt.lbs, tt.lbs.Hash()) | ||
| lbls.Reset() | ||
| lbls.SetErr(tt.err) | ||
| lbls.SetErrorDetails(tt.errDetails) | ||
| dropLabels.Process(0, []byte(""), lbls) | ||
| sort.Sort(tt.want) | ||
| require.Equal(t, tt.want, lbls.LabelsResult().Labels()) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.