diff --git a/mocks/updateMocks.sh b/mocks/updateMocks.sh index c2d44e7bd..07832f3ea 100755 --- a/mocks/updateMocks.sh +++ b/mocks/updateMocks.sh @@ -15,35 +15,35 @@ curl 'http://localhost:3100/loki/api/v1/query_range?query=\{app=%22netobserv-flo # flow_metrics_*.json contains queries result for overview / topology display echo 'Getting metrics' -curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(app)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=5&step=360s'\ +curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(app)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=5&step=360s'\ | jq > ./loki/flow_metrics_app.json -curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(K8S_ClusterName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ +curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(K8S_ClusterName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ | jq > ./loki/flow_metrics_cluster.json - curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Zone,DstK8S_Zone)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ + curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Zone,DstK8S_Zone)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ | jq > ./loki/flow_metrics_zone.json -curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_HostName,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ +curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_HostName,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ | jq > ./loki/flow_metrics_host.json -curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ +curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ | jq > ./loki/flow_metrics_namespace.json -curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_OwnerName,SrcK8S_OwnerType,DstK8S_OwnerName,DstK8S_OwnerType,SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ +curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_OwnerName,SrcK8S_OwnerType,DstK8S_OwnerName,DstK8S_OwnerType,SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ | jq > ./loki/flow_metrics_owner.json -curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ +curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ | jq > ./loki/flow_metrics_resource.json echo 'Getting dropped metrics' -curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(app)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\ +curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(app)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\ | jq > ./loki/flow_metrics_dropped_app.json -curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(PktDropLatestState)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\ +curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(PktDropLatestState)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\ | jq > ./loki/flow_metrics_dropped_state.json -curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(PktDropLatestDropCause)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\ +curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(PktDropLatestDropCause)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\ | jq > ./loki/flow_metrics_dropped_cause.json -curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_HostName,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ +curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_HostName,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ | jq > ./loki/flow_metrics_dropped_host.json -curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ +curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ | jq > ./loki/flow_metrics_dropped_namespace.json -curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_OwnerName,SrcK8S_OwnerType,DstK8S_OwnerName,DstK8S_OwnerType,SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ +curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_OwnerName,SrcK8S_OwnerType,DstK8S_OwnerName,DstK8S_OwnerType,SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ | jq > ./loki/flow_metrics_dropped_owner.json -curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ +curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\ | jq > ./loki/flow_metrics_dropped_resource.json # namespaces.json contains label values for autocomplete diff --git a/pkg/loki/filter.go b/pkg/loki/filter.go index 9abe8776e..9e088bac5 100644 --- a/pkg/loki/filter.go +++ b/pkg/loki/filter.go @@ -43,11 +43,12 @@ type labelFilter struct { // lineFilter represents a condition based on a JSON raw text match. type lineFilter struct { - key string - strictKey bool - values []lineMatch - not bool - moreThan bool + key string + strictKey bool + values []lineMatch + not bool + allowEmpty bool + moreThan bool } type lineMatch struct { @@ -253,14 +254,15 @@ func moreThanRegex(sb *strings.Builder, value string) { // under construction (contained in the provided strings.Builder) func (f *lineFilter) writeInto(sb *strings.Builder) { if f.not { - // the record must contains the field if values are specified - // since FLP skip empty fields / zeros values - if len(f.values) > 0 { - sb.WriteString("|~`\"") - sb.WriteString(f.key) - sb.WriteString("\"`") + if !f.allowEmpty { + // the record must contains the field if values are specified + // since FLP skip empty fields / zeros values + if len(f.values) > 0 { + sb.WriteString("|~`\"") + sb.WriteString(f.key) + sb.WriteString("\"`") + } } - // then we exclude match results sb.WriteString("!~`") } else { diff --git a/pkg/loki/flow_query.go b/pkg/loki/flow_query.go index 57e9ccf4f..e20e3bbc2 100644 --- a/pkg/loki/flow_query.go +++ b/pkg/loki/flow_query.go @@ -57,13 +57,15 @@ func NewFlowQueryBuilder(cfg *Config, start, end, limit string, dedup bool, lineFilters := []lineFilter{} if dedup { if cfg.IsLabel(fields.Duplicate) { - labelFilters = append(labelFilters, stringEqualLabelFilter(fields.Duplicate, "false")) + labelFilters = append(labelFilters, notStringLabelFilter(fields.Duplicate, "true")) } else { lineFilters = append(lineFilters, lineFilter{ - key: fields.Duplicate, + key: fields.Duplicate, + not: true, + allowEmpty: true, values: []lineMatch{{ valueType: typeBool, - value: "false", + value: "true", }}, }) } diff --git a/pkg/server/server_flows_test.go b/pkg/server/server_flows_test.go index 784895661..e5c58d78d 100644 --- a/pkg/server/server_flows_test.go +++ b/pkg/server/server_flows_test.go @@ -65,12 +65,12 @@ func TestLokiFiltering(t *testing.T) { }, { inputPath: "?filters=" + url.QueryEscape("SrcPort=8080&SrcAddr=10.128.0.1&SrcK8S_Namespace=default") + "&dedup=true", outputQueries: []string{ - "?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*default.*\"}|~`Duplicate\":false`|~`SrcPort\":8080[,}]`|json|SrcAddr=ip(\"10.128.0.1\")", + "?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*default.*\"}!~`Duplicate\":true`|~`SrcPort\":8080[,}]`|json|SrcAddr=ip(\"10.128.0.1\")", }, }, { inputPath: "?filters=" + url.QueryEscape("SrcAddr=10.128.0.1&DstAddr=10.128.0.2") + "&dedup=true", outputQueryParts: []string{ - "?query={app=\"netobserv-flowcollector\"}|~`Duplicate\":false`|json", + "?query={app=\"netobserv-flowcollector\"}!~`Duplicate\":true`|json", "|SrcAddr=ip(\"10.128.0.1\")", "|DstAddr=ip(\"10.128.0.2\")", }, diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index cc41df676..056140f9c 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -375,8 +375,8 @@ func TestLokiConfigurationForTopology(t *testing.T) { req2 := lokiMock.Calls[1].Arguments[1].(*http.Request) queries := []string{req1.URL.Query().Get("query"), req2.URL.Query().Get("query")} expected := []string{ - `topk(100,sum by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)(rate({app="netobserv-flowcollector",FlowDirection=~"^0$|^2$"}|~` + "`" + `Duplicate":false` + "`" + `|json|unwrap Bytes|__error__=""[1m])))`, - `topk(100,sum by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)(rate({app="netobserv-flowcollector",FlowDirection="1",DstK8S_OwnerName=""}|~` + "`" + `Duplicate":false` + "`" + `|json|unwrap Bytes|__error__=""[1m])))`, + `topk(100,sum by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)(rate({app="netobserv-flowcollector",FlowDirection=~"^0$|^2$"}!~` + "`" + `Duplicate":true` + "`" + `|json|unwrap Bytes|__error__=""[1m])))`, + `topk(100,sum by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)(rate({app="netobserv-flowcollector",FlowDirection="1",DstK8S_OwnerName=""}!~` + "`" + `Duplicate":true` + "`" + `|json|unwrap Bytes|__error__=""[1m])))`, } // We don't predict the order so sort both actual and expected sort.Strings(queries) @@ -434,7 +434,7 @@ func TestLokiConfigurationForTableHistogram(t *testing.T) { req1 := lokiMock.Calls[0].Arguments[1].(*http.Request) query := req1.URL.Query().Get("query") expected := - `topk(100,sum by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)(count_over_time({app="netobserv-flowcollector"}|~` + "`" + `Duplicate":false` + "`" + `|json[30s])))` + `topk(100,sum by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)(count_over_time({app="netobserv-flowcollector"}!~` + "`" + `Duplicate":true` + "`" + `|json[30s])))` assert.Equal(t, expected, query) // without any multi-tenancy header