Skip to content

Commit

Permalink
rely on duplicate not true
Browse files Browse the repository at this point in the history
  • Loading branch information
@jpinsonneau
jpinsonneau committed Mar 13, 2024
1 parent 6485e1d commit ab6c749
Show file tree
Hide file tree
Showing 5 changed files with 38 additions and 34 deletions.
28 changes: 14 additions & 14 deletions mocks/updateMocks.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,35 +15,35 @@ curl 'http://localhost:3100/loki/api/v1/query_range?query=\{app=%22netobserv-flo

# flow_metrics_*.json contains queries result for overview / topology display
echo 'Getting metrics'
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(app)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=5&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(app)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=5&step=360s'\
| jq > ./loki/flow_metrics_app.json
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(K8S_ClusterName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(K8S_ClusterName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
| jq > ./loki/flow_metrics_cluster.json
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Zone,DstK8S_Zone)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Zone,DstK8S_Zone)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
| jq > ./loki/flow_metrics_zone.json
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_HostName,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_HostName,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
| jq > ./loki/flow_metrics_host.json
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
| jq > ./loki/flow_metrics_namespace.json
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_OwnerName,SrcK8S_OwnerType,DstK8S_OwnerName,DstK8S_OwnerType,SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_OwnerName,SrcK8S_OwnerType,DstK8S_OwnerName,DstK8S_OwnerType,SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
| jq > ./loki/flow_metrics_owner.json
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
| jq > ./loki/flow_metrics_resource.json

echo 'Getting dropped metrics'
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(app)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(app)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\
| jq > ./loki/flow_metrics_dropped_app.json
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(PktDropLatestState)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(PktDropLatestState)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\
| jq > ./loki/flow_metrics_dropped_state.json
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(PktDropLatestDropCause)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(PktDropLatestDropCause)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\
| jq > ./loki/flow_metrics_dropped_cause.json
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_HostName,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_HostName,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
| jq > ./loki/flow_metrics_dropped_host.json
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
| jq > ./loki/flow_metrics_dropped_namespace.json
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_OwnerName,SrcK8S_OwnerType,DstK8S_OwnerName,DstK8S_OwnerType,SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_OwnerName,SrcK8S_OwnerType,DstK8S_OwnerName,DstK8S_OwnerType,SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
| jq > ./loki/flow_metrics_dropped_owner.json
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}!~`Duplicate%22:true`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
| jq > ./loki/flow_metrics_dropped_resource.json

# namespaces.json contains label values for autocomplete
Expand Down
26 changes: 14 additions & 12 deletions pkg/loki/filter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,12 @@ type labelFilter struct {

// lineFilter represents a condition based on a JSON raw text match.
type lineFilter struct {
key string
strictKey bool
values []lineMatch
not bool
moreThan bool
key string
strictKey bool
values []lineMatch
not bool
allowEmpty bool
moreThan bool
}

type lineMatch struct {
Expand Down Expand Up @@ -253,14 +254,15 @@ func moreThanRegex(sb *strings.Builder, value string) {
// under construction (contained in the provided strings.Builder)
func (f *lineFilter) writeInto(sb *strings.Builder) {
if f.not {
// the record must contains the field if values are specified
// since FLP skip empty fields / zeros values
if len(f.values) > 0 {
sb.WriteString("|~`\"")
sb.WriteString(f.key)
sb.WriteString("\"`")
if !f.allowEmpty {
// the record must contains the field if values are specified
// since FLP skip empty fields / zeros values
if len(f.values) > 0 {
sb.WriteString("|~`\"")
sb.WriteString(f.key)
sb.WriteString("\"`")
}
}

// then we exclude match results
sb.WriteString("!~`")
} else {
Expand Down
8 changes: 5 additions & 3 deletions pkg/loki/flow_query.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,15 @@ func NewFlowQueryBuilder(cfg *Config, start, end, limit string, dedup bool,
lineFilters := []lineFilter{}
if dedup {
if cfg.IsLabel(fields.Duplicate) {
labelFilters = append(labelFilters, stringEqualLabelFilter(fields.Duplicate, "false"))
labelFilters = append(labelFilters, notStringLabelFilter(fields.Duplicate, "true"))
} else {
lineFilters = append(lineFilters, lineFilter{
key: fields.Duplicate,
key: fields.Duplicate,
not: true,
allowEmpty: true,
values: []lineMatch{{
valueType: typeBool,
value: "false",
value: "true",
}},
})
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/server/server_flows_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,12 @@ func TestLokiFiltering(t *testing.T) {
}, {
inputPath: "?filters=" + url.QueryEscape("SrcPort=8080&SrcAddr=10.128.0.1&SrcK8S_Namespace=default") + "&dedup=true",
outputQueries: []string{
"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*default.*\"}|~`Duplicate\":false`|~`SrcPort\":8080[,}]`|json|SrcAddr=ip(\"10.128.0.1\")",
"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*default.*\"}!~`Duplicate\":true`|~`SrcPort\":8080[,}]`|json|SrcAddr=ip(\"10.128.0.1\")",
},
}, {
inputPath: "?filters=" + url.QueryEscape("SrcAddr=10.128.0.1&DstAddr=10.128.0.2") + "&dedup=true",
outputQueryParts: []string{
"?query={app=\"netobserv-flowcollector\"}|~`Duplicate\":false`|json",
"?query={app=\"netobserv-flowcollector\"}!~`Duplicate\":true`|json",
"|SrcAddr=ip(\"10.128.0.1\")",
"|DstAddr=ip(\"10.128.0.2\")",
},
Expand Down
6 changes: 3 additions & 3 deletions pkg/server/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,8 +375,8 @@ func TestLokiConfigurationForTopology(t *testing.T) {
req2 := lokiMock.Calls[1].Arguments[1].(*http.Request)
queries := []string{req1.URL.Query().Get("query"), req2.URL.Query().Get("query")}
expected := []string{
`topk(100,sum by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)(rate({app="netobserv-flowcollector",FlowDirection=~"^0$|^2$"}|~` + "`" + `Duplicate":false` + "`" + `|json|unwrap Bytes|__error__=""[1m])))`,
`topk(100,sum by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)(rate({app="netobserv-flowcollector",FlowDirection="1",DstK8S_OwnerName=""}|~` + "`" + `Duplicate":false` + "`" + `|json|unwrap Bytes|__error__=""[1m])))`,
`topk(100,sum by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)(rate({app="netobserv-flowcollector",FlowDirection=~"^0$|^2$"}!~` + "`" + `Duplicate":true` + "`" + `|json|unwrap Bytes|__error__=""[1m])))`,
`topk(100,sum by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)(rate({app="netobserv-flowcollector",FlowDirection="1",DstK8S_OwnerName=""}!~` + "`" + `Duplicate":true` + "`" + `|json|unwrap Bytes|__error__=""[1m])))`,
}
// We don't predict the order so sort both actual and expected
sort.Strings(queries)
Expand Down Expand Up @@ -434,7 +434,7 @@ func TestLokiConfigurationForTableHistogram(t *testing.T) {
req1 := lokiMock.Calls[0].Arguments[1].(*http.Request)
query := req1.URL.Query().Get("query")
expected :=
`topk(100,sum by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)(count_over_time({app="netobserv-flowcollector"}|~` + "`" + `Duplicate":false` + "`" + `|json[30s])))`
`topk(100,sum by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)(count_over_time({app="netobserv-flowcollector"}!~` + "`" + `Duplicate":true` + "`" + `|json[30s])))`
assert.Equal(t, expected, query)

// without any multi-tenancy header
Expand Down

0 comments on commit ab6c749

Please sign in to comment.