NETOBSERV-1185 Console plugin async overview metrics (#389)

* Console plugin async overview metrics

* fix fetchOverview default query condition

* query summary row align center

mocks/updateMocks.sh

@@ -2,45 +2,45 @@
 # Simply run your loki on http://localhost:3100
 # or port forward it using 'oc port-forward service/loki 3100:3100 -n netobserv'
 
-# flows.json contains query result for table display
+# flow_records.json contains query result for table display
 echo 'Getting table flows'
 curl 'http://localhost:3100/loki/api/v1/query_range?query=\{app=%22netobserv-flowcollector%22\}&limit=50'\
- | jq > ./loki/flows.json
+ | jq > ./loki/flow_records.json
 curl 'http://localhost:3100/loki/api/v1/query_range?query=\{app=%22netobserv-flowcollector%22\}|~`Packets%22:0\[,\}\]`|~`PktDropPackets%22:\[1-9\]*\[,\}\]`&limit=50'\
- | jq > ./loki/flows_dropped.json
+ | jq > ./loki/flow_records_dropped.json
 curl 'http://localhost:3100/loki/api/v1/query_range?query=\{app=%22netobserv-flowcollector%22\}|~`PktDropPackets%22:\[1-9\]*\[,\}\]`&limit=50'\
- | jq > ./loki/flows_has_dropped.json
+ | jq > ./loki/flow_records_has_dropped.json
 curl 'http://localhost:3100/loki/api/v1/query_range?query=\{app=%22netobserv-flowcollector%22\}|~`Packets%22:\[1-9\]*\[,\}\]`&limit=50'\
- | jq > ./loki/flows_sent.json
+ | jq > ./loki/flow_records_sent.json
 
-# topology_*.json contains queries result for topology display
+# flow_metrics_*.json contains queries result for overview / topology display
 echo 'Getting metrics'
 curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(app)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=5&step=360s'\
- | jq > ./loki/topology_app.json
+ | jq > ./loki/flow_metrics_app.json
 curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_HostName,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
- | jq > ./loki/topology_host.json
+ | jq > ./loki/flow_metrics_host.json
 curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
- | jq > ./loki/topology_namespace.json
+ | jq > ./loki/flow_metrics_namespace.json
 curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_OwnerName,SrcK8S_OwnerType,DstK8S_OwnerName,DstK8S_OwnerType,SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
- | jq > ./loki/topology_owner.json
+ | jq > ./loki/flow_metrics_owner.json
 curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20Packets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
- | jq > ./loki/topology_resource.json
+ | jq > ./loki/flow_metrics_resource.json
 
 echo 'Getting dropped metrics'
 curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(app)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\
- | jq > ./loki/topology_dropped_app.json
+ | jq > ./loki/flow_metrics_dropped_app.json
 curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(PktDropLatestState)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\
- | jq > ./loki/topology_dropped_state.json
+ | jq > ./loki/flow_metrics_dropped_state.json
 curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(5,sum%20by(PktDropLatestDropCause)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=5&step=360s'\
- | jq > ./loki/topology_dropped_cause.json
+ | jq > ./loki/flow_metrics_dropped_cause.json
 curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_HostName,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
- | jq > ./loki/topology_dropped_host.json
+ | jq > ./loki/flow_metrics_dropped_host.json
 curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
- | jq > ./loki/topology_dropped_namespace.json
+ | jq > ./loki/flow_metrics_dropped_namespace.json
 curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_OwnerName,SrcK8S_OwnerType,DstK8S_OwnerName,DstK8S_OwnerType,SrcK8S_Namespace,DstK8S_Namespace)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
- | jq > ./loki/topology_dropped_owner.json
+ | jq > ./loki/flow_metrics_dropped_owner.json
 curl 'http://localhost:3100/loki/api/v1/query_range?query=topk(50,sum%20by(SrcK8S_Name,SrcK8S_Type,SrcK8S_OwnerName,SrcK8S_OwnerType,SrcK8S_Namespace,SrcAddr,SrcK8S_HostName,DstK8S_Name,DstK8S_Type,DstK8S_OwnerName,DstK8S_OwnerType,DstK8S_Namespace,DstAddr,DstK8S_HostName)%20(rate(\{app=%22netobserv-flowcollector%22,FlowDirection=%221%22\}|~`Duplicate%22:false`|json|unwrap%20PktDropPackets|__error__=%22%22\[720s\])))&limit=50&step=360s'\
- | jq > ./loki/topology_dropped_resource.json
+ | jq > ./loki/flow_metrics_dropped_resource.json
 
 # namespaces.json contains label values for autocomplete
 echo 'Getting namespaces'

pkg/handler/lokiclientmock/loki_client_mock.go

@@ -21,7 +21,7 @@ func (o *LokiClientMock) Get(url string) ([]byte, int, error) {
 		path = "mocks/loki/namespaces.json"
 	} else {
 		if strings.Contains(url, "query=topk") {
-			path = "mocks/loki/topology"
+			path = "mocks/loki/flow_metrics"
 
 			if strings.Contains(url, "|unwrap%20PktDrop") {
 				path += "_dropped"
@@ -43,7 +43,7 @@ func (o *LokiClientMock) Get(url string) ([]byte, int, error) {
 				path += "_resource.json"
 			}
 		} else {
-			path = "mocks/loki/flows"
+			path = "mocks/loki/flow_records"
 			if strings.Contains(url, "|~`\"Packets\":0[,}]|~`\"PktDropPackets\":[1-9][0-9]*[,}]") {
 				path += "_dropped.json"
 			} else if strings.Contains(url, "|~`\"PktDropPackets\":[1-9][0-9]*[,}]") {

pkg/server/routes.go

@@ -33,9 +33,9 @@ func setupRoutes(cfg *Config, authChecker auth.Checker) *mux.Router {
 	api.HandleFunc("/loki/metrics", handler.LokiMetrics(&cfg.Loki))
 	api.HandleFunc("/loki/buildinfo", handler.LokiBuildInfos(&cfg.Loki))
 	api.HandleFunc("/loki/config/limits", handler.LokiConfig(&cfg.Loki, "limits_config"))
-	api.HandleFunc("/loki/flows", handler.GetFlows(&cfg.Loki))
+	api.HandleFunc("/loki/flow/records", handler.GetFlows(&cfg.Loki))
+	api.HandleFunc("/loki/flow/metrics", handler.GetTopology(&cfg.Loki))
 	api.HandleFunc("/loki/export", handler.ExportFlows(&cfg.Loki))
-	api.HandleFunc("/loki/topology", handler.GetTopology(&cfg.Loki))
 	api.HandleFunc("/resources/namespaces", handler.GetNamespaces(&cfg.Loki))
 	api.HandleFunc("/resources/namespace/{namespace}/kind/{kind}/names", handler.GetNames(&cfg.Loki))
 	api.HandleFunc("/resources/kind/{kind}/names", handler.GetNames(&cfg.Loki))

pkg/server/server_flows_test.go

@@ -241,7 +241,7 @@ func TestLokiFiltering(t *testing.T) {
 		t.Run(tc.inputPath, func(t *testing.T) {
 			// WHEN the Loki flows endpoint is queried in the backend
 			now := time.Now().Unix()
-			res, err := backendSvc.Client().Get(backendSvc.URL + "/api/loki/flows" + tc.inputPath)
+			res, err := backendSvc.Client().Get(backendSvc.URL + "/api/loki/flow/records" + tc.inputPath)
 			require.NoError(t, err)
 			body, err := io.ReadAll(res.Body)
 			require.NoError(t, err)

pkg/server/server_test.go

@@ -141,7 +141,7 @@ func TestServerUnauthorized(t *testing.T) {
 	require.Error(t, err)
 	require.Equal(t, "missing Authorization header", msg)
 
-	msg, err = getRequestResults(t, httpClient, serverURL+"/api/loki/flows")
+	msg, err = getRequestResults(t, httpClient, serverURL+"/api/loki/flow/records")
 	require.Error(t, err)
 	require.Equal(t, "missing Authorization header", msg)
 }
@@ -274,7 +274,7 @@ func TestLokiConfiguration(t *testing.T) {
 	defer backendSvc.Close()
 
 	// WHEN the Loki flows endpoint is queried in the backend
-	resp, err := backendSvc.Client().Get(backendSvc.URL + "/api/loki/flows")
+	resp, err := backendSvc.Client().Get(backendSvc.URL + "/api/loki/flow/records")
 	require.NoError(t, err)
 
 	// THEN the query has been properly forwarded to Loki
@@ -317,7 +317,7 @@ func TestLokiConfigurationForTopology(t *testing.T) {
 	defer backendSvc.Close()
 
 	// WHEN the Loki flows endpoint is queried in the backend
-	resp, err := backendSvc.Client().Get(backendSvc.URL + "/api/loki/topology")
+	resp, err := backendSvc.Client().Get(backendSvc.URL + "/api/loki/flow/metrics")
 	require.NoError(t, err)
 
 	// THEN the query has been properly forwarded to Loki
@@ -373,7 +373,7 @@ func TestLokiConfigurationForTableHistogram(t *testing.T) {
 	defer backendSvc.Close()
 
 	// WHEN the Loki flows endpoint is queried in the backend using count type
-	resp, err := backendSvc.Client().Get(backendSvc.URL + "/api/loki/topology?type=count")
+	resp, err := backendSvc.Client().Get(backendSvc.URL + "/api/loki/flow/metrics?type=count")
 	require.NoError(t, err)
 
 	// THEN the query has been properly forwarded to Loki
@@ -441,7 +441,7 @@ func TestLokiConfiguration_MultiTenant(t *testing.T) {
 	defer backendSvc.Close()
 
 	// WHEN the Loki flows endpoint is queried in the backend
-	_, err = backendSvc.Client().Get(backendSvc.URL + "/api/loki/flows")
+	_, err = backendSvc.Client().Get(backendSvc.URL + "/api/loki/flow/records")
 	require.NoError(t, err)
 
 	// THEN the query has been properly forwarded to Loki with the tenant ID header
@@ -454,7 +454,7 @@ func TestLokiConfiguration_MultiTenant(t *testing.T) {
 	require.NoError(t, err)
 
 	// RUN another query
-	_, err = backendSvc.Client().Get(backendSvc.URL + "/api/loki/flows")
+	_, err = backendSvc.Client().Get(backendSvc.URL + "/api/loki/flow/records")
 	require.NoError(t, err)
 
 	// THEN Bearer token is correctly updated

web/locales/en/plugin__netobserv-plugin.json

@@ -209,8 +209,8 @@
   "Reset": "Reset",
   "Select a custom time range. Flows are selected based on their End Time value.": "Select a custom time range. Flows are selected based on their End Time value.",
   "No results found": "No results found",
-  "Unable to get overview": "Unable to get overview",
   "Clear or reset filters and try again.": "Clear or reset filters and try again.",
+  "Unable to get overview": "Unable to get overview",
   "Average latency": "Average latency",
   "Average RTT": "Average RTT",
   "Total flow count": "Total flow count",
@@ -300,7 +300,6 @@
   "Filtered ended conversations count": "Filtered ended conversations count",
   "Flows": "Flows",
   "Ended conversations": "Ended conversations",
-  "Last refresh: {{time}}": "Last refresh: {{time}}",
   "See less": "See less",
   "See more": "See more",
   "Filtered sum of top-k bytes / filtered total bytes": "Filtered sum of top-k bytes / filtered total bytes",
@@ -311,6 +310,10 @@
   "Filtered avg RTT / filtered total avg RTT": "Filtered avg RTT / filtered total avg RTT",
   "Filtered avg RTT": "Filtered avg RTT",
   "ms": "ms",
+  "Last refresh: {{time}}": "Last refresh: {{time}}",
+  "running": "running",
+  "queries": "queries",
+  "query": "query",
   "Configuration": "Configuration",
   "Sampling": "Sampling",
   "Version": "Version",

web/src/api/loki.ts

@@ -26,7 +26,7 @@ export class RecordsResult {
   stats: Stats;
 }
 
-export class TopologyMetricsResult {
+export class FlowMetricsResult {
   metrics: TopologyMetrics[];
   stats: Stats;
 }

web/src/api/routes.ts

@@ -14,11 +14,11 @@ import {
   RecordsResult,
   Stats,
   StreamResult,
-  TopologyMetricsResult
+  FlowMetricsResult
 } from './loki';
 
-export const getFlows = (params: FlowQuery): Promise<RecordsResult> => {
-  return axios.get(ContextSingleton.getHost() + '/api/loki/flows', { params }).then(r => {
+export const getFlowRecords = (params: FlowQuery): Promise<RecordsResult> => {
+  return axios.get(ContextSingleton.getHost() + '/api/loki/flow/records', { params }).then(r => {
     if (r.status >= 400) {
       throw new Error(`${r.statusText} [code=${r.status}]`);
     }
@@ -74,8 +74,8 @@ export const getResources = (namespace: string, kind: string): Promise<string[]>
   });
 };
 
-export const getTopologyMetrics = (params: FlowQuery, range: number | TimeRange): Promise<TopologyMetricsResult> => {
-  return getMetricsGeneric(params, res => {
+export const getFlowMetrics = (params: FlowQuery, range: number | TimeRange): Promise<FlowMetricsResult> => {
+  return getFlowMetricsGeneric(params, res => {
     return parseTopologyMetrics(
       res.result as RawTopologyMetrics[],
       range,
@@ -87,8 +87,8 @@ export const getTopologyMetrics = (params: FlowQuery, range: number | TimeRange)
   });
 };
 
-export const getGenericMetrics = (params: FlowQuery, range: number | TimeRange): Promise<GenericMetricsResult> => {
-  return getMetricsGeneric(params, res => {
+export const getFlowGenericMetrics = (params: FlowQuery, range: number | TimeRange): Promise<GenericMetricsResult> => {
+  return getFlowMetricsGeneric(params, res => {
     return parseGenericMetrics(
       res.result as RawTopologyMetrics[],
       range,
@@ -99,11 +99,11 @@ export const getGenericMetrics = (params: FlowQuery, range: number | TimeRange):
   });
 };
 
-const getMetricsGeneric = <T>(
+const getFlowMetricsGeneric = <T>(
   params: FlowQuery,
   mapper: (raw: AggregatedQueryResponse) => T
 ): Promise<{ metrics: T; stats: Stats }> => {
-  return axios.get(ContextSingleton.getHost() + '/api/loki/topology', { params }).then(r => {
+  return axios.get(ContextSingleton.getHost() + '/api/loki/flow/metrics', { params }).then(r => {
     if (r.status >= 400) {
       throw new Error(`${r.statusText} [code=${r.status}]`);
     }
@@ -148,7 +148,7 @@ export const getLokiReady = (): Promise<string> => {
   });
 };
 
-export const getMetrics = (): Promise<string> => {
+export const getLokiMetrics = (): Promise<string> => {
   return axios.get(ContextSingleton.getHost() + '/api/loki/metrics').then(r => {
     if (r.status >= 400) {
       throw new Error(`${r.statusText} [code=${r.status}]`);

web/src/components/__tests-data__/config.ts

@@ -1,10 +1,21 @@
-export const ConfigResultSample = {
-  recordTypes: ['flowLog'],
+export const FullConfigResultSample = {
+  recordTypes: ['flowLog', 'newConnection', 'heartbeat', 'endConnection'],
   portNaming: {
     enable: true,
     portNames: new Map([['3100', 'loki']])
   },
   quickFilters: [],
   sampling: 1,
-  features: ['pktDrop', 'dnsTracking']
+  features: ['pktDrop', 'dnsTracking', 'flowRTT']
+};
+
+export const SimpleConfigResultSample = {
+  recordTypes: ['flowLog'],
+  portNaming: {
+    enable: true,
+    portNames: new Map([])
+  },
+  quickFilters: [],
+  sampling: 50,
+  features: []
 };

web/src/components/__tests-data__/flows.ts

@@ -1,6 +1,6 @@
 import { parseStream, RecordsResult, StreamResult } from '../../api/loki';
 import { FlowDirection, Record } from '../../api/ipfix';
-import flowsJson from '../../../../mocks/loki/flows.json';
+import flowsJson from '../../../../mocks/loki/flow_records.json';
 
 export const FlowsMock: Record[] = (flowsJson.data.result as StreamResult[]).flatMap(r => parseStream(r));
 

web/src/components/__tests__/netflow-tab.spec.tsx

@@ -7,19 +7,19 @@ import * as React from 'react';
 import NetflowTab from '../netflow-tab';
 import NetflowTraffic from '../netflow-traffic';
 import { PodTabParam, ServiceTabParam, UnknownTabParam } from '../__tests-data__/tabs';
-import { ConfigResultSample } from '../__tests-data__/config';
-import { GenericMetricsResult, TopologyMetricsResult } from '../../api/loki';
+import { FullConfigResultSample } from '../__tests-data__/config';
+import { GenericMetricsResult, FlowMetricsResult } from '../../api/loki';
 import { AlertsResult, SilencedAlert } from '../../api/alert';
 
 const useResolvedExtensionsMock = useResolvedExtensions as jest.Mock;
 
 jest.mock('../../api/routes', () => ({
-  getConfig: jest.fn(() => Promise.resolve(ConfigResultSample)),
-  getFlows: jest.fn(() => Promise.resolve([])),
-  getTopologyMetrics: jest.fn(() =>
-    Promise.resolve({ metrics: [], stats: { numQueries: 0, limitReached: false } } as TopologyMetricsResult)
+  getConfig: jest.fn(() => Promise.resolve(FullConfigResultSample)),
+  getFlowRecords: jest.fn(() => Promise.resolve([])),
+  getFlowMetrics: jest.fn(() =>
+    Promise.resolve({ metrics: [], stats: { numQueries: 0, limitReached: false } } as FlowMetricsResult)
   ),
-  getGenericMetrics: jest.fn(() =>
+  getFlowGenericMetrics: jest.fn(() =>
     Promise.resolve({ metrics: [], stats: { numQueries: 0, limitReached: false } } as GenericMetricsResult)
   ),
   getAlerts: jest.fn(() => Promise.resolve({ data: { groups: [] }, status: 'success' } as AlertsResult)),