NETOBSERV-1400 Feature filters are observed even though they are not enabled

Makefile

@@ -134,10 +134,11 @@ start-frontend-standalone: install-frontend ## Run frontend as standalone
 	cd web && npm run start:standalone
 
 .PHONY: start-standalone
-start-standalone: build-backend install-frontend ## Run backend and frontend as standalone
+start-standalone: YQ build-backend install-frontend ## Run backend and frontend as standalone
+	$(YQ) '.server.port |= 9002 | .server.metricsPort |= 9003 | .loki.useMocks |= false' ./config/sample-config.yaml > ./config/config.yaml
 	@echo "### Starting backend on http://localhost:9002"
 	bash -c "trap 'fuser -k 9002/tcp' EXIT; \
-					./plugin-backend -port 9002 -metrics-port 9003 $(CMDLINE_ARGS) & cd web && npm run start:standalone"
+					./plugin-backend $(CMDLINE_ARGS) & cd web && npm run start:standalone"
 
 .PHONY: start-standalone-mock
 start-standalone-mock: YQ build-backend install-frontend ## Run backend using mocks and frontend as standalone

config/sample-config.yaml

@@ -10,13 +10,20 @@ loki:
     - DstK8S_OwnerName
     - FlowDirection
     - Duplicate
-    - _RecordType
+#    - _RecordType
   tenantID: netobserv
   authCheck: none
   useMocks: false
 frontend: 
   recordTypes:
     - flowLog
+#    - newConnection
+#    - heartbeat
+#    - endConnection
+  features:
+#    - pktDrop
+#    - dnsTracking
+#    - flowRTT
   portNaming:
     enable: true
     portNames:
@@ -324,17 +331,17 @@ frontend:
       width: 15
     - id: K8S_Object
       name: Kubernetes Objects
-      calculated: '[getConcatenatedValue(SrcAddr,SrcPort,SrcK8S_Type,SrcK8S_Namespace,SrcK8S_Name),getConcatenatedValue(DstAddr,DstPort,DstK8S_Type,DstK8S_Namespace,DstK8S_Name)]'
+      calculated: '[column.SrcK8S_Object,column.DstK8S_Object]'
       default: false
       width: 15
     - id: K8S_OwnerObject
       name: Owner Kubernetes Objects
-      calculated: '[getConcatenatedValue(SrcAddr,SrcPort,SrcK8S_OwnerType,SrcK8S_Namespace,SrcK8S_OwnerName),getConcatenatedValue(DstAddr,DstPort,DstK8S_OwnerType,DstK8S_Namespace,DstK8S_OwnerName)]'
+      calculated: '[column.SrcK8S_OwnerObject,column.DstK8S_OwnerObject]'
       default: false
       width: 15
     - id: AddrPort
       name: IPs & Ports
-      calculated: '[getConcatenatedValue(SrcAddr,SrcPort),getConcatenatedValue(DstAddr,DstPort)]'
+      calculated: '[column.SrcAddrPort,column.DstAddrPort]'
       default: false
       width: 15
     - id: Proto
@@ -359,15 +366,15 @@ frontend:
       tooltip: The type of the ICMP message
       field: IcmpType
       filter: icmp_type
-      default: true
+      default: false
       width: 10
     - id: IcmpCode
       group: ICMP
       name: Code
       tooltip: The code of the ICMP message
       field: IcmpCode
-      quickFilter: icmp_code
-      default: true
+      filter: icmp_code
+      default: false
       width: 10
     - id: FlowDirection
       name: Direction
@@ -399,24 +406,20 @@ frontend:
     - id: FlowDuration
       name: Duration
       tooltip: Time elapsed between Start Time and End Time.
+      calculated: substract(TimeFlowEndMs,TimeFlowStartMs)
       default: false
       width: 5
-    - id: TimeFlowRttMs
-      name: Flow RTT
-      tooltip: TCP handshake Round Trip Time
-      field: TimeFlowRttNs
-      filter: time_flow_rtt
-      default: true
-      width: 5
     - id: CollectionTime
       name: Collection Time
       tooltip: Reception time of the record by the collector.
+      calculated: multiply(TimeReceived,1000),
       field: TimeReceived
       default: false
       width: 15
     - id: CollectionLatency
       name: Collection Latency
       tooltip: Time elapsed between End Time and Collection Time.
+      calculated: substract(column.CollectionTime,TimeFlowEndMs)
       default: false
       width: 5
     - id: DNSId
@@ -431,6 +434,8 @@ frontend:
       group: DNS
       name: DNS Latency
       tooltip: Time elapsed between DNS request and response.
+      field: DnsLatencyMs
+      filter: dns_latency
       default: true
       width: 5
     - id: DNSResponseCode
@@ -449,6 +454,13 @@ frontend:
       filter: dns_errno
       default: false
       width: 5
+    - id: TimeFlowRttMs
+      name: Flow RTT
+      tooltip: TCP handshake Round Trip Time
+      field: TimeFlowRttNs
+      filter: time_flow_rtt
+      default: true
+      width: 5
   filters:
     - id: src_namespace
       name: Namespace
@@ -722,10 +734,6 @@ frontend:
       component: text
       placeholder: 'E.g: br-ex, ovn-k8s-mp0'
       hint: Specify a network interface.
-    - id: dscp
-      name: DSCP value
-      component: number
-      hint: Specify a Differentiated Services Code Point value as integer number.
     - id: id
       name: Conversation Id
       component: text
@@ -800,7 +808,3 @@ frontend:
   alertNamespaces:
     - netobserv
   sampling: 50
-  features:
-    - pktDrop
-    - dnsTracking
-    - flowRTT

mocks/loki/flow_records.json

@@ -308,7 +308,7 @@
         "values": [
           [
             "1689619351079000064",
-            "{\"SrcMac\":\"42:01:0A:00:00:01\",\"DstPort\":6443,\"SrcPort\":54082,\"Etype\":2048,\"SrcK8S_Type\":\"Node\",\"AgentIP\":\"10.0.0.5\",\"Bytes\":66,\"Packets\":1,\"SrcK8S_HostIP\":\"10.0.0.4\",\"SrcK8S_HostName\":\"ci-ln-hnd9rjk-72292-hnd5v-master-1\",\"SrcK8S_OwnerType\":\"Node\",\"Proto\":6,\"Flags\":16,\"SrcAddr\":\"10.0.0.4\",\"SrcK8S_Name\":\"ci-ln-hnd9rjk-72292-hnd5v-master-1\",\"DstMac\":\"42:01:0A:00:00:05\",\"TimeFlowStartMs\":1689619351079,\"Duplicate\":false,\"IfDirection\":0,\"TimeFlowEndMs\":1689619351079,\"TimeFlowRttNs\":1234,\"TimeReceived\":1689619351,\"DstAddr\":\"10.0.0.2\",\"Interface\":\"br-ex\"}"
+            "{\"SrcMac\":\"42:01:0A:00:00:01\",\"DstPort\":6443,\"SrcPort\":54082,\"Etype\":2048,\"SrcK8S_Type\":\"Node\",\"AgentIP\":\"10.0.0.5\",\"Bytes\":66,\"Packets\":1,\"SrcK8S_HostIP\":\"10.0.0.4\",\"SrcK8S_HostName\":\"ci-ln-hnd9rjk-72292-hnd5v-master-1\",\"SrcK8S_OwnerType\":\"Node\",\"Proto\":6,\"Flags\":16,\"SrcAddr\":\"10.0.0.4\",\"SrcK8S_Name\":\"ci-ln-hnd9rjk-72292-hnd5v-master-1\",\"DstMac\":\"42:01:0A:00:00:05\",\"TimeFlowStartMs\":1689619351079,\"Duplicate\":false,\"IfDirection\":0,\"TimeFlowEndMs\":1689619351079,\"TimeFlowRttNs\":1234,\"TimeReceived\":1689619351,\"DstAddr\":\"10.0.0.2\",\"Interface\":\"br-ex\",\"Dscp\":0,\"DnsLatencyMs\":10,\"DnsErrno\":0}"
           ],
           [
             "1689619350793999872",

web/src/api/ipfix.ts

@@ -12,7 +12,16 @@ export interface Record {
 }
 
 export const getRecordValue = (record: Record, fieldOrLabel: string, defaultValue: string | number) => {
-  return record.fields[fieldOrLabel as keyof Fields] || record.labels[fieldOrLabel as keyof Labels] || defaultValue;
+  // check if label exists
+  if (record.labels[fieldOrLabel as keyof Labels] !== undefined) {
+    return record.labels[fieldOrLabel as keyof Labels];
+  }
+  // check if field exists
+  if (record.fields[fieldOrLabel as keyof Fields] !== undefined) {
+    return record.fields[fieldOrLabel as keyof Fields];
+  }
+  // fallback on default
+  return defaultValue;
 };
 
 export interface Labels {

web/src/components/netflow-record/record-field.tsx

@@ -535,7 +535,7 @@ export const RecordField: React.FC<{
         return singleContainer(
           typeof value === 'number' && !isNaN(value)
             ? simpleTextWithTooltip(
-                detailed ? `${value}: ${getDNSErrorDescription(value as DNS_ERRORS_VALUES)}` : String(value)
+                detailed && value ? `${value}: ${getDNSErrorDescription(value as DNS_ERRORS_VALUES)}` : String(value)
               )
             : emptyText()
         );

web/src/components/netflow-traffic.tsx

@@ -322,7 +322,13 @@ export const NetflowTraffic: React.FC<{
   );
 
   const getFilterDefs = React.useCallback(() => {
-    return getFilterDefinitions(config.filters, config.columns, t);
+    return getFilterDefinitions(config.filters, config.columns, t).filter(
+      fd =>
+        (isConnectionTracking() || fd.id !== 'id') &&
+        (isDNSTracking() || !fd.id.startsWith('dns_')) &&
+        (isPktDrop() || !fd.id.startsWith('pkt_drop_')) &&
+        (isFlowRTT() || fd.id !== 'time_flow_rtt')
+    );
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [config.columns, config.filters]);
 

web/src/utils/columns.ts

@@ -210,35 +210,66 @@ export const getConcatenatedValue = (
 };
 
 export const getDefaultColumns = (columnDefs: ColumnConfigDef[]): Column[] => {
+  const columns: Column[] = [];
+
+  function getColumnOrRecordValue(record: Record, arg: string, defaultValue: string | number) {
+    if (arg.startsWith('column.')) {
+      const colId = arg.replace('column.', '');
+      const found = columns.find(c => c.id === colId);
+      if (found) {
+        return found.value(record);
+      }
+      return defaultValue;
+    }
+    return getRecordValue(record, arg, defaultValue);
+  }
+
   function calculatedValue(record: Record, calculatedValue: string) {
     if (calculatedValue.startsWith('getSrcOrDstValue')) {
-      const fields = calculatedValue.replaceAll(/getSrcOrDstValue|\(|\)/g, '').split(',');
-      if (fields.length !== 2) {
+      const args = calculatedValue.replaceAll(/getSrcOrDstValue|\(|\)/g, '').split(',');
+      if (args.length !== 2) {
         console.error('getDefaultColumns - invalid parameters for getSrcOrDstValue calculated value', calculatedValue);
         return '';
       }
-      return getSrcOrDstValue(...fields.map(f => getRecordValue(record, f, '')));
+      return getSrcOrDstValue(...args.flatMap(f => getColumnOrRecordValue(record, f, '')));
     } else if (calculatedValue.startsWith('getConcatenatedValue')) {
-      const fields = calculatedValue.replaceAll(/getConcatenatedValue|\(|\)/g, '').split(',');
-      if (fields.length < 2) {
+      const args = calculatedValue.replaceAll(/getConcatenatedValue|\(|\)/g, '').split(',');
+      if (args.length < 2) {
         console.error(
           'getDefaultColumns - invalid parameters for getConcatenatedValue calculated value',
           calculatedValue
         );
         return '';
       }
       return getConcatenatedValue(
-        getRecordValue(record, fields[0], '') as string,
-        getRecordValue(record, fields[1], NaN) as number,
-        fields.length > 2 ? (getRecordValue(record, fields[2], '') as string) : undefined,
-        fields.length > 3 ? (getRecordValue(record, fields[3], '') as string) : undefined
+        getColumnOrRecordValue(record, args[0], '') as string,
+        getColumnOrRecordValue(record, args[1], NaN) as number,
+        args.length > 2 ? (getColumnOrRecordValue(record, args[2], '') as string) : undefined,
+        args.length > 3 ? (getColumnOrRecordValue(record, args[3], '') as string) : undefined
       );
+    } else if (calculatedValue.startsWith('substract')) {
+      const args = calculatedValue.replaceAll(/substract|\(|\)/g, '').split(',');
+      if (args.length < 2) {
+        console.error('getDefaultColumns - invalid parameters for substract calculated value', calculatedValue);
+        return '';
+      }
+      return (
+        (getColumnOrRecordValue(record, args[0], 0) as number) - (getColumnOrRecordValue(record, args[1], 0) as number)
+      );
+    } else if (calculatedValue.startsWith('multiply')) {
+      const args = calculatedValue.replaceAll(/multiply|\(|\)/g, '').split(',');
+      if (args.length < 2) {
+        console.error('getDefaultColumns - invalid parameters for multiply calculated value', calculatedValue);
+        return '';
+      }
+      return (getColumnOrRecordValue(record, args[0], 0) as number) * Number(args[1]);
     }
     return '';
   }
 
-  return columnDefs.map(d => {
-    return {
+  // add a column for each definition
+  columnDefs.forEach(d => {
+    columns.push({
       id: d.id as ColumnsId,
       group: !_.isEmpty(d.group) ? d.group : undefined,
       name: d.name,
@@ -253,10 +284,19 @@ export const getDefaultColumns = (columnDefs: ColumnConfigDef[]): Column[] => {
           if (d.calculated!.startsWith('[') && d.calculated!.endsWith(']')) {
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const result: any = [];
-            const values = d.calculated!.replaceAll(/\[|\]/g, '').split('),');
-            values.forEach(v => {
-              result.push(calculatedValue(r, `${v})`));
-            });
+            if (d.calculated?.includes('column')) {
+              // consider all items as columns or fields
+              const values = d.calculated!.replaceAll(/\[|\]/g, '').split(',');
+              values.forEach(v => {
+                result.push(getColumnOrRecordValue(r, v, ''));
+              });
+            } else {
+              // consider all items as functions
+              const values = d.calculated!.replaceAll(/\[|\]/g, '').split('),');
+              values.forEach(v => {
+                result.push(calculatedValue(r, `${v})`));
+              });
+            }
             return result;
           } else {
             return calculatedValue(r, d.calculated!);
@@ -288,6 +328,7 @@ export const getDefaultColumns = (columnDefs: ColumnConfigDef[]): Column[] => {
         }
       },
       width: d.width || 15
-    };
+    });
   });
+  return columns;
 };