diff --git a/snowflake/dashboards/flows_records_dashboard.json b/snowflake/dashboards/flows_records_dashboard.json new file mode 100644 index 000000000..396b6bd2e --- /dev/null +++ b/snowflake/dashboards/flows_records_dashboard.json @@ -0,0 +1,1027 @@ +{ + "__inputs": [ + { + "name": "DS_SNOWFLAKE", + "label": "Snowflake", + "description": "", + "type": "datasource", + "pluginId": "michelin-snowflake-datasource", + "pluginName": "Snowflake" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.1.6" + }, + { + "type": "datasource", + "id": "michelin-snowflake-datasource", + "name": "Snowflake", + "version": "1.2.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT count(*) as count\nFROM flows\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Flow Records Count", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 19, + "x": 5, + "y": 0 + }, + "id": 6, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT $__timeGroup(flowEndSeconds, $__interval, NULL) as time, \ncount(*) as count\nFROM flows\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nGROUP BY time\nORDER BY time", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Flow Records Count", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": false + }, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "flowType" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "1": { + "index": 0, + "text": "Intra-Node" + }, + "2": { + "index": 1, + "text": "Inter-Node" + }, + "3": { + "index": 2, + "text": "To External" + }, + "4": { + "index": 3, + "text": "From External" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sourcePodNamespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 199 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationPodName" + }, + "properties": [ + { + "id": "custom.width", + "value": 183 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationPodNamespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 214 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationNodeName" + }, + "properties": [ + { + "id": "custom.width", + "value": 183 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sourceNodeName" + }, + "properties": [ + { + "id": "custom.width", + "value": 161 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sourcePodName" + }, + "properties": [ + { + "id": "custom.width", + "value": 149 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationServicePort" + }, + "properties": [ + { + "id": "custom.width", + "value": 184 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationServicePortName" + }, + "properties": [ + { + "id": "custom.width", + "value": 215 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ingressNetworkPolicyName" + }, + "properties": [ + { + "id": "custom.width", + "value": 213 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ingressNetworkPolicyNamespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 256 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ingressNetworkPolicyRuleName" + }, + "properties": [ + { + "id": "custom.width", + "value": 243 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ingressNetworkPolicyRuleAction" + }, + "properties": [ + { + "id": "custom.width", + "value": 249 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "No Action" + }, + "1": { + "index": 1, + "text": "Allow" + }, + "2": { + "index": 2, + "text": "Drop" + }, + "3": { + "index": 3, + "text": "Reject" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ingressNetworkPolicyType" + }, + "properties": [ + { + "id": "custom.width", + "value": 209 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 3, + "text": "N/A" + }, + "1": { + "index": 0, + "text": "K8s NetworkPolicy" + }, + "2": { + "index": 1, + "text": "Antrea NetworkPolicy" + }, + "3": { + "index": 2, + "text": "Antrea ClusterNetworkPolicy" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "egressNetworkPolicyName" + }, + "properties": [ + { + "id": "custom.width", + "value": 215 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "egressNetworkPolicyRuleName" + }, + "properties": [ + { + "id": "custom.width", + "value": 238 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "egressNetworkPolicyNamespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 248 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "egressNetworkPolicyType" + }, + "properties": [ + { + "id": "custom.width", + "value": 207 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 3, + "text": "N/A" + }, + "1": { + "index": 0, + "text": "K8s NetworkPolicy" + }, + "2": { + "index": 1, + "text": "Antrea NetworkPolicy" + }, + "3": { + "index": 2, + "text": "Antrea ClusterNetworkPolicy" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationPodLabels" + }, + "properties": [ + { + "id": "custom.width", + "value": 173 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "throughputFromSourceNode" + }, + "properties": [ + { + "id": "custom.width", + "value": 230 + }, + { + "id": "displayName", + "value": "throughput reported by source" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "throughputFromDestinationNode" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + }, + { + "id": "displayName", + "value": "throughput reported by destination" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reverseThroughputFromSourceNode" + }, + "properties": [ + { + "id": "custom.width", + "value": 280 + }, + { + "id": "displayName", + "value": "reverse throughput reported by source" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reverseThroughputFromDestinationNode" + }, + "properties": [ + { + "id": "custom.width", + "value": 310 + }, + { + "id": "displayName", + "value": "reverse throughput reported by destination" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reverseOctetDeltaCount" + }, + "properties": [ + { + "id": "custom.width", + "value": 186 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reversePacketDeltaCount" + }, + "properties": [ + { + "id": "custom.width", + "value": 204 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reverseOctetTotalCount" + }, + "properties": [ + { + "id": "custom.width", + "value": 191 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reversePacketTotalCount" + }, + "properties": [ + { + "id": "custom.width", + "value": 199 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "flowEndSecondsFromDestinationNode" + }, + "properties": [ + { + "id": "custom.width", + "value": 290 + }, + { + "id": "displayName", + "value": "end time of flow reported by destination" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "flowEndSecondsFromSourceNode" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + }, + { + "id": "displayName", + "value": "end time of flow reported by source" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "protocolIdentifier" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "6": { + "index": 0, + "text": "TCP" + }, + "17": { + "index": 1, + "text": "UDP" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sourceTransportPort" + }, + "properties": [ + { + "id": "custom.width", + "value": 170 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationTransportPort" + }, + "properties": [ + { + "id": "custom.width", + "value": 196 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationClusterIP" + }, + "properties": [ + { + "id": "custom.width", + "value": 168 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "egressNetworkPolicyRuleAction" + }, + "properties": [ + { + "id": "custom.width", + "value": 244 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "No Action" + }, + "1": { + "index": 1, + "text": "Allow" + }, + "2": { + "index": 2, + "text": "Drop" + }, + "3": { + "index": 3, + "text": "Reject" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reverseThroughput" + }, + "properties": [ + { + "id": "custom.width", + "value": 161 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "flowEndSeconds" + }, + "properties": [ + { + "id": "custom.width", + "value": 185 + }, + { + "id": "displayName", + "value": "end time of flow" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "flowStartSeconds" + }, + "properties": [ + { + "id": "custom.width", + "value": 185 + }, + { + "id": "displayName", + "value": "start time of flow" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timeInserted" + }, + "properties": [ + { + "id": "custom.width", + "value": 187 + }, + { + "id": "displayName", + "value": "insert time of flow" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "flowEndReason" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "1": { + "index": 0, + "text": "Idle Timeout" + }, + "2": { + "index": 1, + "text": "Active Timeout" + }, + "3": { + "index": 2, + "text": "End Of Flow" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.width", + "value": 142 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "octetDeltaCount" + }, + "properties": [ + { + "id": "displayName", + "value": "delta bytes from the last record" + }, + { + "id": "custom.width", + "value": 235 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "packetDeltaCount" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + }, + { + "id": "displayName", + "value": "delta packets from the last record" + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 2, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT * FROM flows\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nORDER BY flowEndSeconds\nLIMIT 10000", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Flow Records Table", + "transformations": [ + { + "id": "seriesToColumns", + "options": {} + } + ], + "transparent": true, + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "flow_records_dashboard", + "uid": "dbxiHWV4k", + "version": 3, + "weekStart": "" +} diff --git a/snowflake/dashboards/homepage.json b/snowflake/dashboards/homepage.json new file mode 100644 index 000000000..e01f07a1e --- /dev/null +++ b/snowflake/dashboards/homepage.json @@ -0,0 +1,1171 @@ +{ + "__inputs": [ + { + "name": "DS_SNOWFLAKE", + "label": "Snowflake", + "description": "", + "type": "datasource", + "pluginId": "michelin-snowflake-datasource", + "pluginName": "Snowflake" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "panel", + "id": "dashlist", + "name": "Dashboard list", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.1.6" + }, + { + "type": "datasource", + "id": "michelin-snowflake-datasource", + "name": "Snowflake", + "version": "1.2.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "_UhIbRV4k" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 32, + "panels": [], + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "_UhIbRV4k" + }, + "refId": "A" + } + ], + "title": "Cluster Overview", + "type": "row" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "displayName": "Number of Pods", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 12, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT COUNT(derivedtable.pod) as Number_of_Pods\nFROM\n(\n SELECT DISTINCT CONCAT(sourcePodName, sourcePodNamespace) AS pod FROM flows WHERE pod != '' AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\n UNION ALL\n SELECT DISTINCT CONCAT(destinationPodName, destinationPodNamespace) AS pod FROM flows WHERE pod != '' AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\n) derivedtable\nWHERE derivedtable.pod != ''", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "transformations": [ + { + "id": "labelsToFields", + "options": {} + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "displayName": "Number of Services", + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 14, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT COUNT(DISTINCT destinationServicePortName) as Number_of_Services \nFROM flows \nWHERE destinationServicePortName != '' AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "displayName": "Number of Nodes", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 13, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT COUNT(DISTINCT derivedtable.node) as Number_of_Nodes\nFROM\n(\n SELECT DISTINCT sourceNodeName AS node FROM flows WHERE node != '' AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\n UNION ALL\n SELECT DISTINCT destinationNodeName AS node FROM flows WHERE node != '' AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\n) derivedtable\nWHERE derivedtable.node IS NOT NULL", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "content": "
\n \n

Theia is a network observability and\nanalytics platform for Kubernetes, built on top of\nAntrea. It supports network flow\nvisualization and monitoring with Grafana. Follow this\ndocument\nto know more about the Grafana Flow Collector and network flow visualization\nfunctionality of Theia.

\n
\n\n", + "mode": "html" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "refId": "A" + } + ], + "title": "Overview of Project Theia", + "type": "text" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "displayName": "Number of Active Connections", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 15, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value_and_name" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT COUNT(DISTINCT CONCAT(sourceIP, destinationIP)) as Number_of_Active_Connections\nfrom flows\nWHERE flowEndReason = 2 AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "displayName": "Number of Terminated Connections", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 16, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT COUNT(DISTINCT CONCAT(sourceIP, destinationIP)) as Number_of_Stopped_Connections\nfrom flows WHERE flowEndReason != 2 AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "displayName": "Number of Denied Connections", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 5 + }, + "id": 23, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT COUNT(DISTINCT CONCAT(sourceIP, destinationIP)) as Number_of_Denied_Connections\nfrom flows\nWHERE (ingressNetworkPolicyRuleAction in (2,3) OR egressNetworkPolicyRuleAction in (2,3))\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 27, + "options": { + "content": "The following dashboards are pre-built and are recommended for network flow\nvisualization.\n\nFlow Records Dashboard displays the flow records being captured in the selected\ntime range.\n\nPod-to-Pod Flows Dashboard shows cumulative bytes and throughput of Pod-to-Pod\ntraffic.\n\nNetwork-Policy Flows Dashboard visualizes both the traffic with NetworkPolicies enforced,\nand unprotected traffic.", + "mode": "markdown" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "refId": "A" + } + ], + "title": "Introduction of Pre-built Dashboards", + "type": "text" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "displayName": "Data Transmitted", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 21, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT SUM(octetDeltaCount)+SUM(reverseOctetDeltaCount) as Data_Transmitted\nfrom pods WHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "displayName": "Overall Throughput", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 9 + }, + "id": 22, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT (SUM(octetDeltaCount)+SUM(reverseOctetDeltaCount))/60 as Overall_Throughput\nfrom pods WHERE DATEDIFF(SECOND, CURRENT_TIMESTAMP(), flowEndSeconds) < 60", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "displayName": "Number of NetworkPolicies", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 9 + }, + "id": 26, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value_and_name" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "format": 1, + "queryText": "SELECT (COUNT(DISTINCT ingressNetworkPolicyName) + COUNT(DISTINCT egressNetworkPolicyName)) as Number_of_NetworkPolicies\nfrom flows\nWHERE CONCAT(ingressNetworkPolicyName, egressNetworkPolicyName) != ''\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "displayName": "Data Transmitted with External", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 13 + }, + "id": 28, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT SUM(octetDeltaCount)+SUM(reverseOctetDeltaCount) as Data_Transmitted_With_External\nFROM pods\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND flowType = 3", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "displayName": "Overall Throughput with External", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 13 + }, + "id": 29, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT (SUM(octetDeltaCount)+SUM(reverseOctetDeltaCount))/60 as Overall_Throughput_With_External\nfrom pods WHERE DATEDIFF(SECOND, CURRENT_TIMESTAMP(), flowEndSeconds) < 60\nAND flowType = 3", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "displayName": "Number of ToExternal Connections", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 13 + }, + "id": 33, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "format": 1, + "queryText": "SELECT COUNT(DISTINCT CONCAT(sourceIP, destinationIP)) as Number_of_ToExternal_Connections\nfrom flows\nWHERE flowType = 3\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showUnfilled": true + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT CONCAT(sourcePodNamespace, '/', sourcePodName) as pod,\nSUM(octetDeltaCount) as bytes\nFROM pods\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND pod != '/'\nGROUP BY pod\nORDER BY bytes DESC LIMIT 10", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Top 10 Active Source Pods", + "type": "bargauge" + }, + { + "datasource": { + "type": "grafana", + "uid": "grafana" + }, + "gridPos": { + "h": 18, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 4, + "links": [], + "options": { + "folderId": 0, + "maxItems": 10, + "query": "", + "showHeadings": true, + "showRecentlyViewed": true, + "showSearch": true, + "showStarred": true, + "tags": [] + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "list", + "refId": "A" + } + ], + "title": "Dashboard Links", + "type": "dashlist" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 25, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT $__timeGroup(flowEndSeconds, '1m') as time,\ncount(*) as count\nFROM pods\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nGROUP BY time\nORDER BY time", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Number of Flow Records Per Minute", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "homepage", + "uid": "F_PWNZ44z", + "version": 7, + "weekStart": "" +} diff --git a/snowflake/dashboards/networkpolicy_dashboard.json b/snowflake/dashboards/networkpolicy_dashboard.json new file mode 100644 index 000000000..35c314fd4 --- /dev/null +++ b/snowflake/dashboards/networkpolicy_dashboard.json @@ -0,0 +1,738 @@ +{ + "__inputs": [ + { + "name": "DS_SNOWFLAKE", + "label": "Snowflake", + "description": "", + "type": "datasource", + "pluginId": "michelin-snowflake-datasource", + "pluginName": "Snowflake" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.1.6" + }, + { + "type": "datasource", + "id": "michelin-snowflake-datasource", + "name": "Snowflake", + "version": "1.2.0" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "panel", + "id": "theia-grafana-chord-plugin", + "name": "grafana-chord-plugin", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "gridPos": { + "h": 20, + "w": 15, + "x": 0, + "y": 0 + }, + "id": 2, + "interval": "1s", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT CONCAT(sourcePodNamespace, '/', sourcePodName) as srcPod,\nCONCAT(destinationPodNamespace, '/', destinationPodName) as dstPod,\nsourceTransportPort as srcPort,\ndestinationTransportPort as dstPort,\ndestinationServicePort as dstSvcPort,\ndestinationServicePortName as dstSvc,\ndestinationIP as dstIP,\nSUM(octetDeltaCount) as bytes,\nSUM(reverseOctetDeltaCount) as revBytes,\negressNetworkPolicyName,\negressNetworkPolicyRuleAction,\ningressNetworkPolicyName,\ningressNetworkPolicyRuleAction\nfrom policies\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\n AND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\n AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nGROUP BY srcPod, dstPod, srcPort, dstPort, dstSvcPort, dstSvc, dstIP, egressNetworkPolicyName, egressNetworkPolicyRuleAction, ingressNetworkPolicyName, ingressNetworkPolicyRuleAction\nHAVING bytes > 0\norder by bytes DESC", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Bytes of Flows with NetworkPolicy Information", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "BYTES": "bytes", + "DSTIP": "dstIP", + "DSTPOD": "dstPod", + "DSTPORT": "dstPort", + "DSTSVC": "dstSvc", + "DSTSVCPORT": "dstSvcPort", + "EGRESSNETWORKPOLICYNAME": "egressNetworkPolicyName", + "EGRESSNETWORKPOLICYRULEACTION": "egressNetworkPolicyRuleAction", + "INGRESSNETWORKPOLICYNAME": "ingressNetworkPolicyName", + "INGRESSNETWORKPOLICYRULEACTION": "ingressNetworkPolicyRuleAction", + "REVBYTES": "revBytes", + "SRCPOD": "srcPod", + "SRCPORT": "srcPort" + } + } + } + ], + "transparent": true, + "type": "theia-grafana-chord-plugin" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 25, + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT SUM(octetDeltaCount) as bytes, \nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace, '/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND AS np\nFROM policies\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ingressNetworkPolicyName != ''\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nGROUP BY np\nHAVING SUM(octetDeltaCount) != 0\nORDER BY bytes DESC", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Bytes of Ingress Network Policy", + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 9, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 25, + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT SUM(octetDeltaCount) as bytes,\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace, '/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND AS np\nFROM policies\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND egressNetworkPolicyName != ''\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nGROUP BY np\nHAVING SUM(octetDeltaCount) > 0\nORDER BY bytes DESC", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Bytes of Egress Network Policy", + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 4, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT $__timeGroup(flowEndSeconds, $__interval, NULL) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort = 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort = 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND AS dst,\nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace,'/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND AS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM policies\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ingressNetworkPolicyRuleAction = 1\nAND egressNetworkPolicyRuleAction NOT IN (2,\n3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Ingress Allow NetworkPolicy", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "PAIR" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 5, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT $__timeGroup(flowEndSeconds, $__interval, NULL) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort = 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort = 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace,'/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM policies\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND egressNetworkPolicyRuleAction = 1\nAND ingressNetworkPolicyRuleAction not in (2,3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Egress Allow NetworkPolicy", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "PAIR" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 6, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT $__timeGroup(flowEndSeconds, $__interval, NULL) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort = 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort = 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace,'/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM policies\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ingressNetworkPolicyRuleAction in (2,3)\nGROUP BY time, src, dst, np\nHAVING SUM(octetDeltaCount) > 0\nORDER BY time", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Ingress Deny NetworkPolicy", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "PAIR" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 7, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT $__timeGroup(flowEndSeconds, $__interval, NULL) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort = 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort = 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace,'/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM policies\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND egressNetworkPolicyRuleAction in (2,\n3)\nGROUP BY time, src, dst, np\nHAVING SUM(octetDeltaCount) > 0\nORDER BY time", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Egress Deny NetworkPolicy", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "PAIR" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "networkpolicy_dashboard", + "uid": "LvtkNZ4Vz", + "version": 3, + "weekStart": "" +} diff --git a/snowflake/dashboards/pod_to_pod_dashboard.json b/snowflake/dashboards/pod_to_pod_dashboard.json new file mode 100644 index 000000000..22abd338a --- /dev/null +++ b/snowflake/dashboards/pod_to_pod_dashboard.json @@ -0,0 +1,817 @@ +{ + "__inputs": [ + { + "name": "DS_SNOWFLAKE", + "label": "Snowflake", + "description": "", + "type": "datasource", + "pluginId": "michelin-snowflake-datasource", + "pluginName": "Snowflake" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.1.6" + }, + { + "type": "datasource", + "id": "michelin-snowflake-datasource", + "name": "Snowflake", + "version": "1.2.0" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "panel", + "id": "theia-grafana-sankey-plugin", + "name": "grafana-sankey-plugin", + "version": "1.0.1" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "gridPos": { + "h": 18, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 13, + "options": { + "seriesCountSize": "sm", + "showSeriesCount": false, + "text": "Default value of text input option" + }, + "pluginVersion": "7.5.2", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT SUM(octetDeltaCount) as bytes, \nsource,\ndestination\nFROM pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nGROUP BY source, destination\nORDER BY bytes DESC\nLIMIT 50", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Bytes of Pod-to-Pod", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "BYTES": "bytes", + "DESTINATION": "destination", + "SOURCE": "source" + } + } + } + ], + "transparent": true, + "type": "theia-grafana-sankey-plugin" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "gridPos": { + "h": 18, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 12, + "options": { + "seriesCountSize": "sm", + "showSeriesCount": false, + "text": "Default value of text input option" + }, + "pluginVersion": "7.5.2", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT SUM(reverseOctetDeltaCount) as bytes, \nsource,\ndestination\nFROM pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nGROUP BY source, destination\nORDER BY bytes DESC\nLIMIT 50", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Reverse Bytes of Pod-to-Pod", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "BYTES": "bytes", + "DESTINATION": "destination", + "SOURCE": "source" + } + } + } + ], + "transparent": true, + "type": "theia-grafana-sankey-plugin" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 21, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT $__timeGroup(flowEndSeconds, $__interval, NULL) as time,\nCONCAT(source, ' -> ', destination) as pair,\nAVG(throughput)\nFROM pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nGROUP BY time, pair\nHAVING AVG(throughput) > 0\nORDER BY time", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Pod-to-Pod", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "columns", + "valueLabel": "PAIR" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 22, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT $__timeGroup(flowEndSeconds, $__interval, NULL) as time,\nCONCAT(source, ' -> ', destination) as pair,\nAVG(reverseThroughput)\nFROM pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nGROUP BY time, pair\nHAVING AVG(throughput) > 0\nORDER BY time", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Reverse Throughput of Pod-to-Pod", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "PAIR" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 15, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT $__timeGroup(flowEndSeconds, $__interval, NULL) as time,\nsource as src,\nSUM(throughput) FROM pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nGROUP BY time, src\nHAVING SUM(throughput) > 0\nORDER BY time\n", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Pod as Source", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "SRC" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 18, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 25, + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT SUM(octetDeltaCount) as bytes, \nsourcePodNamespace\nFrom pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nGROUP BY sourcePodNamespace\nHAVING bytes > 0\nORDER BY bytes DESC", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Bytes of Source Pod Namespace", + "transformations": [ + { + "id": "labelsToFields", + "options": {} + } + ], + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 16, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT $__timeGroup(flowEndSeconds, $__interval, NULL) as time,\ndestination as dst,\nSUM(throughput) FROM pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nGROUP BY time, dst\nHAVING SUM(throughput) > 0\nORDER BY time", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Pod as Destination", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "DST" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "destinationPodNamespace" + }, + "properties": [ + { + "id": "noValue", + "value": "N/A" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 25, + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "${DS_SNOWFLAKE}" + }, + "queryText": "SELECT SUM(octetDeltaCount) as bytes, \ndestinationPodNamespace\nFrom pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nGROUP BY destinationPodNamespace\nHAVING bytes > 0\nORDER BY bytes DESC", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Bytes of Destination Pod Namespace", + "transparent": true, + "type": "piechart" + } + ], + "refresh": "", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "pod_to_pod_dashboard", + "uid": "B1t6SZVVz", + "version": 1, + "weekStart": "" +} diff --git a/snowflake/network-observability.md b/snowflake/network-observability.md new file mode 100644 index 000000000..1bdfcbe34 --- /dev/null +++ b/snowflake/network-observability.md @@ -0,0 +1,78 @@ +# Network Observability + +We use Grafana as the tool to query data from snowflake, and visualize the +networking flows in the cluster(s). + +## Deployment + +We suggest install and run Grafana using the official Docker image. + +Before running Grafana, we will first download the snowflake datasource plugin. +We suggest create your own plugin directory. + +```bash +mkdir your-plugin-path && cd your-plugin-path +wget https://github.com/michelin/snowflake-grafana-datasource/releases/download/v1.2.0/snowflake-grafana-datasource.zip +unzip snowflake-grafana-datasource.zip +``` + +Then run Grafana with Docker: + +```bash +docker run -d \ +-p 3000:3000 \ +-v "$(pwd)":/var/lib/grafana/plugins \ +--name=grafana \ +-e "GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=michelin-snowflake-datasource" \ +-e "GF_INSTALL_PLUGINS=https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-sankey-plugin-1.0.2.zip;theia-grafana-sankey-plugin,https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-chord-plugin-1.0.1.zip;theia-grafana-chord-plugin" \ +grafana/grafana:9.1.6 +``` + +Open your web browser and go to `http://localhost:3000/`, login with: + +- username: admin +- password: admin + +## Configure the Datasource + +* In the side menu under the Configuration icon you should find a link named Data Sources. +* Click the `Add data source` button in the top header. +* Select Snowflake. + +Add your authentication and [configuration details](https://docs.snowflake.com/en/user-guide/jdbc-configure.html#connection-parameters). + +Configure the datasource + +Available configuration fields are as follow: + + Name | Description +------------------------- | ------------ + Account Name | Specifies the full name of your account (provided by Snowflake) + Username | Specifies the login name of the user for the connection. + Password | Specifies the password for the specified user. + Role (Optional) | Specifies the default access control role to use in the Snowflake session initiated by Grafana. + Warehouse | Specifies the virtual warehouse to use once connected. + Database | Specifies the default database to use once connected. + Schema | Specifies the default schema to use for the specified database once connected. + Extra Options (Optional) | Specifies a series of one or more parameters, in the form of `=`, with each parameter separated by the ampersand character (&), and no spaces anywhere in the connection string. + +Database and schema names can be found in previous output of the `onboard` +command. Save the datasource by clicking on `Save & Test`. + +## Import dashboards + +To import a dashboard click Import under the Dashboards icon in the side menu. + +Import dashboards + +From here you can upload a dashboard JSON file, or paste dashboard JSON text +directly into the text area. The pre-built dashboards JSON files are under +`theia/snowflake/dashboards`. + +Once uploaded, you should be able to find the dashboards under the Search +Dashboard icon in the side menu. The introduction of these dashboards can +be found at +[Pre-built Dashboards](https://github.com/antrea-io/theia/blob/main/docs/network-flow-visibility.md#pre-built-dashboards). +Currently, with snowflake datasource, we have built the Home Dashboard, Flow +Records Dashboard, Pod-to-Pod Flows Dashboard, and Network-Policy Flows +Dashboard.