From 6192733701d2654b847a2b5270e5d8fc8ecc01a9 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Fri, 2 Apr 2021 17:24:05 +0800 Subject: [PATCH] metrics: update TiKV metrics --- metrics/grafana/ticdc.json | 632 +++++++++++++++++++++++++------------ 1 file changed, 423 insertions(+), 209 deletions(-) diff --git a/metrics/grafana/ticdc.json b/metrics/grafana/ticdc.json index 7f63836568c..9cc0edee827 100644 --- a/metrics/grafana/ticdc.json +++ b/metrics/grafana/ticdc.json @@ -1,8 +1,8 @@ { "__inputs": [ { - "name": "DS_TEST-CLUSTER", - "label": "test-cluster", + "name": "DS_CDC-CLUSTER3", + "label": "cdc-cluster3", "description": "", "type": "datasource", "pluginId": "prometheus", @@ -60,7 +60,7 @@ "useValueForTime": false }, { - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "enable": true, "expr": "max(ticdc_processor_checkpoint_ts_lag) by (changefeed, capture) > BOOL $spike_threshold", "hide": true, @@ -75,10 +75,10 @@ "useValueForTime": false }, { - "datasource": "${DS_TEST-CLUSTER}", - "enable": true, + "datasource": "${DS_CDC-CLUSTER3}", + "enable": false, "expr": "min(up{tidb_cluster=\"$tidb_cluster\", job=~\"tikv|ticdc\"}) by (job, instance) == BOOL 0", - "hide": true, + "hide": false, "iconColor": "#FF9830", "limit": 100, "name": "Server down", @@ -90,7 +90,7 @@ "type": "tags" }, { - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "enable": false, "expr": "sum(ALERTS{tidb_cluster=\"$tidb_cluster\", alertstate=\"firing\", alertname=~\"ticdc.*\"}) by (alertname) > BOOL 0", "hide": false, @@ -109,7 +109,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1617089343502, + "iteration": 1617345981811, "links": [], "panels": [ { @@ -127,7 +127,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Uptime of TiCDC and TiKV", "fill": 1, "fillGradient": 0, @@ -228,7 +228,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Goroutine count of TiCDC", "fill": 1, "fillGradient": 0, @@ -322,7 +322,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The count of open FD count of TiCDC", "fill": 1, "fillGradient": 0, @@ -418,7 +418,7 @@ "value": "current" } ], - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "TiCDC cluster ownership status", "fontSize": "100%", "gridPos": { @@ -498,7 +498,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "CPU usage of TiCDC", "fill": 1, "fillGradient": 0, @@ -593,7 +593,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Memory usage of TiCDC", "fill": 1, "fillGradient": 0, @@ -695,7 +695,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The history of TiCDC cluster ownership, owner node has a value that is great than 0", "fill": 1, "gridPos": { @@ -784,7 +784,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "fill": 1, "fillGradient": 0, "gridPos": { @@ -908,7 +908,7 @@ "value": "current" } ], - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The number of captured table of TiCDC nodes ", "fontSize": "100%", "gridPos": { @@ -971,11 +971,11 @@ "value": "current" } ], - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Internal resolved ts of TiCDC nodes", "fontSize": "100%", "gridPos": { - "h": 10, + "h": 5, "w": 7, "x": 7, "y": 2 @@ -1024,19 +1024,19 @@ ], "targets": [ { - "expr": "max(ticdc_processor_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed)", + "expr": "max(ticdc_processor_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{capture}}--{{changefeed}}", + "legendFormat": "{{capture}}", "refId": "A" }, { - "expr": "max(ticdc_processor_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed) > 0", + "expr": "max(ticdc_processor_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture) > 0", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "checkpoint-{{capture}}--{{changefeed}}", + "legendFormat": "changefeed checkpoint", "refId": "B" } ], @@ -1053,11 +1053,11 @@ "value": "current" } ], - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Internal resolved ts of captured tables", "fontSize": "100%", "gridPos": { - "h": 10, + "h": 5, "w": 10, "x": 14, "y": 2 @@ -1106,20 +1106,20 @@ ], "targets": [ { - "expr": "max(ticdc_processor_table_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed,table)", + "expr": "max(ticdc_processor_table_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,table)", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}-{{table}}", + "legendFormat": "{{capture}}-{{table}}", "refId": "A" }, { - "expr": "max(ticdc_processor_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed,table) > 0", + "expr": "max(ticdc_processor_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture) > 0", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "checkpoint-{{changefeed}}", + "legendFormat": "changefeed checkpoint", "refId": "B" } ], @@ -1129,93 +1129,20 @@ "transform": "timeseries_aggregations", "type": "table" }, - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of replicated tables maintained in owner", - "fontSize": "100%", - "gridPos": { - "h": 5, - "w": 7, - "x": 0, - "y": 7 - }, - "id": 138, - "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": null, - "desc": false - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(ticdc_owner_maintain_table_num{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\",type=\"total\"}) by (capture)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{capture}}-total", - "refId": "A" - }, - { - "expr": "sum(ticdc_owner_maintain_table_num{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\",type=\"wip\"}) by (capture)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{capture}}-wip", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Table count maintained by owner", - "transform": "timeseries_aggregations", - "type": "table" - }, { "aliasColors": {}, "bars": true, "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The checkpoint ts of changefeeds.", "fill": 0, "gridPos": { "h": 7, "w": 9, "x": 0, - "y": 12 + "y": 7 }, "id": 86, "legend": { @@ -1313,7 +1240,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Request count of etcd operation per second", "fill": 1, "fillGradient": 0, @@ -1321,7 +1248,7 @@ "h": 7, "w": 8, "x": 9, - "y": 12 + "y": 7 }, "hiddenSeries": false, "id": 102, @@ -1410,14 +1337,14 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The number of errors that interrupt changefeed per minute ", "fill": 1, "gridPos": { "h": 7, "w": 7, "x": 17, - "y": 12 + "y": 7 }, "id": 82, "legend": { @@ -1498,7 +1425,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.", "fill": 1, "fillGradient": 0, @@ -1506,7 +1433,7 @@ "h": 7, "w": 12, "x": 0, - "y": 19 + "y": 14 }, "hiddenSeries": false, "id": 3, @@ -1593,7 +1520,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The lag between internal resolved ts and the latest ts of upstream TiDB.", "fill": 1, "fillGradient": 0, @@ -1601,7 +1528,7 @@ "h": 7, "w": 12, "x": 12, - "y": 19 + "y": 14 }, "hiddenSeries": false, "id": 2, @@ -1636,11 +1563,11 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_processor_resolved_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed)", + "expr": "sum(ticdc_processor_resolved_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{capture}}--{{changefeed}}", + "legendFormat": "{{capture}}", "refId": "A" } ], @@ -1700,13 +1627,13 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Sink write duration of changefeeds", "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 26 + "y": 21 }, "heatmap": {}, "hideZeroBuckets": true, @@ -1769,7 +1696,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Percentiles of sink write duration of changefeeds", "fill": 1, "fillGradient": 0, @@ -1777,7 +1704,7 @@ "h": 7, "w": 12, "x": 12, - "y": 26 + "y": 21 }, "hiddenSeries": false, "id": 35, @@ -1877,7 +1804,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The number of changed rows that are written to downstream per second", "fill": 1, "fillGradient": 0, @@ -1885,7 +1812,7 @@ "h": 7, "w": 12, "x": 0, - "y": 33 + "y": 28 }, "hiddenSeries": false, "id": 34, @@ -1980,7 +1907,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Percentiles of sink batch size", "fill": 1, "fillGradient": 0, @@ -1988,7 +1915,7 @@ "h": 7, "w": 12, "x": 12, - "y": 33 + "y": 28 }, "hiddenSeries": false, "id": 36, @@ -2099,13 +2026,13 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Asynchronous flush sink duration of changefeeds", "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 40 + "y": 35 }, "heatmap": {}, "hideZeroBuckets": true, @@ -2166,7 +2093,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Percentiles of asynchronous flush sink duration of changefeeds", "fill": 1, "fillGradient": 0, @@ -2174,7 +2101,7 @@ "h": 7, "w": 12, "x": 12, - "y": 40 + "y": 35 }, "hiddenSeries": false, "id": 98, @@ -2284,13 +2211,13 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The duration of detecting and waiting conflict of MySQL sink", "gridPos": { "h": 7, "w": 8, "x": 0, - "y": 47 + "y": 42 }, "heatmap": {}, "hideZeroBuckets": true, @@ -2352,7 +2279,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Percentiles of detecting and waiting conflict duration of MySQL sink", "fill": 1, "fillGradient": 0, @@ -2360,7 +2287,7 @@ "h": 7, "w": 8, "x": 8, - "y": 47 + "y": 42 }, "hiddenSeries": false, "id": 83, @@ -2462,7 +2389,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Distribution of MySQL worker loads", "fill": 1, "fillGradient": 0, @@ -2470,7 +2397,7 @@ "h": 7, "w": 8, "x": 16, - "y": 47 + "y": 42 }, "hiddenSeries": false, "id": 95, @@ -2617,13 +2544,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The number of established Eventfeed RPC between TiCDC and TiKV", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 8, + "w": 6, "x": 0, "y": 3 }, @@ -2712,7 +2639,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "decimals": null, "description": "Percentiles of Eventfeed message size", "fill": 1, @@ -2720,7 +2647,7 @@ "gridPos": { "h": 7, "w": 8, - "x": 8, + "x": 6, "y": 3 }, "hiddenSeries": false, @@ -2814,14 +2741,14 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The number of errors that interrupt Eventfeed RPC", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 8, - "x": 16, + "w": 10, + "x": 14, "y": 3 }, "hiddenSeries": false, @@ -2929,13 +2856,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The number of KV client received events from TiKV per seconds", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 8, + "w": 6, "x": 0, "y": 10 }, @@ -3025,14 +2952,108 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", + "description": "The number of puller received events from kv client per second\n", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 10 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate(ticdc_puller_kv_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}} - {{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Puller receive events/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_CDC-CLUSTER3}", "description": "The number of events that puller outputs to sorter \n per second", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 8, - "x": 8, + "w": 6, + "x": 12, "y": 10 }, "hiddenSeries": false, @@ -3119,14 +3140,14 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The number of rows that sink flushes to downstream per second", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 8, - "x": 16, + "w": 6, + "x": 18, "y": 10 }, "hiddenSeries": false, @@ -3214,7 +3235,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The number of events that are buffered in Puller's memory buffer and output channel", "fill": 1, "fillGradient": 0, @@ -3328,7 +3349,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The number of events that are buffered in Sorter's unsorted events buffer and output channel", "fill": 1, "fillGradient": 0, @@ -3448,7 +3469,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The number of events that are buffered in Processor's output channel and Mounter input channel", "fill": 1, "fillGradient": 0, @@ -3555,7 +3576,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The number of rows(events) that are buffered in Sink's pending flush rows buffer", "fill": 1, "fillGradient": 0, @@ -3660,7 +3681,7 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Duration of sorting unsorted events", "gridPos": { "h": 7, @@ -3727,7 +3748,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Percentiles of sorting events duration", "fill": 1, "fillGradient": 0, @@ -3839,7 +3860,7 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Duration of merging sorted events", "gridPos": { "h": 7, @@ -3906,7 +3927,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Percentiles of merging sorted events duration", "fill": 1, "fillGradient": 0, @@ -4018,7 +4039,7 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Duration of unmarshal events from kv to SQL row", "gridPos": { "h": 7, @@ -4085,7 +4106,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "Percentiles of unmarshal events from kv to SQL row duration", "fill": 0, "fillGradient": 0, @@ -4191,7 +4212,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The number of KV client dispatched event per second", "fill": 1, "fillGradient": 0, @@ -4239,17 +4260,17 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_kvclient_send_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture, changefeed, type)", + "expr": "sum(rate(ticdc_kvclient_send_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture, type)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}-{{type}}", + "legendFormat": "{{capture}}-{{type}}", "refId": "A" }, { - "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture, changefeed, table)", + "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture, table)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}-batch-resolved", + "legendFormat": "{{capture}}-batch-resolved", "refId": "B" } ], @@ -4308,7 +4329,7 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The size of batch resolved ts message from TiKV", "gridPos": { "h": 7, @@ -4389,7 +4410,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "fill": 1, "gridPos": { "h": 8, @@ -4421,10 +4442,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_sorter_consume_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", + "expr": "sum(rate(ticdc_sorter_consume_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}", "refId": "A" } ], @@ -4474,7 +4494,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "fill": 1, "gridPos": { "h": 8, @@ -4506,10 +4526,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", + "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}", "refId": "A" } ], @@ -4559,7 +4578,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "fill": 1, "gridPos": { "h": 8, @@ -4594,7 +4613,6 @@ "expr": "sum(ticdc_sorter_on_disk_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}", "refId": "A" } ], @@ -4644,7 +4662,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "fill": 1, "gridPos": { "h": 8, @@ -4679,7 +4697,6 @@ "expr": "sum(ticdc_sorter_in_memory_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}", "refId": "A" } ], @@ -4739,7 +4756,7 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "gridPos": { "h": 8, "w": 12, @@ -4803,7 +4820,7 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "gridPos": { "h": 8, "w": 12, @@ -4853,6 +4870,90 @@ "yBucketBound": "auto", "yBucketNumber": null, "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_CDC-CLUSTER3}", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 137, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(ticdc_sorter_resolved_ts_gauge{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Unified Sorter resolved ts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dateTimeAsIso", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "title": "Unified Sorter", @@ -4873,14 +4974,14 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "decimals": 1, "fill": 1, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 5 + "y": 54 }, "id": 60, "legend": { @@ -4966,14 +5067,14 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "decimals": 1, "fill": 1, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 5 + "y": 54 }, "id": 62, "legend": { @@ -5078,13 +5179,13 @@ "value": "max" } ], - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "fontSize": "90%", "gridPos": { "h": 7, "w": 7, "x": 0, - "y": 12 + "y": 61 }, "hideTimeOverride": true, "id": 64, @@ -5160,13 +5261,13 @@ "value": "current" } ], - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "fontSize": "90%", "gridPos": { "h": 7, "w": 5, "x": 7, - "y": 12 + "y": 61 }, "hideTimeOverride": true, "id": 66, @@ -5209,7 +5310,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "decimals": 1, "description": "", "fill": 1, @@ -5217,7 +5318,7 @@ "h": 7, "w": 12, "x": 12, - "y": 12 + "y": 61 }, "id": 70, "legend": { @@ -5313,13 +5414,13 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "description": "The time consumed to CDC incremental scan", "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 19 + "y": 68 }, "heatmap": {}, "hideZeroBuckets": true, @@ -5383,7 +5484,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "decimals": 1, "description": "", "fill": 1, @@ -5391,7 +5492,7 @@ "h": 7, "w": 12, "x": 12, - "y": 19 + "y": 68 }, "id": 72, "legend": { @@ -5478,7 +5579,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "decimals": 1, "description": "The memory usage per TiKV instance", "editable": true, @@ -5489,7 +5590,7 @@ "h": 7, "w": 12, "x": 0, - "y": 26 + "y": 75 }, "id": 74, "legend": { @@ -5595,9 +5696,9 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "decimals": 1, - "description": "The memory usage per TiKV instance", + "description": "The speed of TiKV CDC incremental scan", "editable": true, "error": false, "fill": 0, @@ -5606,13 +5707,15 @@ "h": 7, "w": 12, "x": 12, - "y": 26 + "y": 75 }, "id": 76, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, "rightSide": true, @@ -5638,9 +5741,9 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_cdc_min_resolved_ts{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}) by (instance)", + "expr": "sum(rate(tikv_cdc_scan_bytes_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}[30s])) by (instance)", "format": "time_series", - "hide": true, + "hide": false, "intervalFactor": 2, "legendFormat": "tikv-{{instance}}", "refId": "A", @@ -5651,7 +5754,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CDC pending bytes in memory", + "title": "CDC scan speed", "tooltip": { "msResolution": false, "shared": true, @@ -5672,7 +5775,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -5694,7 +5797,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "decimals": 1, "description": "The memory usage per TiKV instance", "editable": true, @@ -5705,7 +5808,7 @@ "h": 7, "w": 12, "x": 0, - "y": 33 + "y": 82 }, "id": 78, "legend": { @@ -5741,9 +5844,18 @@ "format": "time_series", "hide": false, "intervalFactor": 2, - "legendFormat": "tikv-{{instance}}", + "legendFormat": "tikv-{{instance}}-total", "refId": "A", "step": 10 + }, + { + "expr": "avg(tikv_cdc_region_resolve_status{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance, status)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "tikv-{{instance}}-{{status}}", + "refId": "B", + "step": 10 } ], "thresholds": [], @@ -5787,6 +5899,107 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_CDC-CLUSTER3}", + "decimals": 1, + "description": "The total bytes of TiKV CDC incremental scan", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 82 + }, + "id": 138, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(tikv_cdc_scan_bytes_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "tikv-{{instance}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CDC total scan bytes", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "title": "TiKV", @@ -5802,7 +6015,7 @@ { "allValue": null, "current": {}, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "definition": "", "hide": 2, "includeAll": false, @@ -5824,7 +6037,7 @@ { "allValue": ".*", "current": {}, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "definition": "label_values(ticdc_processor_resolved_ts{tidb_cluster=\"$tidb_cluster\"}, changefeed)", "hide": 0, "includeAll": true, @@ -5846,7 +6059,7 @@ { "allValue": ".*", "current": {}, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "definition": "label_values(ticdc_processor_resolved_ts{tidb_cluster=\"$tidb_cluster\"}, capture)", "hide": 0, "includeAll": true, @@ -5868,7 +6081,7 @@ { "allValue": ".*", "current": {}, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_CDC-CLUSTER3}", "definition": "label_values(tikv_engine_size_bytes{tidb_cluster=\"$tidb_cluster\"}, instance)", "hide": 0, "includeAll": true, @@ -5891,6 +6104,7 @@ "allValue": "9999999999", "current": { "selected": true, + "tags": [], "text": "All", "value": "$__all" }, @@ -5943,7 +6157,7 @@ ] }, "time": { - "from": "now-3h", + "from": "now-30m", "to": "now" }, "timepicker": { @@ -5972,7 +6186,7 @@ ] }, "timezone": "browser", - "title": "Test-Cluster-TiCDC", + "title": "cdc-cluster3-TiCDC", "uid": "YiGL8hBZ1", - "version": 13 -} \ No newline at end of file + "version": 2 +}