From 86f57cb78d8576bf61caa02f965037be3f274729 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Fri, 2 Apr 2021 20:10:31 +0800 Subject: [PATCH 1/4] metrics: update TiKV metrics Signed-off-by: Neil Shen --- metrics/grafana/ticdc.json | 229 +++++++++++++++++++++++++++++++++++-- 1 file changed, 218 insertions(+), 11 deletions(-) diff --git a/metrics/grafana/ticdc.json b/metrics/grafana/ticdc.json index 7f63836568c..ee87d9005c9 100644 --- a/metrics/grafana/ticdc.json +++ b/metrics/grafana/ticdc.json @@ -78,7 +78,7 @@ "datasource": "${DS_TEST-CLUSTER}", "enable": true, "expr": "min(up{tidb_cluster=\"$tidb_cluster\", job=~\"tikv|ticdc\"}) by (job, instance) == BOOL 0", - "hide": true, + "hide": false, "iconColor": "#FF9830", "limit": 100, "name": "Server down", @@ -109,7 +109,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1617089343502, + "iteration": 1617365191327, "links": [], "panels": [ { @@ -5389,7 +5389,7 @@ "fill": 1, "gridPos": { "h": 7, - "w": 12, + "w": 6, "x": 12, "y": 19 }, @@ -5402,7 +5402,7 @@ "hideZero": true, "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "sort": "current", @@ -5473,6 +5473,101 @@ "alignLevel": null } }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "The number of incremental scan task in different status.", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 19 + }, + "id": 140, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_cdc_scan_tasks{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}} - {{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Initial scan tasks status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, @@ -5597,7 +5692,7 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The memory usage per TiKV instance", + "description": "The speed of TiKV CDC incremental scan", "editable": true, "error": false, "fill": 0, @@ -5613,6 +5708,8 @@ "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, "rightSide": true, @@ -5638,9 +5735,9 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_cdc_min_resolved_ts{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}) by (instance)", + "expr": "sum(rate(tikv_cdc_scan_bytes_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}[30s])) by (instance)", "format": "time_series", - "hide": true, + "hide": false, "intervalFactor": 2, "legendFormat": "tikv-{{instance}}", "refId": "A", @@ -5651,7 +5748,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CDC pending bytes in memory", + "title": "CDC scan speed", "tooltip": { "msResolution": false, "shared": true, @@ -5672,7 +5769,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -5741,9 +5838,18 @@ "format": "time_series", "hide": false, "intervalFactor": 2, - "legendFormat": "tikv-{{instance}}", + "legendFormat": "tikv-{{instance}}-total", "refId": "A", "step": 10 + }, + { + "expr": "avg(tikv_cdc_region_resolve_status{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance, status)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "tikv-{{instance}}-{{status}}", + "refId": "B", + "step": 10 } ], "thresholds": [], @@ -5787,6 +5893,107 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "The total bytes of TiKV CDC incremental scan", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 139, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(tikv_cdc_scan_bytes_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "tikv-{{instance}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CDC total scan bytes", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "title": "TiKV", @@ -5974,5 +6181,5 @@ "timezone": "browser", "title": "Test-Cluster-TiCDC", "uid": "YiGL8hBZ1", - "version": 13 + "version": 14 } \ No newline at end of file From 2e0fa165b358a72c0c2ef786e46e6e82aa27975d Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Fri, 2 Apr 2021 20:58:17 +0800 Subject: [PATCH 2/4] fix scan tasks metrics Signed-off-by: Neil Shen --- metrics/grafana/ticdc.json | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/metrics/grafana/ticdc.json b/metrics/grafana/ticdc.json index ee87d9005c9..05e998d5206 100644 --- a/metrics/grafana/ticdc.json +++ b/metrics/grafana/ticdc.json @@ -109,7 +109,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1617365191327, + "iteration": 1617365191334, "links": [], "panels": [ { @@ -5520,11 +5520,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_cdc_scan_tasks{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)", + "expr": "sum(tikv_cdc_scan_tasks{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (type, instance)", "format": "time_series", + "hide": false, "intervalFactor": 1, "legendFormat": "{{instance}} - {{type}}", "refId": "A" + }, + { + "expr": "sum(tikv_cdc_scan_tasks{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"total\"}) by (instance) - sum(tikv_cdc_scan_tasks{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"finish\"}) by (instance) - sum(tikv_cdc_scan_tasks{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"abort\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}} - ongoing", + "refId": "B" } ], "thresholds": [], From 18c2965db44e5908b4ceb1b4e007bca5da1c0bf5 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Fri, 2 Apr 2021 21:02:51 +0800 Subject: [PATCH 3/4] move ongoing to yaxis Signed-off-by: Neil Shen --- metrics/grafana/ticdc.json | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/metrics/grafana/ticdc.json b/metrics/grafana/ticdc.json index 05e998d5206..f40f7a5c0f9 100644 --- a/metrics/grafana/ticdc.json +++ b/metrics/grafana/ticdc.json @@ -109,7 +109,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1617365191334, + "iteration": 1617365191337, "links": [], "panels": [ { @@ -5514,7 +5514,12 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*ongoing/", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, @@ -5564,12 +5569,12 @@ "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { From dad201cbd3b7ab0b0b63156e69a0ceab9735c8f1 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Fri, 2 Apr 2021 21:08:54 +0800 Subject: [PATCH 4/4] Apply suggestions from code review --- metrics/grafana/ticdc.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metrics/grafana/ticdc.json b/metrics/grafana/ticdc.json index f40f7a5c0f9..0f3837331d2 100644 --- a/metrics/grafana/ticdc.json +++ b/metrics/grafana/ticdc.json @@ -5857,7 +5857,7 @@ "step": 10 }, { - "expr": "avg(tikv_cdc_region_resolve_status{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance, status)", + "expr": "sum(tikv_cdc_region_resolve_status{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance, status)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -6196,4 +6196,4 @@ "title": "Test-Cluster-TiCDC", "uid": "YiGL8hBZ1", "version": 14 -} \ No newline at end of file +}