From 0ce917719b124de4440c87246b5f72f9fa5c5755 Mon Sep 17 00:00:00 2001 From: Glitter Date: Tue, 9 Jul 2019 13:20:11 +0800 Subject: [PATCH 1/3] add metrics of titan to grafana dashboard (#814) * add metrics of titan --- scripts/tikv_details.json | 2032 ++++++++++++++++++++++++++++++++++++- 1 file changed, 2031 insertions(+), 1 deletion(-) diff --git a/scripts/tikv_details.json b/scripts/tikv_details.json index 619feee8c..810e182c9 100644 --- a/scripts/tikv_details.json +++ b/scripts/tikv_details.json @@ -17225,7 +17225,2037 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 25 + }, + "id": 3301, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 26 + }, + "id": 3414, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "key_size_avg", + "refId": "A" + }, + { + "expr": "quantile(0.95, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "key_size_0.95", + "refId": "C" + }, + { + "expr": "quantile(0.99, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "key_size_0.99", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob key size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 26 + }, + "id": 3446, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "value_size", + "refId": "A" + }, + { + "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "value_size_0.95", + "refId": "B" + }, + { + "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "value_size_0.99", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob value size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 32 + }, + "id": 3412, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "seek_micros", + "refId": "A" + }, + { + "expr": "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "prev_micros", + "refId": "B" + }, + { + "expr": "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "next_micros", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob seek duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "µs", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 32 + }, + "id": 3410, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "blob_gc_micros", + "refId": "A" + }, + { + "expr": "quantile(0.95, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "blob_gc_micros_0.95", + "refId": "B" + }, + { + "expr": "quantile(0.99, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "blob_gc_micros_0.99", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "µs", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 38 + }, + "id": 3342, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "file_bytes_written", + "refId": "A" + }, + { + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "file_bytes_read", + "refId": "B" + }, + { + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "keys_read", + "refId": "C" + }, + { + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "keys_written", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob flow", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 38 + }, + "id": 3408, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_average\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "A" + }, + { + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile99\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "99%", + "refId": "B" + }, + { + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile95\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "95%", + "refId": "C" + }, + { + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_max\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC file write duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "µs", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 44 + }, + "id": 3344, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_input_files_count\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "gc_num_files", + "refId": "B" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_output_files_count\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "gc_num_new_files", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob GC file", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 44 + }, + "id": 3338, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_seek\"}[2m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "seek", + "refId": "A" + }, + { + "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_prev\"}[2m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "prev", + "refId": "B" + }, + { + "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_next\"}[2m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "next", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob seek operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 50 + }, + "id": 3340, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes_written", + "refId": "A" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes_read", + "refId": "B" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_overwritten\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes_overwritten", + "refId": "C" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_relocated\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes_relocated", + "refId": "D" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_relocated\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "keys_relocated", + "refId": "E" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_overwritten\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "keys_overwritten", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob GC flows", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "repeat": "db", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, + "title": "Titan - $db", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 3512, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 28 + }, + "id": 3513, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1562643407086, + "repeatPanelId": 3414, + "repeatedByRow": true, + "scopedVars": { + "db": { + "selected": false, + "text": "raft", + "value": "raft" + } + }, + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "key_size_avg", + "refId": "A" + }, + { + "expr": "quantile(0.95, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "key_size_0.95", + "refId": "C" + }, + { + "expr": "quantile(0.99, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "key_size_0.99", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob key size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 28 + }, + "id": 3514, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1562643407086, + "repeatPanelId": 3446, + "repeatedByRow": true, + "scopedVars": { + "db": { + "selected": false, + "text": "raft", + "value": "raft" + } + }, + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "value_size", + "refId": "A" + }, + { + "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "value_size_0.95", + "refId": "B" + }, + { + "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "value_size_0.99", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob value size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 34 + }, + "id": 3515, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1562643407086, + "repeatPanelId": 3412, + "repeatedByRow": true, + "scopedVars": { + "db": { + "selected": false, + "text": "raft", + "value": "raft" + } + }, + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "seek_micros", + "refId": "A" + }, + { + "expr": "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "prev_micros", + "refId": "B" + }, + { + "expr": "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "next_micros", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob seek duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "µs", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 34 + }, + "id": 3516, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1562643407086, + "repeatPanelId": 3410, + "repeatedByRow": true, + "scopedVars": { + "db": { + "selected": false, + "text": "raft", + "value": "raft" + } + }, + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "blob_gc_micros", + "refId": "A" + }, + { + "expr": "quantile(0.95, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "blob_gc_micros_0.95", + "refId": "B" + }, + { + "expr": "quantile(0.99, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "blob_gc_micros_0.99", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "µs", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 40 + }, + "id": 3517, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1562643407086, + "repeatPanelId": 3342, + "repeatedByRow": true, + "scopedVars": { + "db": { + "selected": false, + "text": "raft", + "value": "raft" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "file_bytes_written", + "refId": "A" + }, + { + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "file_bytes_read", + "refId": "B" + }, + { + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "keys_read", + "refId": "C" + }, + { + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "keys_written", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob flow", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 40 + }, + "id": 3518, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1562643407086, + "repeatPanelId": 3408, + "repeatedByRow": true, + "scopedVars": { + "db": { + "selected": false, + "text": "raft", + "value": "raft" + } + }, + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_average\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "A" + }, + { + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile99\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "99%", + "refId": "B" + }, + { + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile95\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "95%", + "refId": "C" + }, + { + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_max\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC file write duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "µs", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 46 + }, + "id": 3519, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1562643407086, + "repeatPanelId": 3344, + "repeatedByRow": true, + "scopedVars": { + "db": { + "selected": false, + "text": "raft", + "value": "raft" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_input_files_count\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "gc_num_files", + "refId": "B" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_output_files_count\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "gc_num_new_files", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob GC file", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 46 + }, + "id": 3520, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1562643407086, + "repeatPanelId": 3338, + "repeatedByRow": true, + "scopedVars": { + "db": { + "selected": false, + "text": "raft", + "value": "raft" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_seek\"}[2m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "seek", + "refId": "A" + }, + { + "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_prev\"}[2m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "prev", + "refId": "B" + }, + { + "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_next\"}[2m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "next", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob seek operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 52 + }, + "id": 3521, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeatIteration": 1562643407086, + "repeatPanelId": 3340, + "repeatedByRow": true, + "scopedVars": { + "db": { + "selected": false, + "text": "raft", + "value": "raft" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes_written", + "refId": "A" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes_read", + "refId": "B" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_overwritten\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes_overwritten", + "refId": "C" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_relocated\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes_relocated", + "refId": "D" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_relocated\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "keys_relocated", + "refId": "E" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_overwritten\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "keys_overwritten", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob GC flows", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "repeat": null, + "repeatIteration": 1562643407086, + "repeatPanelId": 3301, + "scopedVars": { + "db": { + "selected": false, + "text": "raft", + "value": "raft" + } + }, + "title": "Titan - $db", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 }, "id": 2763, "panels": [ From eface552ad65fbc4ea528ad6a3ad279aad6eec13 Mon Sep 17 00:00:00 2001 From: Glitter Date: Fri, 12 Jul 2019 15:21:02 +0800 Subject: [PATCH 2/3] Add metrics of blob file status to grafana (#821) * add metrics for blob file of titan to grafana and fix a typo --- scripts/tikv_details.json | 299 +++++++++++++++++++++++++++++++++++++- 1 file changed, 298 insertions(+), 1 deletion(-) diff --git a/scripts/tikv_details.json b/scripts/tikv_details.json index 810e182c9..91a996f36 100644 --- a/scripts/tikv_details.json +++ b/scripts/tikv_details.json @@ -17181,7 +17181,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compression reason", + "title": "Compaction reason", "tooltip": { "shared": true, "sort": 0, @@ -18175,6 +18175,303 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 51 + }, + "id": 3523, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tikv_engine_titandb_live_blob_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "live blob size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Live blob size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 57 + }, + "id": 3555, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tikv_engine_titandb_num_live_blob_file{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "live blob file num", + "refId": "A" + }, + { + "expr": "avg(tikv_engine_titandb_num_obsolete_blob_file{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "obsolete blob file num", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob file", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 57 + }, + "id": 3557, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(tikv_engine_titandb_live_blob_file_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "live blob file size", + "refId": "A" + }, + { + "expr": "avg(tikv_engine_titandb_obsolete_blob_file_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "obsolete blob file size", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blob file size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "repeat": "db", From c670efb02973e0348d5b22268ec0fd75a7ee8411 Mon Sep 17 00:00:00 2001 From: liubo0127 Date: Fri, 12 Jul 2019 16:54:19 +0800 Subject: [PATCH 3/3] Update titan metrics --- scripts/tikv_details.json | 6018 +++++++++++++++---------------------- 1 file changed, 2507 insertions(+), 3511 deletions(-) diff --git a/scripts/tikv_details.json b/scripts/tikv_details.json index 91a996f36..f268a34e0 100644 --- a/scripts/tikv_details.json +++ b/scripts/tikv_details.json @@ -14,25 +14,31 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "5.4.3" + "version": "6.1.6" }, { "type": "panel", "id": "graph", "name": "Graph", - "version": "5.0.0" + "version": "" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", - "version": "5.0.0" + "version": "1.0.0" }, { "type": "panel", "id": "singlestat", "name": "Singlestat", - "version": "5.0.0" + "version": "" } ], "annotations": { @@ -52,7 +58,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1556505746692, + "iteration": 1562921521045, "links": [ { "icon": "doc", @@ -8234,7 +8240,7 @@ "lines": true, "linewidth": 1, "links": [], - "minSpan": 24, + "maxPerRow": 1, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8332,7 +8338,7 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": 12, + "maxPerRow": 2, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8459,7 +8465,7 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": 12, + "maxPerRow": 2, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8572,7 +8578,7 @@ "lines": true, "linewidth": 1, "links": [], - "minSpan": 24, + "maxPerRow": 1, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8676,7 +8682,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8794,7 +8799,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8909,7 +8913,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -9024,7 +9027,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -9139,7 +9141,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -9238,7 +9239,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -9337,7 +9337,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -9436,7 +9435,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -9511,73 +9509,10 @@ "h": 1, "w": 24, "x": 0, - "y": 19 + "y": 15 }, - "id": 2757, + "id": 2758, "panels": [ - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#5195ce", - "colorScale": "linear", - "colorScheme": "interpolateBlues", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 19 - }, - "heatmap": {}, - "highlightCards": true, - "id": 3062, - "legend": { - "show": true - }, - "links": [], - "targets": [ - { - "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Request duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, { "aliasColors": {}, "bars": false, @@ -9585,35 +9520,31 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 12, - "y": 19 + "x": 0, + "y": 322 }, - "id": 16, + "id": 26, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -9626,30 +9557,46 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", - "format": "time_series", + "expr": "histogram_quantile(1.0, sum(rate(tikv_storage_mvcc_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{req}}-100%", - "refId": "E" + "legendFormat": " max", + "metric": "", + "refId": "A", + "step": 4 }, { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", - "format": "time_series", + "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_mvcc_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{req}}-99%", - "refId": "A", + "legendFormat": "99%", + "metric": "", + "refId": "B", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_mvcc_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": " 95%", + "metric": "", + "refId": "C", + "step": 4 + }, + { + "expr": "sum(rate(tikv_storage_mvcc_versions_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_storage_mvcc_versions_count{instance=~\"$instance\"}[1m])) ", + "intervalFactor": 2, + "legendFormat": "avg", + "metric": "", + "refId": "D", "step": 4 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Request duration", + "title": "MVCC versions", "tooltip": { "msResolution": false, "shared": true, - "sort": 1, + "sort": 2, "value_type": "cumulative" }, "type": "graph", @@ -9662,9 +9609,8 @@ }, "yaxes": [ { - "decimals": 1, - "format": "s", - "label": null, + "format": "short", + "label": "", "logBase": 1, "max": null, "min": "0", @@ -9676,13 +9622,9 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -9698,10 +9640,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 27 + "x": 12, + "y": 322 }, - "id": 74, + "id": 559, "legend": { "alignAsTable": true, "avg": false, @@ -9710,14 +9652,12 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -9730,28 +9670,48 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_count{instance=~\"$instance\"}[1m])) by (req)", - "format": "time_series", - "interval": "", + "expr": "histogram_quantile(1.0, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{req}}", - "metric": "tikv_coprocessor_request_error", + "legendFormat": " max", + "metric": "", "refId": "A", "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "99%", + "metric": "", + "refId": "B", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": " 95%", + "metric": "", + "refId": "C", + "step": 4 + }, + { + "expr": "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_storage_mvcc_gc_delete_versions_count{instance=~\"$instance\"}[1m])) ", + "intervalFactor": 2, + "legendFormat": "avg", + "metric": "", + "refId": "D", + "step": 4 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total Requests", + "title": "MVCC delete versions", "tooltip": { "msResolution": false, "shared": true, - "sort": 0, - "value_type": "individual" + "sort": 2, + "value_type": "cumulative" }, - "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -9762,8 +9722,7 @@ }, "yaxes": [ { - "decimals": 1, - "format": "ops", + "format": "short", "label": "", "logBase": 1, "max": null, @@ -9776,13 +9735,9 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -9791,17 +9746,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 27 + "x": 0, + "y": 329 }, - "id": 3128, + "id": 121, "legend": { "alignAsTable": true, "avg": false, @@ -9810,9 +9762,7 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, @@ -9830,28 +9780,46 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_request_error{instance=~\"$instance\"}[1m])) by (reason)", + "expr": "sum(rate(tikv_gcworker_gc_tasks_vec{instance=~\"$instance\"}[1m])) by (task)", "format": "time_series", - "interval": "", "intervalFactor": 2, - "legendFormat": "{{reason}}", - "metric": "tikv_coprocessor_request_error", + "legendFormat": "total-{{task}}", + "metric": "tikv_storage_command_total", "refId": "A", "step": 4 + }, + { + "expr": "sum(rate(tikv_storage_gc_skipped_counter{instance=~\"$instance\"}[1m])) by (task)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "skipped-{{task}}", + "metric": "tikv_storage_gc_skipped_counter", + "refId": "B", + "step": 4 + }, + { + "expr": "sum(rate(tikv_gcworker_gc_task_fail_vec{instance=~\"$instance\"}[1m])) by (task)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "failed-{{task}}", + "refId": "C" + }, + { + "expr": "sum(rate(tikv_gc_worker_too_busy{instance=~\"$instance\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "refId": "D" } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total Request Errors", + "title": "GC tasks", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, - "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -9862,12 +9830,11 @@ }, "yaxes": [ { - "decimals": 1, - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -9876,13 +9843,9 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -9891,17 +9854,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 34 + "x": 12, + "y": 329 }, - "id": 52, + "id": 2224, "legend": { "alignAsTable": true, "avg": false, @@ -9910,9 +9870,7 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, @@ -9930,20 +9888,43 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_scan_keys_sum{instance=~\"$instance\"}[1m])) by (req)", + "expr": "histogram_quantile(1, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{req}}", + "legendFormat": "max-{{task}}", + "metric": "tikv_storage_command_total", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%-{{task}}", + "metric": "tikv_storage_gc_skipped_counter", + "refId": "B", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "95%-{{task}}", + "refId": "C" + }, + { + "expr": "sum(rate(tikv_gcworker_gc_task_duration_vec_sum{instance=~\"$instance\"}[1m])) by (task) / sum(rate(tikv_gcworker_gc_task_duration_vec_count{instance=~\"$instance\"}[1m])) by (task)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "average-{{task}}", "refId": "D" } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total KV Cursor Operations", + "title": "GC tasks duration", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -9958,12 +9939,11 @@ }, "yaxes": [ { - "decimals": 0, - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -9972,13 +9952,9 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -9987,17 +9963,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 34 + "x": 0, + "y": 336 }, - "id": 3129, + "id": 2225, "legend": { "alignAsTable": true, "avg": false, @@ -10006,9 +9979,7 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, @@ -10026,27 +9997,20 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\"$instance\"}[1m])) by (le, req)) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "100%-{{req}}", - "refId": "D" - }, - { - "expr": "histogram_quantile(0.99, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\"$instance\"}[1m])) by (le, req)) ", + "expr": "sum(rate(tikv_gcworker_gc_keys{instance=~\"$instance\", cf=\"write\"}[1m])) by (tag)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "99%-{{req}}", - "refId": "A" + "legendFormat": "{{tag}}", + "metric": "tikv_storage_command_total", + "refId": "A", + "step": 4 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "KV Cursor Operations", + "title": "GC keys (write CF)", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -10061,7 +10025,6 @@ }, "yaxes": [ { - "decimals": 0, "format": "short", "label": null, "logBase": 1, @@ -10075,13 +10038,9 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -10089,8 +10048,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "", + "decimals": 2, "editable": true, "error": false, "fill": 1, @@ -10098,10 +10056,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 41 + "x": 12, + "y": 336 }, - "id": 2118, + "id": 966, "legend": { "alignAsTable": true, "avg": false, @@ -10110,51 +10068,42 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "key_skipped", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\" ,metric=\"internal_delete_skipped_count\"}[1m])) by (req)", + "expr": "sum(rate(tidb_tikvclient_gc_worker_actions_total[1m])) by (type)", "format": "time_series", - "interval": "", "intervalFactor": 2, - "legendFormat": "delete_skipped-{{req}}", - "metric": "scan_details", - "refId": "B", + "legendFormat": "{{type}}", + "metric": "", + "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total RocksDB Perf Statistics", + "title": "TiDB GC worker actions", "tooltip": { "msResolution": false, "shared": true, - "sort": 0, - "value_type": "individual" + "sort": 2, + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -10166,28 +10115,22 @@ }, "yaxes": [ { - "decimals": 1, "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { - "decimals": null, "format": "short", - "label": "", + "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -10195,7 +10138,6 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, "editable": true, "error": false, "fill": 1, @@ -10203,26 +10145,25 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 41 + "x": 0, + "y": 343 }, - "id": 551, + "id": 969, "legend": { "alignAsTable": true, "avg": false, - "current": true, + "current": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sort": null, + "sortDesc": null, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -10235,26 +10176,23 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_response_bytes{instance=~\"$instance\"}[1m]))", + "expr": "histogram_quantile(1.0, sum(rate(tidb_tikvclient_gc_seconds_bucket[1m])) by (instance, le))", "format": "time_series", - "interval": "", "intervalFactor": 2, - "legendFormat": "size", - "metric": "", + "legendFormat": "{{instance}}", "refId": "A", - "step": 4 + "step": 40 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total Response Size", + "title": "TiDB GC seconds", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -10266,12 +10204,11 @@ }, "yaxes": [ { - "decimals": 0, - "format": "decbytes", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -10280,29 +10217,10 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "Coprocessor Overview", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 3197, - "panels": [ + ] + }, { "aliasColors": {}, "bars": false, @@ -10310,6 +10228,7 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "description": "keys / second", "editable": true, "error": false, "fill": 1, @@ -10317,10 +10236,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 20 + "x": 12, + "y": 343 }, - "id": 113, + "id": 2589, "legend": { "alignAsTable": true, "avg": false, @@ -10329,13 +10248,12 @@ "min": false, "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -10348,30 +10266,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", + "expr": "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum[1m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{req}}-100%", + "legendFormat": "keys/s", "refId": "E" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-99%", - "refId": "A", - "step": 4 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Handle duration", + "title": "GC speed", "tooltip": { "msResolution": false, "shared": true, - "sort": 1, + "sort": 2, "value_type": "cumulative" }, "type": "graph", @@ -10384,8 +10293,7 @@ }, "yaxes": [ { - "decimals": 1, - "format": "s", + "format": "short", "label": "", "logBase": 1, "max": null, @@ -10393,38 +10301,30 @@ "show": true }, { - "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, - "bars": false, + "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "editable": true, - "error": false, + "decimals": 0, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 20 + "x": 0, + "y": 350 }, - "id": 117, + "id": 2108, "legend": { "alignAsTable": true, "avg": false, @@ -10433,13 +10333,11 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, - "lines": true, + "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null", @@ -10449,28 +10347,28 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,req))", + "expr": "sum(max_over_time(tikv_gcworker_autogc_status{state=\"working\"}[1m])) by (job)", "format": "time_series", + "instant": false, "intervalFactor": 2, - "legendFormat": "{{instance}}-{{req}}", - "refId": "B", + "legendFormat": "{{job}}", + "metric": "tikv_storage_command_total", + "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "95% Handle duration by store", + "title": "TiKV AutoGC Working", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -10482,12 +10380,12 @@ }, "yaxes": [ { - "decimals": 1, - "format": "s", + "decimals": 0, + "format": "none", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -10498,47 +10396,223 @@ "min": null, "show": false } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" ], - "yaxis": { - "align": false, - "alignLevel": null - } + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 350 + }, + "id": 27, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_life_time\"})", + "interval": "", + "intervalFactor": 2, + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "GC lifetime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 350 + }, + "id": 28, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_run_interval\"})", + "intervalFactor": 2, + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "GC interval", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "title": "GC", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 2759, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, - "w": 12, + "w": 8, "x": 0, - "y": 27 + "y": 351 }, - "id": 111, + "id": 35, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, "max": true, "min": false, "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -10549,26 +10623,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-100%", - "refId": "D" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", - "format": "time_series", + "expr": "sum(delta(tikv_raftstore_raft_sent_message_total{instance=~\"$instance\", type=\"snapshot\"}[1m]))", "intervalFactor": 2, - "legendFormat": "{{req}}-99%", + "legendFormat": " ", "refId": "A", - "step": 4 + "step": 60 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Wait duration", + "title": "Rate snapshot message", "tooltip": { "msResolution": false, "shared": true, @@ -10585,12 +10650,11 @@ }, "yaxes": [ { - "decimals": 1, - "format": "s", + "format": "opm", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -10599,13 +10663,9 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -10613,35 +10673,33 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 27 + "w": 8, + "x": 8, + "y": 351 }, - "id": 116, + "id": 36, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -10652,19 +10710,31 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,req))", - "format": "time_series", + "expr": "histogram_quantile(0.99, sum(rate(tikv_server_send_snapshot_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{instance}}-{{req}}", + "legendFormat": "send", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\"$instance\", type=\"apply\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "apply", "refId": "B", - "step": 4 + "step": 60 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\"$instance\", type=\"generate\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "generate", + "refId": "C", + "step": 60 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "95% Wait duration by store", + "title": "99% Handle snapshot duration", "tooltip": { "msResolution": false, "shared": true, @@ -10681,28 +10751,22 @@ }, "yaxes": [ { - "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -10710,36 +10774,33 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, - "w": 12, - "x": 0, - "y": 34 + "w": 8, + "x": 16, + "y": 351 }, - "id": 3195, + "id": 38, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -10747,29 +10808,26 @@ "seriesOverrides": [], "spaceLength": 10, "stack": false, - "steppedLine": false, + "steppedLine": true, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_dag_request_count{instance=~\"$instance\"}[1m])) by (vec_type)", - "format": "time_series", - "interval": "", + "expr": "sum(tikv_raftstore_snapshot_traffic_total{instance=~\"$instance\"}) by (type)", "intervalFactor": 2, - "legendFormat": "{{vec_type}}", + "legendFormat": "{{type}}", "metric": "", "refId": "A", - "step": 4 + "step": 60 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total DAG Requests", + "title": "Snapshot state count", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -10781,12 +10839,11 @@ }, "yaxes": [ { - "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -10795,13 +10852,9 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -10809,7 +10862,6 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, "editable": true, "error": false, "fill": 1, @@ -10817,28 +10869,26 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 34 + "x": 0, + "y": 358 }, - "id": 3264, + "id": 44, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -10849,21 +10899,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_executor_count{instance=~\"$instance\"}[1m])) by (type)", - "format": "time_series", - "interval": "", + "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_size_bucket{instance=~\"$instance\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "", + "legendFormat": "size", + "metric": "tikv_snapshot_size_bucket", "refId": "A", - "step": 4 + "step": 40 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total DAG Executors", + "title": "99.99% Snapshot size", "tooltip": { "msResolution": false, "shared": true, @@ -10880,12 +10927,11 @@ }, "yaxes": [ { - "decimals": 1, - "format": "short", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -10894,13 +10940,9 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -10908,29 +10950,26 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { - "h": 9, + "h": 7, "w": 12, - "x": 0, - "y": 41 + "x": 12, + "y": 358 }, - "id": 552, + "id": 43, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, @@ -10948,21 +10987,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"select\"}[1m])) by (tag)", - "format": "time_series", - "interval": "", + "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_kv_count_bucket{instance=~\"$instance\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{tag}}", - "metric": "scan_details", - "refId": "B", - "step": 4 + "legendFormat": "count", + "metric": "tikv_snapshot_kv_count_bucket", + "refId": "A", + "step": 40 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total Ops Details (Table Scan)", + "title": "99.99% Snapshot KV count", "tooltip": { "msResolution": false, "shared": true, @@ -10979,12 +11015,11 @@ }, "yaxes": [ { - "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -10993,14 +11028,25 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, + ] + } + ], + "repeat": null, + "title": "Snapshot", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 2760, + "panels": [ { "aliasColors": {}, "bars": false, @@ -11013,30 +11059,30 @@ "fill": 1, "grid": {}, "gridPos": { - "h": 9, + "h": 8, "w": 12, - "x": 12, - "y": 41 + "x": 0, + "y": 359 }, - "id": 3263, + "id": 59, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 400, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -11047,21 +11093,17 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"index\"}[1m])) by (tag)", - "format": "time_series", - "interval": "", + "expr": "sum(rate(tikv_worker_handled_task_total{instance=~\"$instance\"}[1m])) by (name)", "intervalFactor": 2, - "legendFormat": "{{tag}}", - "metric": "scan_details", - "refId": "B", + "legendFormat": "{{name}}", + "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total Ops Details (Index Scan)", + "title": "Worker handled tasks", "tooltip": { "msResolution": false, "shared": true, @@ -11078,8 +11120,7 @@ }, "yaxes": [ { - "decimals": 1, - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -11092,13 +11133,9 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -11112,23 +11149,23 @@ "fill": 1, "grid": {}, "gridPos": { - "h": 9, + "h": 8, "w": 12, - "x": 0, - "y": 50 + "x": 12, + "y": 359 }, - "id": 122, + "id": 1395, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 400, "total": false, "values": true }, @@ -11140,28 +11177,23 @@ "pointradius": 5, "points": false, "renderer": "flot", - "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"select\"}[1m])) by (tag,cf)", - "format": "time_series", - "interval": "", + "expr": "sum(rate(tikv_worker_pending_task_total{instance=~\"$instance\"}[1m])) by (name)", "intervalFactor": 2, - "legendFormat": "{{cf}}-{{tag}}", - "metric": "scan_details", - "refId": "B", + "legendFormat": "{{name}}", + "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total Ops Details by CF (Table Scan)", + "title": "Worker pending tasks", "tooltip": { "msResolution": false, "shared": true, @@ -11178,7 +11210,6 @@ }, "yaxes": [ { - "decimals": 1, "format": "short", "label": null, "logBase": 1, @@ -11192,13 +11223,9 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -11212,23 +11239,23 @@ "fill": 1, "grid": {}, "gridPos": { - "h": 9, + "h": 8, "w": 12, - "x": 12, - "y": 50 + "x": 0, + "y": 367 }, - "id": 554, + "id": 1876, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 400, "total": false, "values": true }, @@ -11240,29 +11267,24 @@ "pointradius": 5, "points": false, "renderer": "flot", - "repeat": "cf", - "repeatDirection": "h", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"index\"}[1m])) by (tag,cf)", + "expr": "sum(rate(tikv_futurepool_handled_task_total{instance=~\"$instance\"}[1m])) by (name)", "format": "time_series", - "interval": "", "intervalFactor": 2, - "legendFormat": "{{cf}}-{{tag}}", - "metric": "scan_details", - "refId": "B", + "legendFormat": "{{name}}", + "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total Ops Details by CF (Index Scan)", + "title": "FuturePool handled tasks", "tooltip": { "msResolution": false, "shared": true, @@ -11279,8 +11301,7 @@ }, "yaxes": [ { - "decimals": 1, - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -11293,28 +11314,10 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "title": "Coprocessor Detail", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 2758, - "panels": [ + ] + }, { "aliasColors": {}, "bars": false, @@ -11327,28 +11330,30 @@ "fill": 1, "grid": {}, "gridPos": { - "h": 7, + "h": 8, "w": 12, - "x": 0, - "y": 322 + "x": 12, + "y": 367 }, - "id": 26, + "id": 1877, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 250, + "sideWidth": 400, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -11359,47 +11364,23 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1.0, sum(rate(tikv_storage_mvcc_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", + "expr": "sum(rate(tikv_futurepool_pending_task_total{instance=~\"$instance\"}[1m])) by (name)", + "format": "time_series", "intervalFactor": 2, - "legendFormat": " max", - "metric": "", + "legendFormat": "{{name}}", "refId": "A", "step": 4 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_mvcc_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "B", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_mvcc_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": " 95%", - "metric": "", - "refId": "C", - "step": 4 - }, - { - "expr": "sum(rate(tikv_storage_mvcc_versions_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_storage_mvcc_versions_count{instance=~\"$instance\"}[1m])) ", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "D", - "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "MVCC versions", + "title": "FuturePool pending tasks", "tooltip": { "msResolution": false, "shared": true, - "sort": 2, - "value_type": "cumulative" + "sort": 0, + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -11412,7 +11393,7 @@ "yaxes": [ { "format": "short", - "label": "", + "label": null, "logBase": 1, "max": null, "min": "0", @@ -11427,1148 +11408,84 @@ "show": true } ] - }, + } + ], + "repeat": null, + "title": "Task", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 2757, + "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#5195ce", + "colorScale": "linear", + "colorScheme": "interpolateBlues", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, "gridPos": { - "h": 7, + "h": 8, "w": 12, - "x": 12, - "y": 322 + "x": 0, + "y": 19 }, - "id": 559, + "heatmap": {}, + "highlightCards": true, + "id": 3062, "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true + "show": true }, - "lines": true, - "linewidth": 2, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(1.0, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": " max", - "metric": "", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "B", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": " 95%", - "metric": "", - "refId": "C", - "step": 4 - }, - { - "expr": "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_storage_mvcc_gc_delete_versions_count{instance=~\"$instance\"}[1m])) ", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "D", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "MVCC delete versions", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 329 - }, - "id": 121, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_gcworker_gc_tasks_vec{instance=~\"$instance\"}[1m])) by (task)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "total-{{task}}", - "metric": "tikv_storage_command_total", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(rate(tikv_storage_gc_skipped_counter{instance=~\"$instance\"}[1m])) by (task)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "skipped-{{task}}", - "metric": "tikv_storage_gc_skipped_counter", - "refId": "B", - "step": 4 - }, - { - "expr": "sum(rate(tikv_gcworker_gc_task_fail_vec{instance=~\"$instance\"}[1m])) by (task)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed-{{task}}", - "refId": "C" - }, - { - "expr": "sum(rate(tikv_gc_worker_too_busy{instance=~\"$instance\"}[1m]))", - "format": "time_series", - "intervalFactor": 2, - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "GC tasks", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 329 - }, - "id": 2224, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(1, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max-{{task}}", - "metric": "tikv_storage_command_total", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%-{{task}}", - "metric": "tikv_storage_gc_skipped_counter", - "refId": "B", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%-{{task}}", - "refId": "C" - }, - { - "expr": "sum(rate(tikv_gcworker_gc_task_duration_vec_sum{instance=~\"$instance\"}[1m])) by (task) / sum(rate(tikv_gcworker_gc_task_duration_vec_count{instance=~\"$instance\"}[1m])) by (task)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "average-{{task}}", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "GC tasks duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 336 - }, - "id": 2225, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_gcworker_gc_keys{instance=~\"$instance\", cf=\"write\"}[1m])) by (tag)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{tag}}", - "metric": "tikv_storage_command_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "GC keys (write CF)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 336 - }, - "id": 966, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tidb_tikvclient_gc_worker_actions_total[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "TiDB GC worker actions", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 343 - }, - "id": 969, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": null, - "sortDesc": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(1.0, sum(rate(tidb_tikvclient_gc_seconds_bucket[1m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 40 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "TiDB GC seconds", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "keys / second", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 343 - }, - "id": 2589, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum[1m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "keys/s", - "refId": "E" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "GC speed", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 350 - }, - "id": 2108, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(max_over_time(tikv_gcworker_autogc_status{state=\"working\"}[1m])) by (job)", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{job}}", - "metric": "tikv_storage_command_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "TiKV AutoGC Working", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, - "editable": true, - "error": false, - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 350 - }, - "id": 27, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "null", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_life_time\"})", - "interval": "", - "intervalFactor": 2, - "refId": "A", - "step": 60 - } - ], - "thresholds": "", - "title": "GC lifetime", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, - "editable": true, - "error": false, - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 350 - }, - "id": 28, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "null", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_run_interval\"})", - "intervalFactor": 2, - "refId": "A", - "step": 60 - } - ], - "thresholds": "", - "title": "GC interval", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "title": "GC", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 2759, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 351 - }, - "id": 35, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(delta(tikv_raftstore_raft_sent_message_total{instance=~\"$instance\", type=\"snapshot\"}[1m]))", - "intervalFactor": 2, - "legendFormat": " ", - "refId": "A", - "step": 60 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Rate snapshot message", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 351 - }, - "id": 36, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_server_send_snapshot_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "send", - "refId": "A", - "step": 60 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\"$instance\", type=\"apply\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "apply", - "refId": "B", - "step": 60 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\"$instance\", type=\"generate\"}[1m])) by (le))", + "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, "intervalFactor": 2, - "legendFormat": "generate", - "refId": "C", - "step": 60 + "legendFormat": "{{le}}", + "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "99% Handle snapshot duration", + "title": "Request duration", "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" + "show": true, + "showHistogram": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null }, { "aliasColors": {}, @@ -12576,17 +11493,19 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 351 + "h": 8, + "w": 12, + "x": 12, + "y": 19 }, - "id": 38, + "id": 16, "legend": { "alignAsTable": true, "avg": false, @@ -12596,13 +11515,16 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -12610,25 +11532,33 @@ "seriesOverrides": [], "spaceLength": 10, "stack": false, - "steppedLine": true, + "steppedLine": false, "targets": [ { - "expr": "sum(tikv_raftstore_snapshot_traffic_total{instance=~\"$instance\"}) by (type)", + "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "", + "legendFormat": "{{req}}-100%", + "refId": "E" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{req}}-99%", "refId": "A", - "step": 60 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Snapshot state count", + "title": "Request duration", "tooltip": { "msResolution": false, "shared": true, - "sort": 0, + "sort": 1, "value_type": "cumulative" }, "type": "graph", @@ -12641,11 +11571,12 @@ }, "yaxes": [ { - "format": "short", + "decimals": 1, + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -12654,9 +11585,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -12664,6 +11599,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, @@ -12672,18 +11608,20 @@ "h": 7, "w": 12, "x": 0, - "y": 358 + "y": 27 }, - "id": 44, + "id": 74, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -12701,24 +11639,28 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_size_bucket{instance=~\"$instance\"}[1m])) by (le))", + "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_count{instance=~\"$instance\"}[1m])) by (req)", + "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "size", - "metric": "tikv_snapshot_size_bucket", + "legendFormat": "{{req}}", + "metric": "tikv_coprocessor_request_error", "refId": "A", - "step": 40 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "99.99% Snapshot size", + "title": "Total Requests", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, + "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -12729,11 +11671,12 @@ }, "yaxes": [ { - "format": "bytes", - "label": null, + "decimals": 1, + "format": "ops", + "label": "", "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -12742,9 +11685,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -12752,6 +11699,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, @@ -12760,18 +11708,20 @@ "h": 7, "w": 12, "x": 12, - "y": 358 + "y": 27 }, - "id": 43, + "id": 3128, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -12789,24 +11739,28 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_kv_count_bucket{instance=~\"$instance\"}[1m])) by (le))", + "expr": "sum(rate(tikv_coprocessor_request_error{instance=~\"$instance\"}[1m])) by (reason)", + "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "count", - "metric": "tikv_snapshot_kv_count_bucket", + "legendFormat": "{{reason}}", + "metric": "tikv_coprocessor_request_error", "refId": "A", - "step": 40 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "99.99% Snapshot KV count", + "title": "Total Request Errors", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, + "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -12817,11 +11771,12 @@ }, "yaxes": [ { - "format": "short", + "decimals": 1, + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -12830,25 +11785,14 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] - } - ], - "repeat": null, - "title": "Snapshot", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 2760, - "panels": [ + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, @@ -12861,30 +11805,30 @@ "fill": 1, "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, - "y": 359 + "y": 34 }, - "id": 59, + "id": 52, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 400, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -12895,17 +11839,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_worker_handled_task_total{instance=~\"$instance\"}[1m])) by (name)", + "expr": "sum(rate(tikv_coprocessor_scan_keys_sum{instance=~\"$instance\"}[1m])) by (req)", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 4 + "legendFormat": "{{req}}", + "refId": "D" } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Worker handled tasks", + "title": "Total KV Cursor Operations", "tooltip": { "msResolution": false, "shared": true, @@ -12922,7 +11867,8 @@ }, "yaxes": [ { - "format": "ops", + "decimals": 0, + "format": "short", "label": null, "logBase": 1, "max": null, @@ -12935,9 +11881,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -12951,30 +11901,30 @@ "fill": 1, "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, - "y": 359 + "y": 34 }, - "id": 1395, + "id": 3129, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 400, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -12985,17 +11935,25 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_worker_pending_task_total{instance=~\"$instance\"}[1m])) by (name)", + "expr": "histogram_quantile(1, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\"$instance\"}[1m])) by (le, req)) ", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 4 + "legendFormat": "100%-{{req}}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\"$instance\"}[1m])) by (le, req)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%-{{req}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Worker pending tasks", + "title": "KV Cursor Operations", "tooltip": { "msResolution": false, "shared": true, @@ -13012,6 +11970,7 @@ }, "yaxes": [ { + "decimals": 0, "format": "short", "label": null, "logBase": 1, @@ -13025,9 +11984,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -13036,57 +11999,66 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "description": "", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, - "y": 367 + "y": 41 }, - "id": 1876, + "id": 2118, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 400, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "key_skipped", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_futurepool_handled_task_total{instance=~\"$instance\"}[1m])) by (name)", + "expr": "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\" ,metric=\"internal_delete_skipped_count\"}[1m])) by (req)", "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", + "legendFormat": "delete_skipped-{{req}}", + "metric": "scan_details", + "refId": "B", "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "FuturePool handled tasks", + "title": "Total RocksDB Perf Statistics", "tooltip": { "msResolution": false, "shared": true, @@ -13103,7 +12075,8 @@ }, "yaxes": [ { - "format": "ops", + "decimals": 1, + "format": "short", "label": null, "logBase": 1, "max": null, @@ -13111,14 +12084,19 @@ "show": true }, { + "decimals": null, "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -13132,30 +12110,30 @@ "fill": 1, "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, - "y": 367 + "y": 41 }, - "id": 1877, + "id": 551, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 400, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -13166,18 +12144,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_futurepool_pending_task_total{instance=~\"$instance\"}[1m])) by (name)", + "expr": "sum(rate(tikv_coprocessor_response_bytes{instance=~\"$instance\"}[1m]))", "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "{{name}}", + "legendFormat": "size", + "metric": "", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "FuturePool pending tasks", + "title": "Total Response Size", "tooltip": { "msResolution": false, "shared": true, @@ -13194,7 +12175,8 @@ }, "yaxes": [ { - "format": "short", + "decimals": 0, + "format": "decbytes", "label": null, "logBase": 1, "max": null, @@ -13207,13 +12189,17 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "repeat": null, - "title": "Task", + "title": "Coprocessor Overview", "type": "row" }, { @@ -13222,9 +12208,9 @@ "h": 1, "w": 24, "x": 0, - "y": 21 + "y": 19 }, - "id": 2761, + "id": 3197, "panels": [ { "aliasColors": {}, @@ -13233,14 +12219,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 368 + "y": 20 }, - "id": 2108, + "id": 113, "legend": { "alignAsTable": true, "avg": false, @@ -13249,19 +12238,18 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 250, "sort": "current", - "sortDesc": false, + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, - "pointradius": 2, - "points": true, + "pointradius": 5, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -13269,29 +12257,31 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_threads_state{instance=~\"$instance\"}) by (instance, state)", + "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{state}}", - "refId": "A", - "step": 4 + "intervalFactor": 2, + "legendFormat": "{{req}}-100%", + "refId": "E" }, { - "expr": "sum(tikv_threads_state{instance=~\"$instance\"}) by (instance)", + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}}-total", - "refId": "B" + "legendFormat": "{{req}}-99%", + "refId": "A", + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Threads state", + "title": "Handle duration", "tooltip": { + "msResolution": false, "shared": true, - "sort": 0, - "value_type": "individual" + "sort": 1, + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -13303,22 +12293,28 @@ }, "yaxes": [ { - "format": "none", - "label": null, + "decimals": 1, + "format": "s", + "label": "", "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { + "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -13327,14 +12323,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 368 + "y": 20 }, - "id": 2258, + "id": 117, "legend": { "alignAsTable": true, "avg": false, @@ -13343,8 +12342,8 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 250, - "sort": "max", + "sideWidth": null, + "sort": "current", "sortDesc": true, "total": false, "values": true @@ -13354,8 +12353,8 @@ "links": [], "nullPointMode": "null", "percentage": false, - "pointradius": 2, - "points": true, + "pointradius": 5, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -13363,24 +12362,24 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_threads_io_bytes_total{instance=~\"$instance\"}[30s])) by (name, io) > 1024", + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,req))", "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{name}}-{{io}}", - "refId": "A", + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{req}}", + "refId": "B", "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Threads IO", + "title": "95% Handle duration by store", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -13392,11 +12391,12 @@ }, "yaxes": [ { - "format": "Bps", + "decimals": 1, + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -13405,9 +12405,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -13416,14 +12420,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 375 + "y": 27 }, - "id": 2660, + "id": 111, "legend": { "alignAsTable": true, "avg": false, @@ -13432,8 +12439,7 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 250, - "sort": "max", + "sort": "current", "sortDesc": true, "total": false, "values": true @@ -13441,10 +12447,10 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, - "pointradius": 2, - "points": true, + "pointradius": 5, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -13452,24 +12458,31 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_voluntary_context_switches{instance=~\"$instance\"}[30s])) by (instance, name) > 0", + "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}} - {{name}}", + "intervalFactor": 2, + "legendFormat": "{{req}}-100%", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{req}}-99%", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Thread Voluntary Context Switches", + "title": "Wait duration", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -13481,11 +12494,12 @@ }, "yaxes": [ { - "format": "none", + "decimals": 1, + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -13494,9 +12508,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -13505,14 +12523,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 375 + "y": 27 }, - "id": 2661, + "id": 116, "legend": { "alignAsTable": true, "avg": false, @@ -13521,8 +12542,7 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 250, - "sort": "max", + "sort": "current", "sortDesc": true, "total": false, "values": true @@ -13532,8 +12552,8 @@ "links": [], "nullPointMode": "null", "percentage": false, - "pointradius": 2, - "points": true, + "pointradius": 5, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -13541,24 +12561,24 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_nonvoluntary_context_switches{instance=~\"$instance\"}[30s])) by (instance, name) > 0", + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,req))", "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}} - {{name}}", - "refId": "A", + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{req}}", + "refId": "B", "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Thread Nonvoluntary Context Switches", + "title": "95% Wait duration by store", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -13570,38 +12590,29 @@ }, "yaxes": [ { - "format": "none", + "decimals": 1, + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { + "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] - } - ], - "repeat": null, - "title": "Threads", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 2762, - "panels": [ + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, @@ -13609,14 +12620,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, - "y": 376 + "y": 34 }, - "id": 138, + "id": 3195, "legend": { "alignAsTable": true, "avg": false, @@ -13625,73 +12639,43 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_memtable_efficiency{instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "memtable", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[1m]))", + "expr": "sum(rate(tikv_coprocessor_dag_request_count{instance=~\"$instance\"}[1m])) by (vec_type)", + "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "block_cache", + "legendFormat": "{{vec_type}}", "metric": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "l0", "refId": "A", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "l1", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "l2_and_up", - "refId": "F", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Get operations", + "title": "Total DAG Requests", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -13706,11 +12690,12 @@ }, "yaxes": [ { - "format": "ops", + "decimals": 1, + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -13719,9 +12704,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -13730,14 +12719,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, - "y": 376 + "y": 34 }, - "id": 82, + "id": 3264, "legend": { "alignAsTable": true, "avg": false, @@ -13746,69 +12738,43 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", + "expr": "sum(rate(tikv_coprocessor_executor_count{instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "max", + "legendFormat": "{{type}}", + "metric": "", "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Get duration", + "title": "Total DAG Executors", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -13823,11 +12789,12 @@ }, "yaxes": [ { - "format": "µs", + "decimals": 1, + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -13836,9 +12803,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -13847,14 +12818,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 9, "w": 12, "x": 0, - "y": 384 + "y": 41 }, - "id": 129, + "id": 552, "legend": { "alignAsTable": true, "avg": false, @@ -13863,84 +12837,43 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "seek", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[1m]))", + "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"select\"}[1m])) by (tag)", + "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "seek_found", - "metric": "", + "legendFormat": "{{tag}}", + "metric": "scan_details", "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "next", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "next_found", - "metric": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "prev", - "metric": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "prev_found", - "metric": "", - "refId": "F", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Seek operations", + "title": "Total Ops Details (Table Scan)", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -13955,11 +12888,12 @@ }, "yaxes": [ { - "format": "ops", + "decimals": 1, + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -13968,9 +12902,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -13979,14 +12917,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 9, "w": 12, "x": 12, - "y": 384 + "y": 41 }, - "id": 125, + "id": 3263, "legend": { "alignAsTable": true, "avg": false, @@ -13995,69 +12936,43 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": null, - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", + "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"index\"}[1m])) by (tag)", "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "99%", + "legendFormat": "{{tag}}", + "metric": "scan_details", "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Seek duration", + "title": "Total Ops Details (Index Scan)", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -14072,11 +12987,12 @@ }, "yaxes": [ { - "format": "µs", + "decimals": 1, + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -14085,9 +13001,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -14096,14 +13016,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 9, "w": 12, "x": 0, - "y": 392 + "y": 50 }, - "id": 139, + "id": 122, "legend": { "alignAsTable": true, "avg": false, @@ -14112,7 +13035,9 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -14124,45 +13049,30 @@ "pointradius": 5, "points": false, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, + "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "done", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[1m]))", + "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"select\"}[1m])) by (tag,cf)", + "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "timeout", + "legendFormat": "{{cf}}-{{tag}}", + "metric": "scan_details", "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "with_wal", - "refId": "C", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Write operations", + "title": "Total Ops Details by CF (Table Scan)", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -14177,11 +13087,12 @@ }, "yaxes": [ { - "format": "ops", + "decimals": 1, + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -14190,9 +13101,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -14201,14 +13116,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 9, "w": 12, "x": 12, - "y": 392 + "y": 50 }, - "id": 126, + "id": 554, "legend": { "alignAsTable": true, "avg": false, @@ -14217,69 +13135,45 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, + "repeat": "cf", + "repeatDirection": "h", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, + "targets": [ { - "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_average\"})", + "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"index\"}[1m])) by (tag,cf)", "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 + "legendFormat": "{{cf}}-{{tag}}", + "metric": "scan_details", + "refId": "B", + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Write duration", + "title": "Total Ops Details by CF (Index Scan)", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -14294,11 +13188,12 @@ }, "yaxes": [ { - "format": "µs", + "decimals": 1, + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -14307,10 +13202,28 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] - }, + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Coprocessor Detail", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 2761, + "panels": [ { "aliasColors": {}, "bars": false, @@ -14320,12 +13233,12 @@ "decimals": 1, "fill": 1, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, - "y": 400 + "y": 368 }, - "id": 137, + "id": 2108, "legend": { "alignAsTable": true, "avg": false, @@ -14334,7 +13247,9 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": 250, + "sort": "current", + "sortDesc": false, "total": false, "values": true }, @@ -14343,34 +13258,34 @@ "links": [], "nullPointMode": "null", "percentage": false, - "pointradius": 5, - "points": false, + "pointradius": 2, + "points": true, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_wal_file_synced{instance=~\"$instance\", db=\"$db\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "sync", - "metric": "", + "expr": "sum(tikv_threads_state{instance=~\"$instance\"}) by (instance, state)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{state}}", "refId": "A", - "step": 10 + "step": 4 + }, + { + "expr": "sum(tikv_threads_state{instance=~\"$instance\"}) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}-total", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "WAL sync operations", + "title": "Threads state", "tooltip": { "shared": true, "sort": 0, @@ -14386,7 +13301,7 @@ }, "yaxes": [ { - "format": "ops", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -14410,14 +13325,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "fill": 1, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, - "y": 400 + "y": 368 }, - "id": 135, + "id": 2258, "legend": { "alignAsTable": true, "avg": false, @@ -14426,64 +13341,40 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": 12, "nullPointMode": "null", "percentage": false, - "pointradius": 5, - "points": false, + "pointradius": 2, + "points": true, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", - "intervalFactor": 2, - "legendFormat": "max", + "expr": "sum(rate(tikv_threads_io_bytes_total{instance=~\"$instance\"}[30s])) by (name, io) > 1024", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}}-{{io}}", "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "WAL sync duration", + "title": "Threads IO", "tooltip": { "shared": true, "sort": 0, @@ -14499,11 +13390,11 @@ }, "yaxes": [ { - "format": "µs", + "format": "Bps", "label": null, - "logBase": 10, + "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -14525,12 +13416,12 @@ "decimals": 1, "fill": 1, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, - "y": 408 + "y": 375 }, - "id": 128, + "id": 2660, "legend": { "alignAsTable": true, "avg": false, @@ -14539,7 +13430,9 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, @@ -14548,34 +13441,29 @@ "links": [], "nullPointMode": "null", "percentage": false, - "pointradius": 5, - "points": false, + "pointradius": 2, + "points": true, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_event_total{instance=~\"$instance\", db=\"$db\"}[1m])) by (type)", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_engine_event_total", - "refId": "B", - "step": 10 + "expr": "sum(rate(tikv_thread_voluntary_context_switches{instance=~\"$instance\"}[30s])) by (instance, name) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} - {{name}}", + "refId": "A", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compaction operations", + "title": "Thread Voluntary Context Switches", "tooltip": { "shared": true, "sort": 0, @@ -14591,7 +13479,7 @@ }, "yaxes": [ { - "format": "ops", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -14615,14 +13503,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "fill": 1, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, - "y": 408 + "y": 375 }, - "id": 136, + "id": 2661, "legend": { "alignAsTable": true, "avg": false, @@ -14631,69 +13519,40 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, - "pointradius": 5, - "points": false, + "pointradius": 2, + "points": true, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", + "expr": "sum(rate(tikv_thread_nonvoluntary_context_switches{instance=~\"$instance\"}[30s])) by (instance, name) > 0", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "metric": "", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} - {{name}}", "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compaction duration", + "title": "Thread Nonvoluntary Context Switches", "tooltip": { "shared": true, "sort": 0, @@ -14709,9 +13568,9 @@ }, "yaxes": [ { - "format": "µs", + "format": "none", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -14725,7 +13584,22 @@ "show": true } ] - }, + } + ], + "repeat": null, + "title": "Threads", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 2762, + "panels": [ { "aliasColors": {}, "bars": false, @@ -14738,9 +13612,9 @@ "h": 8, "w": 12, "x": 0, - "y": 416 + "y": 376 }, - "id": 140, + "id": 138, "legend": { "alignAsTable": true, "avg": false, @@ -14774,42 +13648,47 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", + "expr": "sum(rate(tikv_engine_memtable_efficiency{instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m]))", "intervalFactor": 2, - "legendFormat": "max", + "legendFormat": "memtable", "metric": "", - "refId": "A", + "refId": "B", "step": 10 }, { - "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[1m]))", "intervalFactor": 2, - "legendFormat": "99%", + "legendFormat": "block_cache", "metric": "", - "refId": "B", + "refId": "E", "step": 10 }, { - "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", + "expr": "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[1m]))", "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", + "legendFormat": "l0", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "l1", "refId": "C", "step": 10 }, { - "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})", + "expr": "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[1m]))", "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "D", + "legendFormat": "l2_and_up", + "refId": "F", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "SST read duration", + "title": "Get operations", "tooltip": { "shared": true, "sort": 0, @@ -14825,9 +13704,9 @@ }, "yaxes": [ { - "format": "ms", + "format": "ops", "label": null, - "logBase": 10, + "logBase": 1, "max": null, "min": null, "show": true @@ -14849,14 +13728,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 416 + "y": 376 }, - "id": 87, + "id": 82, "legend": { "alignAsTable": true, "avg": false, @@ -14890,34 +13769,34 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", + "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", + "format": "time_series", "intervalFactor": 2, "legendFormat": "max", - "metric": "", "refId": "A", "step": 10 }, { - "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", + "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", + "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", - "metric": "", "refId": "B", "step": 10 }, { - "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", + "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", + "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", - "metric": "", "refId": "C", "step": 10 }, { - "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})", + "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_average\"})", + "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", - "metric": "", "refId": "D", "step": 10 } @@ -14925,7 +13804,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Write stall duration", + "title": "Get duration", "tooltip": { "shared": true, "sort": 0, @@ -14941,9 +13820,9 @@ }, "yaxes": [ { - "format": "ms", + "format": "µs", "label": null, - "logBase": 10, + "logBase": 2, "max": null, "min": null, "show": true @@ -14970,9 +13849,9 @@ "h": 8, "w": 12, "x": 0, - "y": 424 + "y": 384 }, - "id": 103, + "id": 129, "legend": { "alignAsTable": true, "avg": false, @@ -15006,17 +13885,58 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_memory_bytes{instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)", + "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[1m]))", "intervalFactor": 2, - "legendFormat": "{{cf}}", + "legendFormat": "seek", + "metric": "", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "seek_found", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "next", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "next_found", + "metric": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "prev", + "metric": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "prev_found", + "metric": "", + "refId": "F", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Memtable size", + "title": "Seek operations", "tooltip": { "shared": true, "sort": 0, @@ -15032,7 +13952,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -15061,9 +13981,9 @@ "h": 8, "w": 12, "x": 12, - "y": 424 + "y": 384 }, - "id": 88, + "id": 125, "legend": { "alignAsTable": true, "avg": false, @@ -15079,8 +13999,7 @@ "lines": true, "linewidth": 1, "links": [], - "minSpan": null, - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -15098,23 +14017,47 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_memtable_efficiency{instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m])) / (sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_hit\"}[1m])) + sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_miss\"}[1m])))", + "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "hit", + "legendFormat": "max", "refId": "A", "step": 10 + }, + { + "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "B", + "step": 10 + }, + { + "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Memtable hit", + "title": "Seek duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, - "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -15125,20 +14068,20 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 2, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -15154,9 +14097,9 @@ "h": 8, "w": 12, "x": 0, - "y": 432 + "y": 392 }, - "id": 102, + "id": 139, "legend": { "alignAsTable": true, "avg": false, @@ -15190,17 +14133,31 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_block_cache_size_bytes{instance=~\"$instance\", db=\"$db\"}) by(cf)", + "expr": "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[1m]))", "intervalFactor": 2, - "legendFormat": "{{cf}}", + "legendFormat": "done", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "timeout", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "with_wal", + "refId": "C", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Block cache size", + "title": "Write operations", "tooltip": { "shared": true, "sort": 0, @@ -15216,7 +14173,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -15245,9 +14202,9 @@ "h": 8, "w": 12, "x": 12, - "y": 432 + "y": 392 }, - "id": 80, + "id": 126, "legend": { "alignAsTable": true, "avg": false, @@ -15263,8 +14220,7 @@ "lines": true, "linewidth": 1, "links": [], - "minSpan": 12, - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -15282,56 +14238,47 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_miss\"}[1m])))", + "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "all", - "metric": "", + "legendFormat": "max", "refId": "A", "step": 10 }, { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_miss\"}[1m])))", - "intervalFactor": 2, - "legendFormat": "data", - "metric": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_miss\"}[1m])))", + "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "filter", - "metric": "", + "legendFormat": "99%", "refId": "B", "step": 10 }, { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_miss\"}[1m])))", + "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "index", - "metric": "", + "legendFormat": "95%", "refId": "C", "step": 10 }, { - "expr": "sum(rate(tikv_engine_bloom_efficiency{instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[1m])) / sum(rate(tikv_engine_bloom_efficiency{db=\"$db\", type=\"bloom_prefix_checked\"}[1m]))", + "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_average\"})", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "bloom prefix", - "metric": "", - "refId": "E", + "legendFormat": "avg", + "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Block cache hit", + "title": "Write duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, - "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -15342,20 +14289,20 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 2, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -15366,15 +14313,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 440 + "y": 400 }, - "height": "", - "id": 467, + "id": 137, "legend": { "alignAsTable": true, "avg": false, @@ -15408,78 +14354,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "total_read", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "total_written", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "data_insert", - "metric": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "filter_insert", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "filter_evict", - "metric": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "index_insert", - "metric": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[1m]))", - "hide": false, - "interval": "", + "expr": "sum(rate(tikv_engine_wal_file_synced{instance=~\"$instance\", db=\"$db\"}[1m]))", "intervalFactor": 2, - "legendFormat": "index_evict", + "legendFormat": "sync", "metric": "", - "refId": "G", + "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Block cache flow", + "title": "WAL sync operations", "tooltip": { "shared": true, "sort": 0, @@ -15495,19 +14381,19 @@ }, "yaxes": [ { - "format": "Bps", + "format": "ops", "label": null, - "logBase": 10, + "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -15519,14 +14405,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 440 + "y": 400 }, - "id": 468, + "id": 135, "legend": { "alignAsTable": true, "avg": false, @@ -15542,6 +14428,7 @@ "lines": true, "linewidth": 1, "links": [], + "maxPerRow": 2, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -15560,50 +14447,38 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[1m]))", + "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", "intervalFactor": 2, - "legendFormat": "total_add", - "metric": "", + "legendFormat": "max", "refId": "A", "step": 10 }, { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "data_add", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[1m]))", + "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", "intervalFactor": 2, - "legendFormat": "filter_add", - "metric": "", - "refId": "D", + "legendFormat": "99%", + "refId": "B", "step": 10 }, { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[1m]))", + "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", "intervalFactor": 2, - "legendFormat": "index_add", - "metric": "", - "refId": "E", + "legendFormat": "95%", + "refId": "C", "step": 10 }, { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[1m]))", + "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})", "intervalFactor": 2, - "legendFormat": "add_failures", - "metric": "", - "refId": "B", + "legendFormat": "avg", + "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Block cache operations", + "title": "WAL sync duration", "tooltip": { "shared": true, "sort": 0, @@ -15619,11 +14494,11 @@ }, "yaxes": [ { - "format": "ops", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -15643,15 +14518,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 448 + "y": 408 }, - "height": "", - "id": 132, + "id": 128, "legend": { "alignAsTable": true, "avg": false, @@ -15685,38 +14559,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[1m]))", - "hide": false, - "interval": "", + "expr": "sum(rate(tikv_engine_event_total{instance=~\"$instance\", db=\"$db\"}[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "read", + "legendFormat": "{{type}}", + "metric": "tikv_engine_event_total", "refId": "B", "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "written", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_compaction_num_corrupt_keys{instance=~\"$instance\", db=\"$db\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "corrupt", - "metric": "", - "refId": "A", - "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Keys flow", + "title": "Compaction operations", "tooltip": { "shared": true, "sort": 0, @@ -15736,7 +14590,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -15744,7 +14598,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -15756,14 +14610,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 448 + "y": 408 }, - "id": 131, + "id": 136, "legend": { "alignAsTable": true, "avg": false, @@ -15797,19 +14651,43 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_engine_estimate_num_keys{instance=~\"$instance\", db=\"$db\"}) by (cf)", - "hide": false, + "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{cf}}", - "metric": "tikv_engine_estimate_num_keys", + "legendFormat": "max", + "metric": "", "refId": "A", "step": 10 + }, + { + "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "B", + "step": 10 + }, + { + "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Total keys", + "title": "Compaction duration", "tooltip": { "shared": true, "sort": 0, @@ -15825,11 +14703,11 @@ }, "yaxes": [ { - "format": "short", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 2, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -15837,7 +14715,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -15849,15 +14727,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 456 + "y": 416 }, - "height": "", - "id": 85, + "id": 140, "legend": { "alignAsTable": true, "avg": false, @@ -15891,28 +14768,42 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", - "hide": false, - "interval": "", + "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", "intervalFactor": 2, - "legendFormat": "get", + "legendFormat": "max", + "metric": "", "refId": "A", "step": 10 }, { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[1m]))", - "hide": false, - "interval": "", + "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", + "intervalFactor": 2, + "legendFormat": "99%", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", + "intervalFactor": 2, + "legendFormat": "95%", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})", "intervalFactor": 2, - "legendFormat": "scan", - "refId": "C", + "legendFormat": "avg", + "metric": "", + "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Read flow", + "title": "SST read duration", "tooltip": { "shared": true, "sort": 0, @@ -15928,11 +14819,11 @@ }, "yaxes": [ { - "format": "Bps", + "format": "ms", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -15940,7 +14831,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -15952,14 +14843,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 456 + "y": 416 }, - "id": 133, + "id": 87, "legend": { "alignAsTable": true, "avg": false, @@ -15975,7 +14866,6 @@ "lines": true, "linewidth": 1, "links": [], - "minSpan": 12, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -15994,30 +14884,34 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", + "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", "intervalFactor": 2, "legendFormat": "max", + "metric": "", "refId": "A", "step": 10 }, { - "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", + "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", "intervalFactor": 2, "legendFormat": "99%", + "metric": "", "refId": "B", "step": 10 }, { - "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", + "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", "intervalFactor": 2, "legendFormat": "95%", + "metric": "", "refId": "C", "step": 10 }, { - "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})", + "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})", "intervalFactor": 2, "legendFormat": "avg", + "metric": "", "refId": "D", "step": 10 } @@ -16025,7 +14919,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Bytes / Read", + "title": "Write stall duration", "tooltip": { "shared": true, "sort": 0, @@ -16041,11 +14935,11 @@ }, "yaxes": [ { - "format": "decbytes", + "format": "ms", "label": null, "logBase": 10, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -16070,10 +14964,9 @@ "h": 8, "w": 12, "x": 0, - "y": 464 + "y": 424 }, - "height": "", - "id": 86, + "id": 103, "legend": { "alignAsTable": true, "avg": false, @@ -16107,18 +15000,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[1m]))", - "hide": false, - "intervalFactor": 2, - "legendFormat": "wal", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", - "hide": false, + "expr": "avg(tikv_engine_memory_bytes{instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)", "intervalFactor": 2, - "legendFormat": "write", + "legendFormat": "{{cf}}", "refId": "A", "step": 10 } @@ -16126,7 +15010,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Write flow", + "title": "Memtable size", "tooltip": { "shared": true, "sort": 0, @@ -16142,11 +15026,11 @@ }, "yaxes": [ { - "format": "Bps", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -16154,7 +15038,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -16171,9 +15055,9 @@ "h": 8, "w": 12, "x": 12, - "y": 464 + "y": 424 }, - "id": 134, + "id": 88, "legend": { "alignAsTable": true, "avg": false, @@ -16189,8 +15073,7 @@ "lines": true, "linewidth": 1, "links": [], - "minSpan": 12, - "nullPointMode": "null", + "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, @@ -16208,43 +15091,23 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", + "expr": "sum(rate(tikv_engine_memtable_efficiency{instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m])) / (sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_hit\"}[1m])) + sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_miss\"}[1m])))", "intervalFactor": 2, - "legendFormat": "max", + "legendFormat": "hit", "refId": "A", "step": 10 - }, - { - "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Bytes / Write", + "title": "Memtable hit", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, + "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -16255,20 +15118,20 @@ }, "yaxes": [ { - "format": "decbytes", + "format": "percentunit", "label": null, - "logBase": 10, + "logBase": 1, "max": null, "min": "0", "show": true }, { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ] }, @@ -16284,9 +15147,9 @@ "h": 8, "w": 12, "x": 0, - "y": 472 + "y": 432 }, - "id": 90, + "id": 102, "legend": { "alignAsTable": true, "avg": false, @@ -16320,34 +15183,17 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_compaction_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", - "hide": false, + "expr": "avg(tikv_engine_block_cache_size_bytes{instance=~\"$instance\", db=\"$db\"}) by(cf)", "intervalFactor": 2, - "legendFormat": "read", + "legendFormat": "{{cf}}", "refId": "A", "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_compaction_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", - "hide": false, - "intervalFactor": 2, - "legendFormat": "written", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[1m]))", - "hide": false, - "intervalFactor": 2, - "legendFormat": "flushed", - "refId": "B", - "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compaction flow", + "title": "Block cache size", "tooltip": { "shared": true, "sort": 0, @@ -16363,19 +15209,19 @@ }, "yaxes": [ { - "format": "Bps", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -16387,14 +15233,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 472 + "y": 432 }, - "id": 127, + "id": 80, "legend": { "alignAsTable": true, "avg": false, @@ -16410,7 +15256,8 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "maxPerRow": 2, + "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, @@ -16428,24 +15275,56 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_pending_compaction_bytes{instance=~\"$instance\", db=\"$db\"}[1m])) by (cf)", - "hide": false, + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_miss\"}[1m])))", "intervalFactor": 2, - "legendFormat": "{{cf}}", - "metric": "tikv_engine_pending_compaction_bytes", + "legendFormat": "all", + "metric": "", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_miss\"}[1m])))", + "intervalFactor": 2, + "legendFormat": "data", + "metric": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_miss\"}[1m])))", + "intervalFactor": 2, + "legendFormat": "filter", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_miss\"}[1m])))", + "intervalFactor": 2, + "legendFormat": "index", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_bloom_efficiency{instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[1m])) / sum(rate(tikv_engine_bloom_efficiency{db=\"$db\", type=\"bloom_prefix_checked\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "bloom prefix", + "metric": "", + "refId": "E", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compaction pending bytes", + "title": "Block cache hit", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, + "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -16456,7 +15335,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -16464,12 +15343,12 @@ "show": true }, { - "format": "Bps", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": "0", - "show": true + "min": null, + "show": false } ] }, @@ -16480,14 +15359,15 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 480 + "y": 440 }, - "id": 518, + "height": "", + "id": 467, "legend": { "alignAsTable": true, "avg": false, @@ -16521,19 +15401,78 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_read_amp_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[1m])) by (instance) / sum(rate(tikv_engine_read_amp_flow_bytes{db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[1m])) by (instance)", + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "total_read", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "total_written", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "data_insert", + "metric": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "filter_insert", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "filter_evict", + "metric": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[1m]))", "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}", + "legendFormat": "index_insert", "metric": "", - "refId": "A", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "index_evict", + "metric": "", + "refId": "G", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Read amplication", + "title": "Block cache flow", "tooltip": { "shared": true, "sort": 0, @@ -16549,15 +15488,15 @@ }, "yaxes": [ { - "format": "short", + "format": "Bps", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": "0", "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -16578,9 +15517,9 @@ "h": 8, "w": 12, "x": 12, - "y": 480 + "y": 440 }, - "id": 863, + "id": 468, "legend": { "alignAsTable": true, "avg": false, @@ -16614,19 +15553,50 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_compression_ratio{instance=~\"$instance\", db=\"$db\"}) by (level)", - "hide": false, + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[1m]))", "intervalFactor": 2, - "legendFormat": "level - {{level}}", + "legendFormat": "total_add", "metric": "", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "data_add", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "filter_add", + "metric": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "index_add", + "metric": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "add_failures", + "metric": "", + "refId": "B", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compression ratio", + "title": "Block cache operations", "tooltip": { "shared": true, "sort": 0, @@ -16642,11 +15612,11 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -16654,7 +15624,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -16666,14 +15636,15 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 488 + "y": 448 }, - "id": 516, + "height": "", + "id": 132, "legend": { "alignAsTable": true, "avg": false, @@ -16707,10 +15678,29 @@ "steppedLine": false, "targets": [ { - "expr": "tikv_engine_num_snapshots{instance=~\"$instance\", db=\"$db\"}", + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[1m]))", "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}", + "legendFormat": "read", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "written", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_compaction_num_corrupt_keys{instance=~\"$instance\", db=\"$db\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "corrupt", "metric": "", "refId": "A", "step": 10 @@ -16719,7 +15709,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Number of snapshots", + "title": "Keys flow", "tooltip": { "shared": true, "sort": 0, @@ -16735,7 +15725,7 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -16764,9 +15754,9 @@ "h": 8, "w": 12, "x": 12, - "y": 488 + "y": 448 }, - "id": 517, + "id": 131, "legend": { "alignAsTable": true, "avg": false, @@ -16800,11 +15790,11 @@ "steppedLine": false, "targets": [ { - "expr": "tikv_engine_oldest_snapshot_duration{instance=~\"$instance\", db=\"$db\"}", + "expr": "sum(tikv_engine_estimate_num_keys{instance=~\"$instance\", db=\"$db\"}) by (cf)", "hide": false, "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_engine_oldest_snapshot_duration", + "legendFormat": "{{cf}}", + "metric": "tikv_engine_estimate_num_keys", "refId": "A", "step": 10 } @@ -16812,7 +15802,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Oldest snapshots duration", + "title": "Total keys", "tooltip": { "shared": true, "sort": 0, @@ -16828,7 +15818,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -16851,22 +15841,25 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "fill": 1, + "decimals": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 496 + "y": 456 }, - "id": 2002, + "height": "", + "id": 85, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, - "min": true, + "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -16891,17 +15884,28 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_num_files_at_level{instance=~\"$instance\", db=\"$db\"}) by (cf, level)", - "format": "time_series", + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", + "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "cf-{{cf}}, level-{{level}}", - "refId": "A" + "legendFormat": "get", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "scan", + "refId": "C", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Number files at each level", + "title": "Read flow", "tooltip": { "shared": true, "sort": 0, @@ -16917,11 +15921,11 @@ }, "yaxes": [ { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -16929,7 +15933,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ] @@ -16940,28 +15944,31 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "fill": 1, + "decimals": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 496 + "y": 456 }, - "id": 2003, + "id": 133, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, - "min": true, + "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], + "maxPerRow": 2, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -16980,24 +15987,38 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_snapshot_ingest_sst_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", - "format": "time_series", + "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", + "intervalFactor": 2, + "legendFormat": "max", + "refId": "A", + "step": 10 + }, + { + "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", "intervalFactor": 2, "legendFormat": "99%", - "refId": "A" + "refId": "B", + "step": 10 }, { - "expr": "sum(rate(tikv_snapshot_ingest_sst_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_snapshot_ingest_sst_duration_seconds_count{instance=~\"$instance\"}[1m]))", - "format": "time_series", + "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", "intervalFactor": 2, - "legendFormat": "average", - "refId": "B" + "legendFormat": "95%", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Ingest SST duration seconds", + "title": "Bytes / Read", "tooltip": { "shared": true, "sort": 0, @@ -17013,11 +16034,11 @@ }, "yaxes": [ { - "format": "s", + "format": "decbytes", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -17036,23 +16057,25 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 504 + "y": 464 }, - "id": 2381, + "height": "", + "id": 86, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideZero": true, "max": true, - "min": true, + "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17077,17 +16100,26 @@ "steppedLine": false, "targets": [ { - "expr": "tikv_engine_stall_conditions_changed{instance=~\"$instance\", db=\"$db\"}", - "format": "time_series", + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[1m]))", + "hide": false, "intervalFactor": 2, - "legendFormat": "{{instance}}-{{cf}}-{{type}}", - "refId": "B" + "legendFormat": "wal", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "write", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Stall conditions changed of each CF", + "title": "Write flow", "tooltip": { "shared": true, "sort": 0, @@ -17103,11 +16135,11 @@ }, "yaxes": [ { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -17115,7 +16147,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ] @@ -17127,14 +16159,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 504 + "y": 464 }, - "id": 2451, + "id": 134, "legend": { "alignAsTable": true, "avg": false, @@ -17150,6 +16182,7 @@ "lines": true, "linewidth": 1, "links": [], + "maxPerRow": 2, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -17167,21 +16200,39 @@ "stack": false, "steppedLine": false, "targets": [ - { - "expr": "sum(rate(tikv_engine_compaction_reason{instance=~\"$instance\", db=\"$db\"}[1m])) by (cf, reason)", - "format": "time_series", - "hide": false, + { + "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", "intervalFactor": 2, - "legendFormat": "{{cf}} - {{reason}}", - "metric": "", + "legendFormat": "max", "refId": "A", "step": 10 + }, + { + "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "B", + "step": 10 + }, + { + "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compaction reason", + "title": "Bytes / Write", "tooltip": { "shared": true, "sort": 0, @@ -17197,9 +16248,9 @@ }, "yaxes": [ { - "format": "short", + "format": "decbytes", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": "0", "show": true @@ -17209,48 +16260,35 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] - } - ], - "repeat": "db", - "title": "RocksDB - $db", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 3301, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, + "h": 8, + "w": 12, "x": 0, - "y": 26 + "y": 472 }, - "id": 3414, + "id": 90, "legend": { "alignAsTable": true, - "avg": true, - "current": false, - "max": false, + "avg": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17258,44 +16296,51 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", + "expr": "sum(rate(tikv_engine_compaction_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", "hide": false, "intervalFactor": 2, - "legendFormat": "key_size_avg", - "refId": "A" + "legendFormat": "read", + "refId": "A", + "step": 10 }, { - "expr": "quantile(0.95, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "key_size_0.95", - "refId": "C" + "expr": "sum(rate(tikv_engine_compaction_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "written", + "refId": "C", + "step": 10 }, { - "expr": "quantile(0.99, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[1m]))", "hide": false, - "intervalFactor": 1, - "legendFormat": "key_size_0.99", - "refId": "D" + "intervalFactor": 2, + "legendFormat": "flushed", + "refId": "B", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob key size", + "title": "Compaction flow", "tooltip": { "shared": true, "sort": 0, @@ -17311,26 +16356,22 @@ }, "yaxes": [ { - "format": "bytes", + "format": "Bps", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -17338,22 +16379,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, - "x": 10, - "y": 26 + "h": 8, + "w": 12, + "x": 12, + "y": 472 }, - "id": 3446, + "id": 127, "legend": { "alignAsTable": true, - "avg": true, - "current": false, - "max": false, + "avg": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17361,42 +16404,36 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "value_size", - "refId": "A" - }, - { - "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "value_size_0.95", - "refId": "B" - }, - { - "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "value_size_0.99", - "refId": "C" + "expr": "sum(rate(tikv_engine_pending_compaction_bytes{instance=~\"$instance\", db=\"$db\"}[1m])) by (cf)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{cf}}", + "metric": "tikv_engine_pending_compaction_bytes", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob value size", + "title": "Compaction pending bytes", "tooltip": { "shared": true, "sort": 0, @@ -17414,24 +16451,20 @@ { "format": "bytes", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -17439,22 +16472,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, + "h": 8, + "w": 12, "x": 0, - "y": 32 + "y": 480 }, - "id": 3412, + "id": 518, "legend": { "alignAsTable": true, - "avg": true, - "current": false, + "avg": false, + "current": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17462,42 +16497,36 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", + "expr": "sum(rate(tikv_engine_read_amp_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[1m])) by (instance) / sum(rate(tikv_engine_read_amp_flow_bytes{db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[1m])) by (instance)", + "hide": false, "intervalFactor": 2, - "legendFormat": "seek_micros", - "refId": "A" - }, - { - "expr": "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "prev_micros", - "refId": "B" - }, - { - "expr": "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "next_micros", - "refId": "C" + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob seek duration", + "title": "Read amplication", "tooltip": { "shared": true, "sort": 0, @@ -17513,11 +16542,11 @@ }, "yaxes": [ { - "format": "µs", + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -17525,14 +16554,10 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -17540,22 +16565,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, - "x": 10, - "y": 32 + "h": 8, + "w": 12, + "x": 12, + "y": 480 }, - "id": 3410, + "id": 863, "legend": { "alignAsTable": true, - "avg": true, - "current": false, - "max": false, + "avg": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17563,42 +16590,36 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", + "expr": "avg(tikv_engine_compression_ratio{instance=~\"$instance\", db=\"$db\"}) by (level)", + "hide": false, "intervalFactor": 2, - "legendFormat": "blob_gc_micros", - "refId": "A" - }, - { - "expr": "quantile(0.95, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "blob_gc_micros_0.95", - "refId": "B" - }, - { - "expr": "quantile(0.99, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "blob_gc_micros_0.99", - "refId": "C" + "legendFormat": "level - {{level}}", + "metric": "", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "GC duration", + "title": "Compression ratio", "tooltip": { "shared": true, "sort": 0, @@ -17614,11 +16635,11 @@ }, "yaxes": [ { - "format": "µs", + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -17626,14 +16647,10 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -17641,22 +16658,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, + "h": 8, + "w": 12, "x": 0, - "y": 38 + "y": 488 }, - "id": 3342, + "id": 516, "legend": { "alignAsTable": true, - "avg": true, - "current": false, + "avg": false, + "current": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17664,50 +16683,36 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "file_bytes_written", - "refId": "A" - }, - { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "file_bytes_read", - "refId": "B" - }, - { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "keys_read", - "refId": "C" - }, - { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", - "format": "time_series", + "expr": "tikv_engine_num_snapshots{instance=~\"$instance\", db=\"$db\"}", + "hide": false, "intervalFactor": 2, - "legendFormat": "keys_written", - "refId": "D" + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob flow", + "title": "Number of snapshots", "tooltip": { "shared": true, "sort": 0, @@ -17723,11 +16728,11 @@ }, "yaxes": [ { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -17735,14 +16740,10 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -17750,22 +16751,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, - "x": 10, - "y": 38 + "h": 8, + "w": 12, + "x": 12, + "y": 488 }, - "id": 3408, + "id": 517, "legend": { "alignAsTable": true, - "avg": true, - "current": false, + "avg": false, + "current": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17773,49 +16776,36 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_average\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "avg", - "refId": "A" - }, - { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile99\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "B" - }, - { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile95\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "C" - }, - { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_max\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "D" + "expr": "tikv_engine_oldest_snapshot_duration{instance=~\"$instance\", db=\"$db\"}", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "tikv_engine_oldest_snapshot_duration", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "GC file write duration", + "title": "Oldest snapshots duration", "tooltip": { "shared": true, "sort": 0, @@ -17831,11 +16821,11 @@ }, "yaxes": [ { - "format": "µs", + "format": "s", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -17843,14 +16833,10 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -17860,57 +16846,55 @@ "datasource": "${DS_TEST-CLUSTER}", "fill": 1, "gridPos": { - "h": 6, - "w": 10, + "h": 8, + "w": 12, "x": 0, - "y": 44 + "y": 496 }, - "id": 3344, + "id": 2002, "legend": { "alignAsTable": true, - "avg": true, - "current": false, - "max": false, - "min": false, + "avg": false, + "current": true, + "max": true, + "min": true, "rightSide": true, "show": true, - "total": true, + "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_input_files_count\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "gc_num_files", - "refId": "B" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_output_files_count\"}[2m]))", + "expr": "avg(tikv_engine_num_files_at_level{instance=~\"$instance\", db=\"$db\"}) by (cf, level)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "gc_num_new_files", - "refId": "C" + "legendFormat": "cf-{{cf}}, level-{{level}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob GC file", + "title": "Number files at each level", "tooltip": { "shared": true, "sort": 0, @@ -17941,11 +16925,7 @@ "min": null, "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -17955,18 +16935,18 @@ "datasource": "${DS_TEST-CLUSTER}", "fill": 1, "gridPos": { - "h": 6, - "w": 10, - "x": 10, - "y": 44 + "h": 8, + "w": 12, + "x": 12, + "y": 496 }, - "id": 3338, + "id": 2003, "legend": { "alignAsTable": true, - "avg": true, - "current": false, + "avg": false, + "current": true, "max": true, - "min": false, + "min": true, "rightSide": true, "show": true, "total": false, @@ -17976,47 +16956,41 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_seek\"}[2m]))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_snapshot_ingest_sst_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", - "hide": false, - "interval": "", "intervalFactor": 2, - "legendFormat": "seek", + "legendFormat": "99%", "refId": "A" }, { - "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_prev\"}[2m]))", + "expr": "sum(rate(tikv_snapshot_ingest_sst_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_snapshot_ingest_sst_duration_seconds_count{instance=~\"$instance\"}[1m]))", "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "prev", + "intervalFactor": 2, + "legendFormat": "average", "refId": "B" - }, - { - "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_next\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "next", - "refId": "C" } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob seek operations", + "title": "Ingest SST duration seconds", "tooltip": { "shared": true, "sort": 0, @@ -18032,7 +17006,7 @@ }, "yaxes": [ { - "format": "ops", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -18047,11 +17021,7 @@ "min": null, "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -18061,18 +17031,19 @@ "datasource": "${DS_TEST-CLUSTER}", "fill": 1, "gridPos": { - "h": 6, - "w": 10, + "h": 8, + "w": 12, "x": 0, - "y": 50 + "y": 504 }, - "id": 3340, + "id": 2381, "legend": { "alignAsTable": true, - "avg": true, - "current": false, + "avg": false, + "current": true, + "hideZero": true, "max": true, - "min": false, + "min": true, "rightSide": true, "show": true, "total": false, @@ -18082,64 +17053,34 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "bytes_written", - "refId": "A" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", + "expr": "tikv_engine_stall_conditions_changed{instance=~\"$instance\", db=\"$db\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "bytes_read", + "legendFormat": "{{instance}}-{{cf}}-{{type}}", "refId": "B" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_overwritten\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "bytes_overwritten", - "refId": "C" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_relocated\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "bytes_relocated", - "refId": "D" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_relocated\"}[2m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "keys_relocated", - "refId": "E" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_overwritten\"}[2m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "keys_overwritten", - "refId": "F" } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob GC flows", + "title": "Stall conditions changed of each CF", "tooltip": { "shared": true, "sort": 0, @@ -18155,7 +17096,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -18163,18 +17104,14 @@ "show": true }, { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -18182,22 +17119,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, - "x": 10, - "y": 51 + "h": 8, + "w": 12, + "x": 12, + "y": 504 }, - "id": 3523, + "id": 2451, "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -18205,9 +17144,8 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "scopedVars": { @@ -18218,22 +17156,25 @@ } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_titandb_live_blob_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "sum(rate(tikv_engine_compaction_reason{instance=~\"$instance\", db=\"$db\"}[1m])) by (cf, reason)", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "live blob size", - "refId": "A" + "legendFormat": "{{cf}} - {{reason}}", + "metric": "", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Live blob size", + "title": "Compaction reason", "tooltip": { "shared": true, "sort": 0, @@ -18249,11 +17190,11 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -18261,15 +17202,26 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, + ] + } + ], + "repeat": "db", + "title": "RocksDB - $db", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 3301, + "panels": [ { "aliasColors": {}, "bars": false, @@ -18279,16 +17231,16 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, + "w": 12, "x": 0, - "y": 57 + "y": 27 }, - "id": 3555, + "id": 3414, "legend": { "alignAsTable": true, "avg": true, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, "rightSide": true, "show": true, @@ -18312,30 +17264,39 @@ } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_titandb_num_live_blob_file{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "live blob file num", + "legendFormat": "key_size_avg", "refId": "A" }, { - "expr": "avg(tikv_engine_titandb_num_obsolete_blob_file{instance=~\"$instance\", db=\"$db\"})", + "expr": "quantile(0.95, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "key_size_0.95", + "refId": "C" + }, + { + "expr": "quantile(0.99, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", "hide": false, "intervalFactor": 1, - "legendFormat": "obsolete blob file num", - "refId": "B" + "legendFormat": "key_size_0.99", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob file", + "title": "Blob key size", "tooltip": { "shared": true, "sort": 0, @@ -18351,9 +17312,9 @@ }, "yaxes": [ { - "format": "short", + "format": "bytes", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -18381,16 +17342,16 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, - "x": 10, - "y": 57 + "w": 12, + "x": 12, + "y": 27 }, - "id": 3557, + "id": 3446, "legend": { "alignAsTable": true, "avg": true, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, "rightSide": true, "show": true, @@ -18414,29 +17375,37 @@ } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_titandb_live_blob_file_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "live blob file size", + "intervalFactor": 1, + "legendFormat": "value_size", "refId": "A" }, { - "expr": "avg(tikv_engine_titandb_obsolete_blob_file_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "obsolete blob file size", + "intervalFactor": 1, + "legendFormat": "value_size_0.95", "refId": "B" + }, + { + "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "value_size_0.99", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob file size", + "title": "Blob value size", "tooltip": { "shared": true, "sort": 0, @@ -18454,7 +17423,7 @@ { "format": "bytes", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -18472,29 +17441,7 @@ "align": false, "alignLevel": null } - } - ], - "repeat": "db", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, - "title": "Titan - $db", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 3512, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -18504,16 +17451,16 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, + "w": 12, "x": 0, - "y": 28 + "y": 33 }, - "id": 3513, + "id": 3412, "legend": { "alignAsTable": true, "avg": true, "current": false, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, @@ -18529,49 +17476,45 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3414, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "hide": false, "intervalFactor": 2, - "legendFormat": "key_size_avg", + "legendFormat": "seek_micros", "refId": "A" }, { - "expr": "quantile(0.95, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", "intervalFactor": 1, - "legendFormat": "key_size_0.95", - "refId": "C" + "legendFormat": "prev_micros", + "refId": "B" }, { - "expr": "quantile(0.99, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "key_size_0.99", - "refId": "D" + "legendFormat": "next_micros", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob key size", + "title": "Blob seek duration", "tooltip": { "shared": true, "sort": 0, @@ -18587,7 +17530,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "µs", "label": null, "logBase": 2, "max": null, @@ -18617,11 +17560,11 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, - "x": 10, - "y": 28 + "w": 12, + "x": 12, + "y": 33 }, - "id": 3514, + "id": 3410, "legend": { "alignAsTable": true, "avg": true, @@ -18642,39 +17585,37 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3446, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "value_size", + "intervalFactor": 2, + "legendFormat": "blob_gc_micros", "refId": "A" }, { - "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "quantile(0.95, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", "intervalFactor": 1, - "legendFormat": "value_size_0.95", + "legendFormat": "blob_gc_micros_0.95", "refId": "B" }, { - "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "quantile(0.99, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", "intervalFactor": 1, - "legendFormat": "value_size_0.99", + "legendFormat": "blob_gc_micros_0.99", "refId": "C" } ], @@ -18682,7 +17623,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob value size", + "title": "GC duration", "tooltip": { "shared": true, "sort": 0, @@ -18698,7 +17639,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "µs", "label": null, "logBase": 2, "max": null, @@ -18728,11 +17669,11 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, + "w": 12, "x": 0, - "y": 34 + "y": 39 }, - "id": 3515, + "id": 3342, "legend": { "alignAsTable": true, "avg": true, @@ -18753,47 +17694,52 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3412, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "seek_micros", + "legendFormat": "file_bytes_written", "refId": "A" }, { - "expr": "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "prev_micros", + "intervalFactor": 2, + "legendFormat": "file_bytes_read", "refId": "B" }, { - "expr": "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "next_micros", + "intervalFactor": 2, + "legendFormat": "keys_read", "refId": "C" + }, + { + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "keys_written", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob seek duration", + "title": "Blob flow", "tooltip": { "shared": true, "sort": 0, @@ -18809,9 +17755,9 @@ }, "yaxes": [ { - "format": "µs", + "format": "Bps", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -18839,16 +17785,16 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, - "x": 10, - "y": 34 + "w": 12, + "x": 12, + "y": 39 }, - "id": 3516, + "id": 3408, "legend": { "alignAsTable": true, "avg": true, "current": false, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, @@ -18864,47 +17810,52 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3410, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_average\"})", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "blob_gc_micros", + "intervalFactor": 1, + "legendFormat": "avg", "refId": "A" }, { - "expr": "quantile(0.95, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile99\"})", "format": "time_series", "intervalFactor": 1, - "legendFormat": "blob_gc_micros_0.95", + "legendFormat": "99%", "refId": "B" }, { - "expr": "quantile(0.99, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile95\"})", "format": "time_series", "intervalFactor": 1, - "legendFormat": "blob_gc_micros_0.99", + "legendFormat": "95%", "refId": "C" + }, + { + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_max\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "GC duration", + "title": "GC file write duration", "tooltip": { "shared": true, "sort": 0, @@ -18950,20 +17901,20 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, + "w": 12, "x": 0, - "y": 40 + "y": 45 }, - "id": 3517, + "id": 3344, "legend": { "alignAsTable": true, "avg": true, "current": false, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, - "total": false, + "total": true, "values": true }, "lines": true, @@ -18975,14 +17926,11 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3342, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], @@ -18991,39 +17939,25 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "file_bytes_written", - "refId": "A" - }, - { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", + "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_input_files_count\"}[2m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "file_bytes_read", + "legendFormat": "gc_num_files", "refId": "B" }, { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", + "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_output_files_count\"}[2m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "keys_read", + "legendFormat": "gc_num_new_files", "refId": "C" - }, - { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "keys_written", - "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob flow", + "title": "Blob GC file", "tooltip": { "shared": true, "sort": 0, @@ -19039,7 +17973,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -19069,11 +18003,11 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, - "x": 10, - "y": 40 + "w": 12, + "x": 12, + "y": 45 }, - "id": 3518, + "id": 3338, "legend": { "alignAsTable": true, "avg": true, @@ -19094,54 +18028,49 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3408, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_average\"})", + "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_seek\"}[2m]))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "avg", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "seek", "refId": "A" }, { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile99\"})", + "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_prev\"}[2m]))", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "99%", + "legendFormat": "prev", "refId": "B" }, { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile95\"})", + "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_next\"}[2m]))", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "95%", + "legendFormat": "next", "refId": "C" - }, - { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_max\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "GC file write duration", + "title": "Blob seek operations", "tooltip": { "shared": true, "sort": 0, @@ -19157,9 +18086,9 @@ }, "yaxes": [ { - "format": "µs", + "format": "ops", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -19187,20 +18116,20 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, + "w": 12, "x": 0, - "y": 46 + "y": 51 }, - "id": 3519, + "id": 3340, "legend": { "alignAsTable": true, "avg": true, "current": false, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, - "total": true, + "total": false, "values": true }, "lines": true, @@ -19212,14 +18141,11 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3344, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], @@ -19228,25 +18154,53 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_input_files_count\"}[2m]))", + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "gc_num_files", + "legendFormat": "bytes_written", + "refId": "A" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes_read", "refId": "B" }, { - "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_output_files_count\"}[2m]))", + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_overwritten\"}[2m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "gc_num_new_files", + "legendFormat": "bytes_overwritten", "refId": "C" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_relocated\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes_relocated", + "refId": "D" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_relocated\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "keys_relocated", + "refId": "E" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_overwritten\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "keys_overwritten", + "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob GC file", + "title": "Blob GC flows", "tooltip": { "shared": true, "sort": 0, @@ -19262,7 +18216,7 @@ }, "yaxes": [ { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -19270,7 +18224,7 @@ "show": true }, { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -19292,15 +18246,15 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, - "x": 10, - "y": 46 + "w": 12, + "x": 12, + "y": 51 }, - "id": 3520, + "id": 3523, "legend": { "alignAsTable": true, "avg": true, - "current": false, + "current": true, "max": true, "min": false, "rightSide": true, @@ -19317,14 +18271,11 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3338, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], @@ -19333,36 +18284,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_seek\"}[2m]))", + "expr": "avg(tikv_engine_titandb_live_blob_size{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "hide": false, - "interval": "", "intervalFactor": 2, - "legendFormat": "seek", + "legendFormat": "live blob size", "refId": "A" - }, - { - "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_prev\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "prev", - "refId": "B" - }, - { - "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_next\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "next", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob seek operations", + "title": "Live blob size", "tooltip": { "shared": true, "sort": 0, @@ -19378,7 +18311,7 @@ }, "yaxes": [ { - "format": "ops", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -19408,15 +18341,15 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, + "w": 12, "x": 0, - "y": 52 + "y": 57 }, - "id": 3521, + "id": 3555, "legend": { "alignAsTable": true, "avg": true, - "current": false, + "current": true, "max": true, "min": false, "rightSide": true, @@ -19433,14 +18366,11 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3340, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], @@ -19449,53 +18379,26 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", + "expr": "avg(tikv_engine_titandb_num_live_blob_file{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "bytes_written", + "legendFormat": "live blob file num", "refId": "A" }, { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "bytes_read", - "refId": "B" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_overwritten\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "bytes_overwritten", - "refId": "C" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_relocated\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "bytes_relocated", - "refId": "D" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_relocated\"}[2m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "keys_relocated", - "refId": "E" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_overwritten\"}[2m]))", + "expr": "avg(tikv_engine_titandb_num_obsolete_blob_file{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "keys_overwritten", - "refId": "F" + "legendFormat": "obsolete blob file num", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob GC flows", + "title": "Blob file", "tooltip": { "shared": true, "sort": 0, @@ -19511,7 +18414,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -19519,7 +18422,7 @@ "show": true }, { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -19531,31 +18434,7 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "repeatIteration": 1562643407086, - "repeatPanelId": 3301, - "scopedVars": { - "db": { - "selected": false, - "text": "raft", - "value": "raft" - } - }, - "title": "Titan - $db", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 2763, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -19564,18 +18443,18 @@ "datasource": "${DS_TEST-CLUSTER}", "fill": 1, "gridPos": { - "h": 7, + "h": 6, "w": 12, - "x": 0, - "y": 505 + "x": 12, + "y": 57 }, - "id": 2696, + "id": 3557, "legend": { "alignAsTable": true, - "avg": false, + "avg": true, "current": true, "max": true, - "min": true, + "min": false, "rightSide": true, "show": true, "total": false, @@ -19585,28 +18464,43 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "tikv_jemalloc_stats{instance=~\"$instance\"}", + "expr": "avg(tikv_engine_titandb_live_blob_file_size{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "hide": false, "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "live blob file size", "refId": "A" + }, + { + "expr": "avg(tikv_engine_titandb_obsolete_blob_file_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "obsolete blob file size", + "refId": "B" } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Jemalloc Stats", + "title": "Blob file size", "tooltip": { "shared": true, "sort": 0, @@ -19622,7 +18516,7 @@ }, "yaxes": [ { - "format": "decbytes", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -19637,11 +18531,15 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], - "repeat": null, - "title": "Memory", + "repeat": "db", + "title": "Titan - $db", "type": "row" }, { @@ -19650,7 +18548,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 23 }, "id": 2820, "panels": [ @@ -20143,10 +19041,108 @@ ], "title": "Lock manager", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 2763, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 505 + }, + "id": 2696, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "tikv_jemalloc_stats{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Jemalloc Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "title": "Memory", + "type": "row" } ], "refresh": "1m", - "schemaVersion": 16, + "schemaVersion": 18, "style": "dark", "tags": [], "templating": { @@ -20220,7 +19216,7 @@ ] }, "time": { - "from": "now-5m", + "from": "now-1h", "to": "now" }, "timepicker": { @@ -20249,7 +19245,7 @@ ] }, "timezone": "browser", - "title": "Test-Cluster-TiKV-Details", - "uid": "RDVQiEzZz", + "title": "Test-Cluster-TiKV-Details2", + "uid": "RDVQiEzZz2", "version": 2 } \ No newline at end of file