diff --git a/executor/adapter.go b/executor/adapter.go index 4b2428753e16c..3360f9b8898d1 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -1604,6 +1604,9 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) { totalQueryProcHistogramGeneral.Observe(costTime.Seconds()) totalCopProcHistogramGeneral.Observe(execDetail.TimeDetail.ProcessTime.Seconds()) totalCopWaitHistogramGeneral.Observe(execDetail.TimeDetail.WaitTime.Seconds()) + if execDetail.ScanDetail != nil && execDetail.ScanDetail.ProcessedKeys != 0 { + metrics.CopMVCCRatioHistogram.WithLabelValues(metrics.LblGeneral).Observe(float64(execDetail.ScanDetail.TotalKeys) / float64(execDetail.ScanDetail.ProcessedKeys)) + } } var userString string if sessVars.User != nil { diff --git a/metrics/grafana/tidb.json b/metrics/grafana/tidb.json index 78ab973f7277a..9e2f362c74c84 100644 --- a/metrics/grafana/tidb.json +++ b/metrics/grafana/tidb.json @@ -814,31 +814,44 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "$$hashKey": "object:211", + "alias": "/mvcc_ratio/", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_process_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,sql_type))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_process_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[1m])) by (le,sql_type))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "all_proc_{{sql_type}}", + "legendFormat": "all_proc", "refId": "A" }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,sql_type))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[1m])) by (le,sql_type))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "all_cop_proc_{{sql_type}}", + "legendFormat": "all_cop_proc", "refId": "B" }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,sql_type))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[1m])) by (le,sql_type))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "all_cop_wait_{{sql_type}}", + "legendFormat": "all_cop_wait", "refId": "C" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_mvcc_ratio_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[1m])) by (le,sql_type))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mvcc_ratio", + "refId": "D" } ], "thresholds": [], @@ -861,6 +874,7 @@ }, "yaxes": [ { + "$$hashKey": "object:119", "format": "s", "label": null, "logBase": 2, @@ -869,6 +883,7 @@ "show": true }, { + "$$hashKey": "object:120", "format": "short", "label": null, "logBase": 1, diff --git a/metrics/metrics.go b/metrics/metrics.go index 70eff78bc4318..bdf5592e2a916 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -174,6 +174,7 @@ func RegisterMetrics() { prometheus.MustRegister(TotalQueryProcHistogram) prometheus.MustRegister(TotalCopProcHistogram) prometheus.MustRegister(TotalCopWaitHistogram) + prometheus.MustRegister(CopMVCCRatioHistogram) prometheus.MustRegister(HandleSchemaValidate) prometheus.MustRegister(MaxProcs) prometheus.MustRegister(GOGC) diff --git a/metrics/server.go b/metrics/server.go index 6bd34c4a8b903..ab0962d98bc01 100644 --- a/metrics/server.go +++ b/metrics/server.go @@ -221,6 +221,15 @@ var ( Buckets: prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days }, []string{LblSQLType}) + CopMVCCRatioHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "tidb", + Subsystem: "server", + Name: "slow_query_cop_mvcc_ratio", + Help: "Bucketed histogram of all cop total keys / processed keys in slow queries.", + Buckets: prometheus.ExponentialBuckets(0.5, 2, 21), // 0.5 ~ 262144 + }, []string{LblSQLType}) + MaxProcs = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "tidb",