From d599182d70b13b0b6d4f804e0f337ed01c7cd399 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 22 Mar 2024 17:55:27 +0800 Subject: [PATCH 1/5] metrics: fix dr-auto-sync metics Signed-off-by: lhy1024 --- metrics/grafana/pd.json | 12 ++++++------ pkg/replication/replication_mode.go | 5 ++++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index 30014a959dc..91823d5fb18 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -1738,7 +1738,7 @@ "tableColumn": "idalloc", "targets": [ { - "expr": "pd_cluster_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"idalloc\"}!=0", + "expr": "max(pd_cluster_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"idalloc\"})by(type)", "format": "time_series", "hide": false, "instant": true, @@ -2284,7 +2284,7 @@ "tableColumn": "tso", "targets": [ { - "expr": "pd_cluster_tso{type=\"tso\", dc=\"global\"}!=0", + "expr": "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})by(type)", "format": "time_series", "instant": true, "interval": "", @@ -2588,7 +2588,7 @@ "tableColumn": "tso", "targets": [ { - "expr": "pd_cluster_tso{type=\"tso\", dc=\"global\"}!=0", + "expr": "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})by(type)", "format": "time_series", "instant": true, "interval": "", @@ -13266,7 +13266,7 @@ "targets": [ { "exemplar": true, - "expr": "rate(pd_replication_dr_tick_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[5m])", + "expr": "rate(pd_replication_dr_tick_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])!=0", "instant": false, "interval": "", "legendFormat": "{{instance}}", @@ -13368,7 +13368,7 @@ "targets": [ { "exemplar": true, - "expr": "pd_replication_dr_state_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", + "expr": "pd_replication_dr_state_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}!=0", "instant": false, "interval": "", "legendFormat": "{{instance}}", @@ -13763,4 +13763,4 @@ "title": "Test-Cluster-PD", "uid": "Q6RuHYIWk", "version": 1 -} +} \ No newline at end of file diff --git a/pkg/replication/replication_mode.go b/pkg/replication/replication_mode.go index 9776a36a8f3..5f6b212529b 100644 --- a/pkg/replication/replication_mode.go +++ b/pkg/replication/replication_mode.go @@ -366,7 +366,10 @@ func (m *ModeManager) Run(ctx context.Context) { }() go func() { - defer wg.Done() + defer func() { + wg.Done() + drStateGauge.Set(0) + }() ticker := time.NewTicker(replicateStateInterval) defer ticker.Stop() for { From 7ef86f419bfaaf08c21d6e6b4380bcc81d99c208 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Mon, 25 Mar 2024 23:26:01 +0800 Subject: [PATCH 2/5] fix metrics Signed-off-by: lhy1024 --- metrics/grafana/pd.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index 91823d5fb18..c37c4307fcf 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -13098,7 +13098,7 @@ "targets": [ { "exemplar": true, - "expr": "pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}", + "expr": "pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}!=0", "instant": false, "interval": "", "legendFormat": "{{instance}}", @@ -13266,7 +13266,7 @@ "targets": [ { "exemplar": true, - "expr": "rate(pd_replication_dr_tick_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])!=0", + "expr": "rate(pd_replication_dr_tick_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[5m])", "instant": false, "interval": "", "legendFormat": "{{instance}}", @@ -13368,7 +13368,7 @@ "targets": [ { "exemplar": true, - "expr": "pd_replication_dr_state_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}!=0", + "expr": "pd_replication_dr_state_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "instant": false, "interval": "", "legendFormat": "{{instance}}", @@ -13763,4 +13763,4 @@ "title": "Test-Cluster-PD", "uid": "Q6RuHYIWk", "version": 1 -} \ No newline at end of file +} From 33a1ed2b37c8ec4270f67e3339cb5ad8719fda67 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Mon, 25 Mar 2024 23:45:06 +0800 Subject: [PATCH 3/5] remove instance Signed-off-by: lhy1024 --- metrics/grafana/pd.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index c37c4307fcf..802b2a81140 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -13098,7 +13098,7 @@ "targets": [ { "exemplar": true, - "expr": "pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}!=0", + "expr": "pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}!=0", "instant": false, "interval": "", "legendFormat": "{{instance}}", @@ -13763,4 +13763,4 @@ "title": "Test-Cluster-PD", "uid": "Q6RuHYIWk", "version": 1 -} +} \ No newline at end of file From 92c7de0c875a9d4e8a5b4ed4e69262dbb5a04979 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Tue, 26 Mar 2024 12:28:32 +0800 Subject: [PATCH 4/5] fix metrics Signed-off-by: lhy1024 --- metrics/grafana/pd.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index 802b2a81140..841662722e8 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -13081,7 +13081,7 @@ "id": 1601, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { @@ -13098,7 +13098,7 @@ "targets": [ { "exemplar": true, - "expr": "pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}!=0", + "expr": "max(pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", "instant": false, "interval": "", "legendFormat": "{{instance}}", @@ -13266,7 +13266,7 @@ "targets": [ { "exemplar": true, - "expr": "rate(pd_replication_dr_tick_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[5m])", + "expr": "rate(pd_replication_dr_tick_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])", "instant": false, "interval": "", "legendFormat": "{{instance}}", From c163323e1cb06d18d9ce98080ff80da0fb43805f Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Tue, 26 Mar 2024 12:29:30 +0800 Subject: [PATCH 5/5] fix lint Signed-off-by: lhy1024 --- metrics/grafana/pd.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index 841662722e8..89f2828757f 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -13763,4 +13763,4 @@ "title": "Test-Cluster-PD", "uid": "Q6RuHYIWk", "version": 1 -} \ No newline at end of file +}