cherry pick tikv#3582 to release-4.0

Signed-off-by: ti-srebot <ti-srebot@pingcap.com>
ti-srebot · Apr 15, 2021 · cb9d150 · cb9d150
1 parent 05b9d6d
commit cb9d150
Show file tree

Hide file tree

Showing 4 changed files with 435 additions and 0 deletions.
diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json
@@ -6222,6 +6222,220 @@
         "x": 0,
         "y": 26
       },
+      "id": 1437,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "tidb-cluster",
+          "description": "",
+          "fill": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 20
+          },
+          "id": 1433,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(delta(pd_schedule_scatter_operators_count{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"skip\"}[1m])) by (event)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "skip-{{event}}",
+              "refId": "A"
+            },
+            {
+              "expr": "delta(pd_schedule_scatter_operators_count{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"fail\"}[1m])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "fail",
+              "refId": "B"
+            },
+            {
+              "expr": "delta(pd_schedule_scatter_operators_count{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"success\"}[1m])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "success",
+              "refId": "C"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "scatter operator event",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "opm",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "tidb-cluster",
+          "fill": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 20
+          },
+          "id": 1435,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(delta(pd_schedule_scatter_distribution{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",engine=\"tikv\",is_leader=\"false\"}[1m])) by (store)",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "peer-{{store}}",
+              "refId": "A"
+            },
+            {
+              "expr": "sum(delta(pd_schedule_scatter_distribution{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",engine=\"tikv\",is_leader=\"true\"}[1m])) by (store)",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "leader-{{store}}",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "scatter store selection",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "opm",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "title": "Scatter and Splitter",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 19
+      },
       "id": 123,
       "panels": [
         {

diff --git a/server/schedule/filter/filters.go b/server/schedule/filter/filters.go
@@ -537,11 +537,36 @@ const (
 	BlacklistTarget
 )
 
+<<<<<<< HEAD
 // BlacklistStoreFilter filters the store according to the blacklist.
 type BlacklistStoreFilter struct {
 	scope     string
 	blacklist map[uint64]struct{}
 	flag      BlacklistType
+=======
+func (f *StoreStateFilter) anyConditionMatch(typ int, opt *config.PersistOptions, store *core.StoreInfo) bool {
+	var funcs []conditionFunc
+	switch typ {
+	case leaderSource:
+		funcs = []conditionFunc{f.isTombstone, f.isDown, f.pauseLeaderTransfer, f.isDisconnected}
+	case regionSource:
+		funcs = []conditionFunc{f.isBusy, f.exceedRemoveLimit, f.tooManySnapshots}
+	case leaderTarget:
+		funcs = []conditionFunc{f.isTombstone, f.isOffline, f.isDown, f.pauseLeaderTransfer,
+			f.isDisconnected, f.isBusy, f.hasRejectLeaderProperty}
+	case regionTarget:
+		funcs = []conditionFunc{f.isTombstone, f.isOffline, f.isDown, f.isDisconnected, f.isBusy,
+			f.exceedAddLimit, f.tooManySnapshots, f.tooManyPendingPeers}
+	case scatterRegionTarget:
+		funcs = []conditionFunc{f.isTombstone, f.isOffline, f.isDown, f.isDisconnected, f.isBusy}
+	}
+	for _, cf := range funcs {
+		if cf(opt, store) {
+			return true
+		}
+	}
+	return false
+>>>>>>> b8c3d144... schedule: add metrcis for region scatter (#3582)
 }
 
 // NewBlacklistStoreFilter creates a blacklist filter.
@@ -840,6 +865,8 @@ const (
 	EngineKey = "engine"
 	// EngineTiFlash is the tiflash value of the engine label.
 	EngineTiFlash = "tiflash"
+	// EngineTiKV indicates the tikv engine in metrics
+	EngineTiKV = "tikv"
 )
 
 var allSpecialUses = []string{SpecialUseHotRegion, SpecialUseReserved}

diff --git a/server/schedule/metrics.go b/server/schedule/metrics.go
@@ -73,6 +73,22 @@ var (
 			Name:      "store_limit_cost",
 			Help:      "limit rate cost of store.",
 		}, []string{"store", "limit_type"})
+
+	scatterCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "pd",
+			Subsystem: "schedule",
+			Name:      "scatter_operators_count",
+			Help:      "Counter of region scatter operators.",
+		}, []string{"type", "event"})
+
+	scatterDistributionCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "pd",
+			Subsystem: "schedule",
+			Name:      "scatter_distribution",
+			Help:      "Counter of the distribution in scatter.",
+		}, []string{"store", "is_leader", "engine"})
 )
 
 func init() {
@@ -83,4 +99,6 @@ func init() {
 	prometheus.MustRegister(storeLimitRateGauge)
 	prometheus.MustRegister(storeLimitCostCounter)
 	prometheus.MustRegister(operatorWaitCounter)
+	prometheus.MustRegister(scatterCounter)
+	prometheus.MustRegister(scatterDistributionCounter)
 }