schedulers: split transfer-witness-leader's limit from balance-leader

ref tikv#5638 Signed-off-by: Wenbo Zhang <ethercflow@gmail.com>
ethercflow · Jan 18, 2023 · dfbd581 · dfbd581
1 parent 082fc6a
commit dfbd581
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 39 deletions.
diff --git a/server/config/config.go b/server/config/config.go
@@ -670,6 +670,8 @@ type ScheduleConfig struct {
 	MaxStorePreparingTime typeutil.Duration `toml:"max-store-preparing-time" json:"max-store-preparing-time"`
 	// LeaderScheduleLimit is the max coexist leader schedules.
 	LeaderScheduleLimit uint64 `toml:"leader-schedule-limit" json:"leader-schedule-limit"`
+	// WitnessLeaderScheduleLimit is the max coexist witness leader schedules.
+	WitnessLeaderScheduleLimit uint64 `toml:"witness-leader-schedule-limit" json:"witness-leader-schedule-limit"`
 	// LeaderSchedulePolicy is the option to balance leader, there are some policies supported: ["count", "size"], default: "count"
 	LeaderSchedulePolicy string `toml:"leader-schedule-policy" json:"leader-schedule-policy"`
 	// RegionScheduleLimit is the max coexist region schedules.
@@ -800,25 +802,26 @@ func (c *ScheduleConfig) Clone() *ScheduleConfig {
 }
 
 const (
-	defaultMaxReplicas               = 3
-	defaultMaxSnapshotCount          = 64
-	defaultMaxPendingPeerCount       = 64
-	defaultMaxMergeRegionSize        = 20
-	defaultSplitMergeInterval        = time.Hour
-	defaultSwitchWitnessInterval     = time.Hour
-	defaultEnableDiagnostic          = false
-	defaultPatrolRegionInterval      = 10 * time.Millisecond
-	defaultMaxStoreDownTime          = 30 * time.Minute
-	defaultLeaderScheduleLimit       = 4
-	defaultRegionScheduleLimit       = 2048
-	defaultWitnessScheduleLimit      = 4
-	defaultReplicaScheduleLimit      = 64
-	defaultMergeScheduleLimit        = 8
-	defaultHotRegionScheduleLimit    = 4
-	defaultTolerantSizeRatio         = 0
-	defaultLowSpaceRatio             = 0.8
-	defaultHighSpaceRatio            = 0.7
-	defaultRegionScoreFormulaVersion = "v2"
+	defaultMaxReplicas                = 3
+	defaultMaxSnapshotCount           = 64
+	defaultMaxPendingPeerCount        = 64
+	defaultMaxMergeRegionSize         = 20
+	defaultSplitMergeInterval         = time.Hour
+	defaultSwitchWitnessInterval      = time.Hour
+	defaultEnableDiagnostic           = false
+	defaultPatrolRegionInterval       = 10 * time.Millisecond
+	defaultMaxStoreDownTime           = 30 * time.Minute
+	defaultLeaderScheduleLimit        = 4
+	defaultWitnessLeaderScheduleLimit = 2048
+	defaultRegionScheduleLimit        = 2048
+	defaultWitnessScheduleLimit       = 4
+	defaultReplicaScheduleLimit       = 64
+	defaultMergeScheduleLimit         = 8
+	defaultHotRegionScheduleLimit     = 4
+	defaultTolerantSizeRatio          = 0
+	defaultLowSpaceRatio              = 0.8
+	defaultHighSpaceRatio             = 0.7
+	defaultRegionScoreFormulaVersion  = "v2"
 	// defaultHotRegionCacheHitsThreshold is the low hit number threshold of the
 	// hot region.
 	defaultHotRegionCacheHitsThreshold = 3
@@ -853,6 +856,9 @@ func (c *ScheduleConfig) adjust(meta *configMetaData, reloading bool) error {
 	if !meta.IsDefined("leader-schedule-limit") {
 		adjustUint64(&c.LeaderScheduleLimit, defaultLeaderScheduleLimit)
 	}
+	if !meta.IsDefined("witness-leader-schedule-limit") {
+		adjustUint64(&c.WitnessLeaderScheduleLimit, defaultWitnessLeaderScheduleLimit)
+	}
 	if !meta.IsDefined("region-schedule-limit") {
 		adjustUint64(&c.RegionScheduleLimit, defaultRegionScheduleLimit)
 	}

diff --git a/server/schedule/operator/kind.go b/server/schedule/operator/kind.go
@@ -42,32 +42,36 @@ const (
 	OpRegion
 	// Include leader transfer.
 	OpLeader
+	// Include witness leader transfer.
+	OpWitnessLeader
 	// Include witness transfer.
 	OpWitness
 	opMax
 )
 
 var flagToName = map[OpKind]string{
-	OpLeader:    "leader",
-	OpRegion:    "region",
-	OpSplit:     "split",
-	OpAdmin:     "admin",
-	OpHotRegion: "hot-region",
-	OpReplica:   "replica",
-	OpMerge:     "merge",
-	OpRange:     "range",
-	OpWitness:   "witness",
+	OpLeader:        "leader",
+	OpRegion:        "region",
+	OpSplit:         "split",
+	OpAdmin:         "admin",
+	OpHotRegion:     "hot-region",
+	OpReplica:       "replica",
+	OpMerge:         "merge",
+	OpRange:         "range",
+	OpWitness:       "witness",
+	OpWitnessLeader: "witness-leader",
 }
 
 var nameToFlag = map[string]OpKind{
-	"leader":     OpLeader,
-	"region":     OpRegion,
-	"split":      OpSplit,
-	"admin":      OpAdmin,
-	"hot-region": OpHotRegion,
-	"replica":    OpReplica,
-	"merge":      OpMerge,
-	"range":      OpRange,
+	"leader":         OpLeader,
+	"region":         OpRegion,
+	"split":          OpSplit,
+	"admin":          OpAdmin,
+	"hot-region":     OpHotRegion,
+	"replica":        OpReplica,
+	"merge":          OpMerge,
+	"range":          OpRange,
+	"witness-leader": OpWitnessLeader,
 }
 
 func (k OpKind) String() string {

diff --git a/server/schedulers/transfer_witness_leader.go b/server/schedulers/transfer_witness_leader.go
@@ -33,7 +33,7 @@ const (
 	TransferWitnessLeaderType = "transfer-witness-leader"
 	// TransferWitnessLeaderBatchSize is the number of operators to to transfer
 	// leaders by one scheduling
-	transferWitnessLeaderBatchSize = 3
+	transferWitnessLeaderBatchSize = 10
 	// TransferWitnessLeaderRecvMaxRegionSize is the max number of region can receive
 	// TODO: make it a reasonable value
 	transferWitnessLeaderRecvMaxRegionSize = 1000
@@ -81,9 +81,9 @@ func (s *trasferWitnessLeaderScheduler) GetType() string {
 
 func (s *trasferWitnessLeaderScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool {
 	// TODO: make sure the restriction is reasonable
-	allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetOpts().GetLeaderScheduleLimit()
+	allowed := s.OpController.OperatorCount(operator.OpWitnessLeader) < cluster.GetOpts().GetLeaderScheduleLimit()
 	if !allowed {
-		operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpLeader.String()).Inc()
+		operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpWitnessLeader.String()).Inc()
 	}
 	return allowed
 }