From dfbd5812292093c51681f4283f8957f7a97fa2a9 Mon Sep 17 00:00:00 2001 From: Wenbo Zhang Date: Wed, 18 Jan 2023 20:35:15 +0800 Subject: [PATCH 1/3] schedulers: split transfer-witness-leader's limit from balance-leader ref tikv/pd#5638 Signed-off-by: Wenbo Zhang --- server/config/config.go | 44 +++++++++++--------- server/schedule/operator/kind.go | 38 +++++++++-------- server/schedulers/transfer_witness_leader.go | 6 +-- 3 files changed, 49 insertions(+), 39 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index 8d483117fd1..89875b55d96 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -670,6 +670,8 @@ type ScheduleConfig struct { MaxStorePreparingTime typeutil.Duration `toml:"max-store-preparing-time" json:"max-store-preparing-time"` // LeaderScheduleLimit is the max coexist leader schedules. LeaderScheduleLimit uint64 `toml:"leader-schedule-limit" json:"leader-schedule-limit"` + // WitnessLeaderScheduleLimit is the max coexist witness leader schedules. + WitnessLeaderScheduleLimit uint64 `toml:"witness-leader-schedule-limit" json:"witness-leader-schedule-limit"` // LeaderSchedulePolicy is the option to balance leader, there are some policies supported: ["count", "size"], default: "count" LeaderSchedulePolicy string `toml:"leader-schedule-policy" json:"leader-schedule-policy"` // RegionScheduleLimit is the max coexist region schedules. @@ -800,25 +802,26 @@ func (c *ScheduleConfig) Clone() *ScheduleConfig { } const ( - defaultMaxReplicas = 3 - defaultMaxSnapshotCount = 64 - defaultMaxPendingPeerCount = 64 - defaultMaxMergeRegionSize = 20 - defaultSplitMergeInterval = time.Hour - defaultSwitchWitnessInterval = time.Hour - defaultEnableDiagnostic = false - defaultPatrolRegionInterval = 10 * time.Millisecond - defaultMaxStoreDownTime = 30 * time.Minute - defaultLeaderScheduleLimit = 4 - defaultRegionScheduleLimit = 2048 - defaultWitnessScheduleLimit = 4 - defaultReplicaScheduleLimit = 64 - defaultMergeScheduleLimit = 8 - defaultHotRegionScheduleLimit = 4 - defaultTolerantSizeRatio = 0 - defaultLowSpaceRatio = 0.8 - defaultHighSpaceRatio = 0.7 - defaultRegionScoreFormulaVersion = "v2" + defaultMaxReplicas = 3 + defaultMaxSnapshotCount = 64 + defaultMaxPendingPeerCount = 64 + defaultMaxMergeRegionSize = 20 + defaultSplitMergeInterval = time.Hour + defaultSwitchWitnessInterval = time.Hour + defaultEnableDiagnostic = false + defaultPatrolRegionInterval = 10 * time.Millisecond + defaultMaxStoreDownTime = 30 * time.Minute + defaultLeaderScheduleLimit = 4 + defaultWitnessLeaderScheduleLimit = 2048 + defaultRegionScheduleLimit = 2048 + defaultWitnessScheduleLimit = 4 + defaultReplicaScheduleLimit = 64 + defaultMergeScheduleLimit = 8 + defaultHotRegionScheduleLimit = 4 + defaultTolerantSizeRatio = 0 + defaultLowSpaceRatio = 0.8 + defaultHighSpaceRatio = 0.7 + defaultRegionScoreFormulaVersion = "v2" // defaultHotRegionCacheHitsThreshold is the low hit number threshold of the // hot region. defaultHotRegionCacheHitsThreshold = 3 @@ -853,6 +856,9 @@ func (c *ScheduleConfig) adjust(meta *configMetaData, reloading bool) error { if !meta.IsDefined("leader-schedule-limit") { adjustUint64(&c.LeaderScheduleLimit, defaultLeaderScheduleLimit) } + if !meta.IsDefined("witness-leader-schedule-limit") { + adjustUint64(&c.WitnessLeaderScheduleLimit, defaultWitnessLeaderScheduleLimit) + } if !meta.IsDefined("region-schedule-limit") { adjustUint64(&c.RegionScheduleLimit, defaultRegionScheduleLimit) } diff --git a/server/schedule/operator/kind.go b/server/schedule/operator/kind.go index 73927ced7cd..265eea5ade6 100644 --- a/server/schedule/operator/kind.go +++ b/server/schedule/operator/kind.go @@ -42,32 +42,36 @@ const ( OpRegion // Include leader transfer. OpLeader + // Include witness leader transfer. + OpWitnessLeader // Include witness transfer. OpWitness opMax ) var flagToName = map[OpKind]string{ - OpLeader: "leader", - OpRegion: "region", - OpSplit: "split", - OpAdmin: "admin", - OpHotRegion: "hot-region", - OpReplica: "replica", - OpMerge: "merge", - OpRange: "range", - OpWitness: "witness", + OpLeader: "leader", + OpRegion: "region", + OpSplit: "split", + OpAdmin: "admin", + OpHotRegion: "hot-region", + OpReplica: "replica", + OpMerge: "merge", + OpRange: "range", + OpWitness: "witness", + OpWitnessLeader: "witness-leader", } var nameToFlag = map[string]OpKind{ - "leader": OpLeader, - "region": OpRegion, - "split": OpSplit, - "admin": OpAdmin, - "hot-region": OpHotRegion, - "replica": OpReplica, - "merge": OpMerge, - "range": OpRange, + "leader": OpLeader, + "region": OpRegion, + "split": OpSplit, + "admin": OpAdmin, + "hot-region": OpHotRegion, + "replica": OpReplica, + "merge": OpMerge, + "range": OpRange, + "witness-leader": OpWitnessLeader, } func (k OpKind) String() string { diff --git a/server/schedulers/transfer_witness_leader.go b/server/schedulers/transfer_witness_leader.go index def426cfff3..a02ff055666 100644 --- a/server/schedulers/transfer_witness_leader.go +++ b/server/schedulers/transfer_witness_leader.go @@ -33,7 +33,7 @@ const ( TransferWitnessLeaderType = "transfer-witness-leader" // TransferWitnessLeaderBatchSize is the number of operators to to transfer // leaders by one scheduling - transferWitnessLeaderBatchSize = 3 + transferWitnessLeaderBatchSize = 10 // TransferWitnessLeaderRecvMaxRegionSize is the max number of region can receive // TODO: make it a reasonable value transferWitnessLeaderRecvMaxRegionSize = 1000 @@ -81,9 +81,9 @@ func (s *trasferWitnessLeaderScheduler) GetType() string { func (s *trasferWitnessLeaderScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { // TODO: make sure the restriction is reasonable - allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetOpts().GetLeaderScheduleLimit() + allowed := s.OpController.OperatorCount(operator.OpWitnessLeader) < cluster.GetOpts().GetLeaderScheduleLimit() if !allowed { - operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpLeader.String()).Inc() + operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpWitnessLeader.String()).Inc() } return allowed } From 2a97f50b505041ff5a3c20bf65c2138056160e3e Mon Sep 17 00:00:00 2001 From: Wenbo Zhang Date: Thu, 19 Jan 2023 08:25:12 +0800 Subject: [PATCH 2/3] schedulers: remove limit ref tikv/pd#5638 Signed-off-by: Wenbo Zhang --- server/config/config.go | 44 +++++++++----------- server/schedulers/transfer_witness_leader.go | 7 +--- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index 89875b55d96..8d483117fd1 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -670,8 +670,6 @@ type ScheduleConfig struct { MaxStorePreparingTime typeutil.Duration `toml:"max-store-preparing-time" json:"max-store-preparing-time"` // LeaderScheduleLimit is the max coexist leader schedules. LeaderScheduleLimit uint64 `toml:"leader-schedule-limit" json:"leader-schedule-limit"` - // WitnessLeaderScheduleLimit is the max coexist witness leader schedules. - WitnessLeaderScheduleLimit uint64 `toml:"witness-leader-schedule-limit" json:"witness-leader-schedule-limit"` // LeaderSchedulePolicy is the option to balance leader, there are some policies supported: ["count", "size"], default: "count" LeaderSchedulePolicy string `toml:"leader-schedule-policy" json:"leader-schedule-policy"` // RegionScheduleLimit is the max coexist region schedules. @@ -802,26 +800,25 @@ func (c *ScheduleConfig) Clone() *ScheduleConfig { } const ( - defaultMaxReplicas = 3 - defaultMaxSnapshotCount = 64 - defaultMaxPendingPeerCount = 64 - defaultMaxMergeRegionSize = 20 - defaultSplitMergeInterval = time.Hour - defaultSwitchWitnessInterval = time.Hour - defaultEnableDiagnostic = false - defaultPatrolRegionInterval = 10 * time.Millisecond - defaultMaxStoreDownTime = 30 * time.Minute - defaultLeaderScheduleLimit = 4 - defaultWitnessLeaderScheduleLimit = 2048 - defaultRegionScheduleLimit = 2048 - defaultWitnessScheduleLimit = 4 - defaultReplicaScheduleLimit = 64 - defaultMergeScheduleLimit = 8 - defaultHotRegionScheduleLimit = 4 - defaultTolerantSizeRatio = 0 - defaultLowSpaceRatio = 0.8 - defaultHighSpaceRatio = 0.7 - defaultRegionScoreFormulaVersion = "v2" + defaultMaxReplicas = 3 + defaultMaxSnapshotCount = 64 + defaultMaxPendingPeerCount = 64 + defaultMaxMergeRegionSize = 20 + defaultSplitMergeInterval = time.Hour + defaultSwitchWitnessInterval = time.Hour + defaultEnableDiagnostic = false + defaultPatrolRegionInterval = 10 * time.Millisecond + defaultMaxStoreDownTime = 30 * time.Minute + defaultLeaderScheduleLimit = 4 + defaultRegionScheduleLimit = 2048 + defaultWitnessScheduleLimit = 4 + defaultReplicaScheduleLimit = 64 + defaultMergeScheduleLimit = 8 + defaultHotRegionScheduleLimit = 4 + defaultTolerantSizeRatio = 0 + defaultLowSpaceRatio = 0.8 + defaultHighSpaceRatio = 0.7 + defaultRegionScoreFormulaVersion = "v2" // defaultHotRegionCacheHitsThreshold is the low hit number threshold of the // hot region. defaultHotRegionCacheHitsThreshold = 3 @@ -856,9 +853,6 @@ func (c *ScheduleConfig) adjust(meta *configMetaData, reloading bool) error { if !meta.IsDefined("leader-schedule-limit") { adjustUint64(&c.LeaderScheduleLimit, defaultLeaderScheduleLimit) } - if !meta.IsDefined("witness-leader-schedule-limit") { - adjustUint64(&c.WitnessLeaderScheduleLimit, defaultWitnessLeaderScheduleLimit) - } if !meta.IsDefined("region-schedule-limit") { adjustUint64(&c.RegionScheduleLimit, defaultRegionScheduleLimit) } diff --git a/server/schedulers/transfer_witness_leader.go b/server/schedulers/transfer_witness_leader.go index a02ff055666..7d8ead34692 100644 --- a/server/schedulers/transfer_witness_leader.go +++ b/server/schedulers/transfer_witness_leader.go @@ -80,12 +80,7 @@ func (s *trasferWitnessLeaderScheduler) GetType() string { } func (s *trasferWitnessLeaderScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { - // TODO: make sure the restriction is reasonable - allowed := s.OpController.OperatorCount(operator.OpWitnessLeader) < cluster.GetOpts().GetLeaderScheduleLimit() - if !allowed { - operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpWitnessLeader.String()).Inc() - } - return allowed + return true } func (s *trasferWitnessLeaderScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { From 78aa2749f0556877502e4773692c783e3338991d Mon Sep 17 00:00:00 2001 From: Wenbo Zhang Date: Thu, 19 Jan 2023 10:20:36 +0800 Subject: [PATCH 3/3] address comments ref tikv/pd#5638 Signed-off-by: Wenbo Zhang --- server/schedulers/transfer_witness_leader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/schedulers/transfer_witness_leader.go b/server/schedulers/transfer_witness_leader.go index 7d8ead34692..b8895ae651d 100644 --- a/server/schedulers/transfer_witness_leader.go +++ b/server/schedulers/transfer_witness_leader.go @@ -133,7 +133,7 @@ func (s *trasferWitnessLeaderScheduler) scheduleTransferWitnessLeader(name, typ for _, t := range targets { targetIDs = append(targetIDs, t.GetID()) } - return operator.CreateTransferLeaderOperator(typ, cluster, region, region.GetLeader().GetStoreId(), target.GetID(), targetIDs, operator.OpLeader) + return operator.CreateTransferLeaderOperator(typ, cluster, region, region.GetLeader().GetStoreId(), target.GetID(), targetIDs, operator.OpWitnessLeader) } // RecvRegionInfo receives a checked region from coordinator