From 56045ac91d862dedf2c73970d2e81ea83fde0dfa Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Tue, 9 Jan 2024 23:09:24 +0800 Subject: [PATCH 1/2] This is an automated cherry-pick of #7674 close tikv/pd#7672 Signed-off-by: ti-chi-bot --- pkg/schedule/schedulers/evict_leader.go | 2 +- pkg/schedule/schedulers/evict_slow_store.go | 48 +++++++++++++++++++++ pkg/schedule/schedulers/evict_slow_trend.go | 2 +- pkg/schedule/schedulers/grant_leader.go | 2 +- server/api/router.go | 9 ++-- server/api/scheduler.go | 2 +- server/api/server_test.go | 9 ++++ tests/pdctl/scheduler/scheduler_test.go | 37 +++++++++++++++- 8 files changed, 102 insertions(+), 9 deletions(-) diff --git a/pkg/schedule/schedulers/evict_leader.go b/pkg/schedule/schedulers/evict_leader.go index cd635c1d592..f9e8cd27057 100644 --- a/pkg/schedule/schedulers/evict_leader.go +++ b/pkg/schedule/schedulers/evict_leader.go @@ -408,7 +408,7 @@ func (handler *evictLeaderHandler) UpdateConfig(w http.ResponseWriter, r *http.R handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } - handler.rd.JSON(w, http.StatusOK, nil) + handler.rd.JSON(w, http.StatusOK, "The scheduler has been applied to the store.") } func (handler *evictLeaderHandler) ListConfig(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/schedule/schedulers/evict_slow_store.go b/pkg/schedule/schedulers/evict_slow_store.go index a6665c3e5e7..515d257e95b 100644 --- a/pkg/schedule/schedulers/evict_slow_store.go +++ b/pkg/schedule/schedulers/evict_slow_store.go @@ -92,6 +92,54 @@ func (conf *evictSlowStoreSchedulerConfig) clearAndPersist() (oldID uint64, err return } +<<<<<<< HEAD +======= +type evictSlowStoreHandler struct { + rd *render.Render + config *evictSlowStoreSchedulerConfig +} + +func newEvictSlowStoreHandler(config *evictSlowStoreSchedulerConfig) http.Handler { + h := &evictSlowStoreHandler{ + config: config, + rd: render.New(render.Options{IndentJSON: true}), + } + router := mux.NewRouter() + router.HandleFunc("/config", h.UpdateConfig).Methods(http.MethodPost) + router.HandleFunc("/list", h.ListConfig).Methods(http.MethodGet) + return router +} + +func (handler *evictSlowStoreHandler) UpdateConfig(w http.ResponseWriter, r *http.Request) { + var input map[string]interface{} + if err := apiutil.ReadJSONRespondError(handler.rd, w, r.Body, &input); err != nil { + return + } + recoveryDurationGapFloat, ok := input["recovery-duration"].(float64) + if !ok { + handler.rd.JSON(w, http.StatusInternalServerError, errors.New("invalid argument for 'recovery-duration'").Error()) + return + } + handler.config.Lock() + defer handler.config.Unlock() + prevRecoveryDurationGap := handler.config.RecoveryDurationGap + recoveryDurationGap := uint64(recoveryDurationGapFloat) + handler.config.RecoveryDurationGap = recoveryDurationGap + if err := handler.config.persistLocked(); err != nil { + handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) + handler.config.RecoveryDurationGap = prevRecoveryDurationGap + return + } + log.Info("evict-slow-store-scheduler update 'recovery-duration' - unit: s", zap.Uint64("prev", prevRecoveryDurationGap), zap.Uint64("cur", recoveryDurationGap)) + handler.rd.JSON(w, http.StatusOK, "Config updated.") +} + +func (handler *evictSlowStoreHandler) ListConfig(w http.ResponseWriter, r *http.Request) { + conf := handler.config.Clone() + handler.rd.JSON(w, http.StatusOK, conf) +} + +>>>>>>> 8b8c78a78 (scheduler: add aduit log for scheduler config API and add resp msg for evict-leader (#7674)) type evictSlowStoreScheduler struct { *BaseScheduler conf *evictSlowStoreSchedulerConfig diff --git a/pkg/schedule/schedulers/evict_slow_trend.go b/pkg/schedule/schedulers/evict_slow_trend.go index 3983e9c345d..c7eb0f61a56 100644 --- a/pkg/schedule/schedulers/evict_slow_trend.go +++ b/pkg/schedule/schedulers/evict_slow_trend.go @@ -223,7 +223,7 @@ func (handler *evictSlowTrendHandler) UpdateConfig(w http.ResponseWriter, r *htt prevRecoveryDurationGap := atomic.LoadUint64(&handler.config.RecoveryDurationGap) atomic.StoreUint64(&handler.config.RecoveryDurationGap, recoveryDurationGap) log.Info("evict-slow-trend-scheduler update 'recovery-duration' - unit: s", zap.Uint64("prev", prevRecoveryDurationGap), zap.Uint64("cur", recoveryDurationGap)) - handler.rd.JSON(w, http.StatusOK, nil) + handler.rd.JSON(w, http.StatusOK, "Config updated.") } func (handler *evictSlowTrendHandler) ListConfig(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/schedule/schedulers/grant_leader.go b/pkg/schedule/schedulers/grant_leader.go index a085e69412d..ba5dbe02280 100644 --- a/pkg/schedule/schedulers/grant_leader.go +++ b/pkg/schedule/schedulers/grant_leader.go @@ -294,7 +294,7 @@ func (handler *grantLeaderHandler) UpdateConfig(w http.ResponseWriter, r *http.R handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } - handler.rd.JSON(w, http.StatusOK, nil) + handler.rd.JSON(w, http.StatusOK, "The scheduler has been applied to the store.") } func (handler *grantLeaderHandler) ListConfig(w http.ResponseWriter, r *http.Request) { diff --git a/server/api/router.go b/server/api/router.go index 93811e264f1..e02e4240e3e 100644 --- a/server/api/router.go +++ b/server/api/router.go @@ -88,10 +88,10 @@ func getFunctionName(f interface{}) string { // @BasePath /pd/api/v1 func createRouter(prefix string, svr *server.Server) *mux.Router { serviceMiddle := newServiceMiddlewareBuilder(svr) - registerPrefix := func(router *mux.Router, prefixPath string, + registerPrefix := func(router *mux.Router, prefixPath, name string, handleFunc func(http.ResponseWriter, *http.Request), opts ...createRouteOption) { routeCreateFunc(router.PathPrefix(prefixPath), serviceMiddle.createHandler(handleFunc), - getFunctionName(handleFunc), opts...) + name, opts...) } registerFunc := func(router *mux.Router, path string, handleFunc func(http.ResponseWriter, *http.Request), opts ...createRouteOption) { @@ -151,7 +151,8 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(clusterRouter, "/schedulers/diagnostic/{name}", diagnosticHandler.GetDiagnosticResult, setMethods(http.MethodGet), setAuditBackend(prometheus)) schedulerConfigHandler := newSchedulerConfigHandler(svr, rd) - registerPrefix(apiRouter, "/scheduler-config", schedulerConfigHandler.GetSchedulerConfig, setAuditBackend(prometheus)) + registerPrefix(apiRouter, "/scheduler-config", "HandleSchedulerConfig", schedulerConfigHandler.HandleSchedulerConfig, setMethods(http.MethodPost, http.MethodDelete, http.MethodPut, http.MethodPatch), setAuditBackend(localLog, prometheus)) + registerPrefix(apiRouter, "/scheduler-config", "GetSchedulerConfig", schedulerConfigHandler.HandleSchedulerConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) clusterHandler := newClusterHandler(svr, rd) registerFunc(apiRouter, "/cluster", clusterHandler.GetCluster, setMethods(http.MethodGet), setAuditBackend(prometheus)) @@ -371,7 +372,7 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { // API to set or unset failpoints if enableFailPointAPI { - registerPrefix(apiRouter, "/fail", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + registerPrefix(apiRouter, "/fail", "FailPoint", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // The HTTP handler of failpoint requires the full path to be the failpoint path. r.URL.Path = strings.TrimPrefix(r.URL.Path, prefix+apiPrefix+"/fail") new(failpoint.HttpHandler).ServeHTTP(w, r) diff --git a/server/api/scheduler.go b/server/api/scheduler.go index dea798edb39..e28e852b006 100644 --- a/server/api/scheduler.go +++ b/server/api/scheduler.go @@ -309,7 +309,7 @@ func newSchedulerConfigHandler(svr *server.Server, rd *render.Render) *scheduler } } -func (h *schedulerConfigHandler) GetSchedulerConfig(w http.ResponseWriter, r *http.Request) { +func (h *schedulerConfigHandler) HandleSchedulerConfig(w http.ResponseWriter, r *http.Request) { handler := h.svr.GetHandler() sh, err := handler.GetSchedulerConfigHandler() if err == nil && sh != nil { diff --git a/server/api/server_test.go b/server/api/server_test.go index 22989b92a03..8e747d063ae 100644 --- a/server/api/server_test.go +++ b/server/api/server_test.go @@ -220,9 +220,18 @@ func (suite *serviceTestSuite) TestServiceLabels() { suite.Equal("Profile", serviceLabel) accessPaths = suite.svr.GetServiceLabels("GetSchedulerConfig") +<<<<<<< HEAD suite.Len(accessPaths, 1) suite.Equal("/pd/api/v1/scheduler-config", accessPaths[0].Path) suite.Equal("", accessPaths[0].Method) +======= + re.Len(accessPaths, 1) + re.Equal("/pd/api/v1/scheduler-config", accessPaths[0].Path) + re.Equal("GET", accessPaths[0].Method) + accessPaths = suite.svr.GetServiceLabels("HandleSchedulerConfig") + re.Len(accessPaths, 4) + re.Equal("/pd/api/v1/scheduler-config", accessPaths[0].Path) +>>>>>>> 8b8c78a78 (scheduler: add aduit log for scheduler config API and add resp msg for evict-leader (#7674)) accessPaths = suite.svr.GetServiceLabels("ResignLeader") suite.Len(accessPaths, 1) diff --git a/tests/pdctl/scheduler/scheduler_test.go b/tests/pdctl/scheduler/scheduler_test.go index c5b118a9f5e..16c5f3197e5 100644 --- a/tests/pdctl/scheduler/scheduler_test.go +++ b/tests/pdctl/scheduler/scheduler_test.go @@ -272,9 +272,13 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"}, nil) re.NotContains(echo, "Success!") echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-leader-scheduler", "1"}, nil) - re.Contains(echo, "Success!") + re.Contains(echo, "Success! The scheduler is created.") + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-leader-scheduler", "2"}, nil) + re.Contains(echo, "Success! The scheduler has been applied to the store.") echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "evict-leader-scheduler-1"}, nil) re.Contains(echo, "Success!") + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "evict-leader-scheduler-2"}, nil) + re.Contains(echo, "Success!") echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "evict-leader-scheduler-1"}, nil) re.Contains(echo, "404") @@ -407,8 +411,39 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-leader-scheduler"}, nil) re.Contains(echo, "Success!") +<<<<<<< HEAD:tests/pdctl/scheduler/scheduler_test.go // test evict-slow-trend scheduler config echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-slow-trend-scheduler"}, nil) +======= + // test evict-slow-store && evict-slow-trend schedulers config + evictSlownessSchedulers := []string{"evict-slow-store-scheduler", "evict-slow-trend-scheduler"} + for _, schedulerName := range evictSlownessSchedulers { + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", schedulerName}, nil) + if strings.Contains(echo, "Success!") { + re.Contains(echo, "Success!") + } else { + re.Contains(echo, "scheduler existed") + } + testutil.Eventually(re, func() bool { + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) + return strings.Contains(echo, schedulerName) + }) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", schedulerName, "set", "recovery-duration", "100"}, nil) + re.Contains(echo, "Success! Config updated.") + conf = make(map[string]interface{}) + testutil.Eventually(re, func() bool { + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", schedulerName, "show"}, &conf) + return conf["recovery-duration"] == 100. + }) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", schedulerName}, nil) + re.Contains(echo, "Success!") + testutil.Eventually(re, func() bool { + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) + return !strings.Contains(echo, schedulerName) + }) + } + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-slow-store-scheduler"}, nil) +>>>>>>> 8b8c78a78 (scheduler: add aduit log for scheduler config API and add resp msg for evict-leader (#7674)):tools/pd-ctl/tests/scheduler/scheduler_test.go re.Contains(echo, "Success!") echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) re.Contains(echo, "evict-slow-trend-scheduler") From 66eb822bc0bf6679723ce5aa53222b0008a5cb05 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 10 Jan 2024 10:34:38 +0800 Subject: [PATCH 2/2] fix conflict Signed-off-by: Cabinfever_B --- pkg/schedule/schedulers/evict_slow_store.go | 48 --------------------- server/api/server_test.go | 12 ++---- tests/pdctl/scheduler/scheduler_test.go | 31 ------------- 3 files changed, 3 insertions(+), 88 deletions(-) diff --git a/pkg/schedule/schedulers/evict_slow_store.go b/pkg/schedule/schedulers/evict_slow_store.go index 515d257e95b..a6665c3e5e7 100644 --- a/pkg/schedule/schedulers/evict_slow_store.go +++ b/pkg/schedule/schedulers/evict_slow_store.go @@ -92,54 +92,6 @@ func (conf *evictSlowStoreSchedulerConfig) clearAndPersist() (oldID uint64, err return } -<<<<<<< HEAD -======= -type evictSlowStoreHandler struct { - rd *render.Render - config *evictSlowStoreSchedulerConfig -} - -func newEvictSlowStoreHandler(config *evictSlowStoreSchedulerConfig) http.Handler { - h := &evictSlowStoreHandler{ - config: config, - rd: render.New(render.Options{IndentJSON: true}), - } - router := mux.NewRouter() - router.HandleFunc("/config", h.UpdateConfig).Methods(http.MethodPost) - router.HandleFunc("/list", h.ListConfig).Methods(http.MethodGet) - return router -} - -func (handler *evictSlowStoreHandler) UpdateConfig(w http.ResponseWriter, r *http.Request) { - var input map[string]interface{} - if err := apiutil.ReadJSONRespondError(handler.rd, w, r.Body, &input); err != nil { - return - } - recoveryDurationGapFloat, ok := input["recovery-duration"].(float64) - if !ok { - handler.rd.JSON(w, http.StatusInternalServerError, errors.New("invalid argument for 'recovery-duration'").Error()) - return - } - handler.config.Lock() - defer handler.config.Unlock() - prevRecoveryDurationGap := handler.config.RecoveryDurationGap - recoveryDurationGap := uint64(recoveryDurationGapFloat) - handler.config.RecoveryDurationGap = recoveryDurationGap - if err := handler.config.persistLocked(); err != nil { - handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) - handler.config.RecoveryDurationGap = prevRecoveryDurationGap - return - } - log.Info("evict-slow-store-scheduler update 'recovery-duration' - unit: s", zap.Uint64("prev", prevRecoveryDurationGap), zap.Uint64("cur", recoveryDurationGap)) - handler.rd.JSON(w, http.StatusOK, "Config updated.") -} - -func (handler *evictSlowStoreHandler) ListConfig(w http.ResponseWriter, r *http.Request) { - conf := handler.config.Clone() - handler.rd.JSON(w, http.StatusOK, conf) -} - ->>>>>>> 8b8c78a78 (scheduler: add aduit log for scheduler config API and add resp msg for evict-leader (#7674)) type evictSlowStoreScheduler struct { *BaseScheduler conf *evictSlowStoreSchedulerConfig diff --git a/server/api/server_test.go b/server/api/server_test.go index 8e747d063ae..98c9070c348 100644 --- a/server/api/server_test.go +++ b/server/api/server_test.go @@ -220,18 +220,12 @@ func (suite *serviceTestSuite) TestServiceLabels() { suite.Equal("Profile", serviceLabel) accessPaths = suite.svr.GetServiceLabels("GetSchedulerConfig") -<<<<<<< HEAD suite.Len(accessPaths, 1) suite.Equal("/pd/api/v1/scheduler-config", accessPaths[0].Path) - suite.Equal("", accessPaths[0].Method) -======= - re.Len(accessPaths, 1) - re.Equal("/pd/api/v1/scheduler-config", accessPaths[0].Path) - re.Equal("GET", accessPaths[0].Method) + suite.Equal("GET", accessPaths[0].Method) accessPaths = suite.svr.GetServiceLabels("HandleSchedulerConfig") - re.Len(accessPaths, 4) - re.Equal("/pd/api/v1/scheduler-config", accessPaths[0].Path) ->>>>>>> 8b8c78a78 (scheduler: add aduit log for scheduler config API and add resp msg for evict-leader (#7674)) + suite.Len(accessPaths, 4) + suite.Equal("/pd/api/v1/scheduler-config", accessPaths[0].Path) accessPaths = suite.svr.GetServiceLabels("ResignLeader") suite.Len(accessPaths, 1) diff --git a/tests/pdctl/scheduler/scheduler_test.go b/tests/pdctl/scheduler/scheduler_test.go index 16c5f3197e5..9caea7c975c 100644 --- a/tests/pdctl/scheduler/scheduler_test.go +++ b/tests/pdctl/scheduler/scheduler_test.go @@ -411,39 +411,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-leader-scheduler"}, nil) re.Contains(echo, "Success!") -<<<<<<< HEAD:tests/pdctl/scheduler/scheduler_test.go // test evict-slow-trend scheduler config echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-slow-trend-scheduler"}, nil) -======= - // test evict-slow-store && evict-slow-trend schedulers config - evictSlownessSchedulers := []string{"evict-slow-store-scheduler", "evict-slow-trend-scheduler"} - for _, schedulerName := range evictSlownessSchedulers { - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", schedulerName}, nil) - if strings.Contains(echo, "Success!") { - re.Contains(echo, "Success!") - } else { - re.Contains(echo, "scheduler existed") - } - testutil.Eventually(re, func() bool { - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) - return strings.Contains(echo, schedulerName) - }) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", schedulerName, "set", "recovery-duration", "100"}, nil) - re.Contains(echo, "Success! Config updated.") - conf = make(map[string]interface{}) - testutil.Eventually(re, func() bool { - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", schedulerName, "show"}, &conf) - return conf["recovery-duration"] == 100. - }) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", schedulerName}, nil) - re.Contains(echo, "Success!") - testutil.Eventually(re, func() bool { - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) - return !strings.Contains(echo, schedulerName) - }) - } - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-slow-store-scheduler"}, nil) ->>>>>>> 8b8c78a78 (scheduler: add aduit log for scheduler config API and add resp msg for evict-leader (#7674)):tools/pd-ctl/tests/scheduler/scheduler_test.go re.Contains(echo, "Success!") echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) re.Contains(echo, "evict-slow-trend-scheduler")