diff --git a/Makefile b/Makefile index 01558556ee4..77ad51a1a87 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,10 @@ else BUILD_CGO_ENABLED := 1 endif +ifeq ($(FAILPOINT), 1) + BUILD_TAGS += with_fail +endif + ifeq ("$(WITH_RACE)", "1") BUILD_FLAGS += -race BUILD_CGO_ENABLED := 1 @@ -73,6 +77,11 @@ PD_SERVER_DEP += dashboard-ui pd-server: ${PD_SERVER_DEP} CGO_ENABLED=$(BUILD_CGO_ENABLED) go build $(BUILD_FLAGS) -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -tags "$(BUILD_TAGS)" -o $(BUILD_BIN_PATH)/pd-server cmd/pd-server/main.go +pd-server-failpoint: + @$(FAILPOINT_ENABLE) + FAILPOINT=1 $(MAKE) pd-server || { $(FAILPOINT_DISABLE); exit 1; } + @$(FAILPOINT_DISABLE) + pd-server-basic: SWAGGER=0 DASHBOARD=0 $(MAKE) pd-server diff --git a/pkg/utils/etcdutil/etcdutil.go b/pkg/utils/etcdutil/etcdutil.go index b59a9581996..9d7fe7bfeca 100644 --- a/pkg/utils/etcdutil/etcdutil.go +++ b/pkg/utils/etcdutil/etcdutil.go @@ -106,6 +106,10 @@ func AddEtcdMember(client *clientv3.Client, urls []string) (*clientv3.MemberAddR // ListEtcdMembers returns a list of internal etcd members. func ListEtcdMembers(client *clientv3.Client) (*clientv3.MemberListResponse, error) { + failpoint.Inject("SlowEtcdMemberList", func(val failpoint.Value) { + d := val.(int) + time.Sleep(time.Duration(d) * time.Second) + }) ctx, cancel := context.WithTimeout(client.Ctx(), DefaultRequestTimeout) listResp, err := client.MemberList(ctx) cancel() @@ -132,6 +136,10 @@ func EtcdKVGet(c *clientv3.Client, key string, opts ...clientv3.OpOption) (*clie defer cancel() start := time.Now() + failpoint.Inject("SlowEtcdKVGet", func(val failpoint.Value) { + d := val.(int) + time.Sleep(time.Duration(d) * time.Second) + }) resp, err := clientv3.NewKV(c).Get(ctx, key, opts...) if cost := time.Since(start); cost > DefaultSlowRequestTime { log.Warn("kv gets too slow", zap.String("request-key", key), zap.Duration("cost", cost), errs.ZapError(err)) diff --git a/server/api/failpoint.go b/server/api/failpoint.go new file mode 100644 index 00000000000..05c3850fcd8 --- /dev/null +++ b/server/api/failpoint.go @@ -0,0 +1,22 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build with_fail +// +build with_fail + +package api + +func init() { + enableFailPointAPI = true +} diff --git a/server/api/router.go b/server/api/router.go index ea99e82cdd8..ce649ba9aef 100644 --- a/server/api/router.go +++ b/server/api/router.go @@ -31,6 +31,9 @@ import ( "github.com/unrolled/render" ) +// enableFailPointAPI enable fail point API handler. +var enableFailPointAPI bool + // createRouteOption is used to register service for mux.Route type createRouteOption func(route *mux.Route) @@ -365,15 +368,13 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(apiRouter, "/admin/reset-ts", tsoAdminHandler.ResetTS, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) // API to set or unset failpoints - failpoint.Inject("enableFailpointAPI", func() { - // this function will be named to "func2". It may be used in test + if enableFailPointAPI { registerPrefix(apiRouter, "/fail", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // The HTTP handler of failpoint requires the full path to be the failpoint path. r.URL.Path = strings.TrimPrefix(r.URL.Path, prefix+apiPrefix+"/fail") new(failpoint.HttpHandler).ServeHTTP(w, r) - }), setAuditBackend("test")) - }) - + }), setAuditBackend(localLog)) + } // Deprecated: use /pd/api/v1/health instead. rootRouter.HandleFunc("/health", healthHandler.GetHealthStatus).Methods(http.MethodGet) // Deprecated: use /pd/api/v1/ping instead.