From 09728d33b942ad64aee5545d899286af226b1106 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Tue, 7 Feb 2023 22:13:47 +0800 Subject: [PATCH 01/14] *: decouple the dependency between server and mcs Signed-off-by: lhy1024 --- pkg/autoscaling/service.go | 5 +- pkg/basic_server/basic_server.go | 9 ++- pkg/dashboard/dashboard.go | 9 +-- pkg/dashboard/without_dashboard.go | 5 +- pkg/mcs/registry/registry.go | 15 ++--- .../resource_manager/server/apis/v1/api.go | 6 +- .../resource_manager/server/grpc_service.go | 11 ++-- pkg/mcs/resource_manager/server/manager.go | 4 +- pkg/swaggerserver/swaggerserver.go | 5 +- pkg/utils/apiutil/apiutil.go | 45 +++++++++++++ pkg/utils/apiutil/serverapi/middleware.go | 7 +- server/api/server.go | 5 +- server/apiv2/router.go | 5 +- server/server.go | 12 +--- server/server_test.go | 16 ++--- server/util.go | 66 ++----------------- tests/pdctl/global_test.go | 4 +- tests/registry/registry_test.go | 18 ++--- 18 files changed, 114 insertions(+), 133 deletions(-) diff --git a/pkg/autoscaling/service.go b/pkg/autoscaling/service.go index ce86235104e..a8d84b2e598 100644 --- a/pkg/autoscaling/service.go +++ b/pkg/autoscaling/service.go @@ -18,6 +18,7 @@ import ( "context" "net/http" + "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/apiutil/serverapi" "github.com/tikv/pd/server" "github.com/unrolled/render" @@ -27,7 +28,7 @@ import ( const autoScalingPrefix = "/autoscaling" var ( - autoscalingServiceGroup = server.APIServiceGroup{ + autoscalingServiceGroup = apiutil.APIServiceGroup{ Name: "autoscaling", Version: "v1alpha", IsCore: false, @@ -36,7 +37,7 @@ var ( ) // NewHandler creates a HTTP handler for auto scaling. -func NewHandler(_ context.Context, svr *server.Server) (http.Handler, server.APIServiceGroup, error) { +func NewHandler(_ context.Context, svr *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { autoScalingHandler := http.NewServeMux() rd := render.New(render.Options{ IndentJSON: true, diff --git a/pkg/basic_server/basic_server.go b/pkg/basic_server/basic_server.go index b5758485718..d9ca2028018 100644 --- a/pkg/basic_server/basic_server.go +++ b/pkg/basic_server/basic_server.go @@ -18,6 +18,7 @@ import ( "context" "net/http" + "github.com/tikv/pd/pkg/member" "go.etcd.io/etcd/clientv3" ) @@ -27,14 +28,18 @@ type Server interface { Name() string // Context returns the context of server. Context() context.Context - // Run runs the server. Run() error // Close closes the server. Close() - // GetClient returns builtin etcd client. GetClient() *clientv3.Client // GetHTTPClient returns builtin etcd client. GetHTTPClient() *http.Client + // AddStartCallback adds a callback in the startServer phase. + AddStartCallback(callbacks ...func()) + // GetMember returns the member information. + GetMember() *member.Member + // AddLeaderCallback adds a callback in the leader campaign phase. + AddLeaderCallback(callbacks ...func(context.Context)) } diff --git a/pkg/dashboard/dashboard.go b/pkg/dashboard/dashboard.go index fc42c67aca2..9cd61a6f332 100644 --- a/pkg/dashboard/dashboard.go +++ b/pkg/dashboard/dashboard.go @@ -30,18 +30,19 @@ import ( "github.com/tikv/pd/pkg/dashboard/distroutil" "github.com/tikv/pd/pkg/dashboard/keyvisual" ui "github.com/tikv/pd/pkg/dashboard/uiserver" + "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" ) var ( - apiServiceGroup = server.APIServiceGroup{ + apiServiceGroup = apiutil.APIServiceGroup{ Name: "dashboard-api", Version: "v1", IsCore: false, PathPrefix: config.APIPathPrefix, } - uiServiceGroup = server.APIServiceGroup{ + uiServiceGroup = apiutil.APIServiceGroup{ Name: "dashboard-ui", Version: "v1", IsCore: false, @@ -68,7 +69,7 @@ func GetServiceBuilders() []server.HandlerBuilder { // The order of execution must be sequential. return []server.HandlerBuilder{ // Dashboard API Service - func(ctx context.Context, srv *server.Server) (http.Handler, server.APIServiceGroup, error) { + func(ctx context.Context, srv *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { distroutil.MustLoadAndReplaceStrings() if cfg, err = adapter.GenDashboardConfig(srv); err != nil { @@ -98,7 +99,7 @@ func GetServiceBuilders() []server.HandlerBuilder { return apiserver.Handler(s), apiServiceGroup, nil }, // Dashboard UI - func(context.Context, *server.Server) (http.Handler, server.APIServiceGroup, error) { + func(context.Context, *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { if err != nil { return nil, uiServiceGroup, err } diff --git a/pkg/dashboard/without_dashboard.go b/pkg/dashboard/without_dashboard.go index 879b4de3193..bd63e3ab2f7 100644 --- a/pkg/dashboard/without_dashboard.go +++ b/pkg/dashboard/without_dashboard.go @@ -25,11 +25,12 @@ import ( "github.com/pingcap/tidb-dashboard/pkg/config" + "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" ) var ( - serviceGroup = server.APIServiceGroup{ + serviceGroup = apiutil.APIServiceGroup{ Name: "dashboard", Version: "v1", IsCore: false, @@ -43,7 +44,7 @@ func SetCheckInterval(time.Duration) {} // GetServiceBuilders returns a empty Dashboard Builder func GetServiceBuilders() []server.HandlerBuilder { return []server.HandlerBuilder{ - func(context.Context, *server.Server) (http.Handler, server.APIServiceGroup, error) { + func(context.Context, *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = io.WriteString(w, "Dashboard is not built.\n") }) diff --git a/pkg/mcs/registry/registry.go b/pkg/mcs/registry/registry.go index 98de31dfd3d..e3f299a6c98 100644 --- a/pkg/mcs/registry/registry.go +++ b/pkg/mcs/registry/registry.go @@ -13,7 +13,6 @@ // limitations under the License. // Package registry is used to register the services. -// TODO: Remove the `pd/server` dependencies // TODO: Use the `uber/fx` to manage the lifecycle of services. package registry @@ -22,7 +21,7 @@ import ( "net/http" "github.com/pingcap/log" - "github.com/tikv/pd/server" + bs "github.com/tikv/pd/pkg/basic_server" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -33,7 +32,7 @@ var ( ) // ServiceBuilder is a function that creates a grpc service. -type ServiceBuilder func(*server.Server) RegistrableService +type ServiceBuilder func(bs.Server) RegistrableService // RegistrableService is the interface that should wraps the RegisterService method. type RegistrableService interface { @@ -60,7 +59,7 @@ func createServiceName(prefix, name string) string { } // InstallAllGRPCServices installs all registered grpc services. -func (r *ServiceRegistry) InstallAllGRPCServices(srv *server.Server, g *grpc.Server) { +func (r *ServiceRegistry) InstallAllGRPCServices(srv bs.Server, g *grpc.Server) { prefix := srv.Name() for name, builder := range r.builders { serviceName := createServiceName(prefix, name) @@ -77,7 +76,7 @@ func (r *ServiceRegistry) InstallAllGRPCServices(srv *server.Server, g *grpc.Ser } // InstallAllRESTHandler installs all registered REST services. -func (r *ServiceRegistry) InstallAllRESTHandler(srv *server.Server, h map[string]http.Handler) { +func (r *ServiceRegistry) InstallAllRESTHandler(srv bs.Server, h map[string]http.Handler) { prefix := srv.Name() for name, builder := range r.builders { serviceName := createServiceName(prefix, name) @@ -97,9 +96,3 @@ func (r *ServiceRegistry) InstallAllRESTHandler(srv *server.Server, h map[string func (r ServiceRegistry) RegisterService(name string, service ServiceBuilder) { r.builders[name] = service } - -func init() { - server.NewServiceRegistry = func() server.ServiceRegistry { - return ServerServiceRegistry - } -} diff --git a/pkg/mcs/resource_manager/server/apis/v1/api.go b/pkg/mcs/resource_manager/server/apis/v1/api.go index 08597cbb278..ec0af1941ed 100644 --- a/pkg/mcs/resource_manager/server/apis/v1/api.go +++ b/pkg/mcs/resource_manager/server/apis/v1/api.go @@ -23,14 +23,14 @@ import ( "github.com/gin-gonic/gin" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" rmserver "github.com/tikv/pd/pkg/mcs/resource_manager/server" - "github.com/tikv/pd/server" + "github.com/tikv/pd/pkg/utils/apiutil" ) // APIPathPrefix is the prefix of the API path. const APIPathPrefix = "/resource-manager/api/v1/" var ( - apiServiceGroup = server.APIServiceGroup{ + apiServiceGroup = apiutil.APIServiceGroup{ Name: "resource-manager", Version: "v1", IsCore: false, @@ -39,7 +39,7 @@ var ( ) func init() { - rmserver.SetUpRestHandler = func(srv *rmserver.Service) (http.Handler, server.APIServiceGroup) { + rmserver.SetUpRestHandler = func(srv *rmserver.Service) (http.Handler, apiutil.APIServiceGroup) { s := NewService(srv) return s.handler(), apiServiceGroup } diff --git a/pkg/mcs/resource_manager/server/grpc_service.go b/pkg/mcs/resource_manager/server/grpc_service.go index 52d41730643..cba2b3f3473 100644 --- a/pkg/mcs/resource_manager/server/grpc_service.go +++ b/pkg/mcs/resource_manager/server/grpc_service.go @@ -23,8 +23,9 @@ import ( "github.com/pingcap/errors" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/log" + bs "github.com/tikv/pd/pkg/basic_server" "github.com/tikv/pd/pkg/mcs/registry" - "github.com/tikv/pd/server" + "github.com/tikv/pd/pkg/utils/apiutil" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -39,8 +40,8 @@ var ( var _ rmpb.ResourceManagerServer = (*Service)(nil) // SetUpRestHandler is a hook to sets up the REST service. -var SetUpRestHandler = func(srv *Service) (http.Handler, server.APIServiceGroup) { - return dummyRestService{}, server.APIServiceGroup{} +var SetUpRestHandler = func(srv *Service) (http.Handler, apiutil.APIServiceGroup) { + return dummyRestService{}, apiutil.APIServiceGroup{} } type dummyRestService struct{} @@ -58,7 +59,7 @@ type Service struct { } // NewService creates a new resource manager service. -func NewService(svr *server.Server) registry.RegistrableService { +func NewService(svr bs.Server) registry.RegistrableService { manager := NewManager(svr) return &Service{ @@ -75,7 +76,7 @@ func (s *Service) RegisterGRPCService(g *grpc.Server) { // RegisterRESTHandler registers the service to REST server. func (s *Service) RegisterRESTHandler(userDefineHandlers map[string]http.Handler) { handler, group := SetUpRestHandler(s) - server.RegisterUserDefinedHandlers(userDefineHandlers, &group, handler) + apiutil.RegisterUserDefinedHandlers(userDefineHandlers, &group, handler) } // GetManager returns the resource manager. diff --git a/pkg/mcs/resource_manager/server/manager.go b/pkg/mcs/resource_manager/server/manager.go index 6de093f22f8..867fe27f088 100644 --- a/pkg/mcs/resource_manager/server/manager.go +++ b/pkg/mcs/resource_manager/server/manager.go @@ -23,10 +23,10 @@ import ( "github.com/pingcap/errors" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/log" + bs "github.com/tikv/pd/pkg/basic_server" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" - "github.com/tikv/pd/server" "go.uber.org/zap" ) @@ -47,7 +47,7 @@ type Manager struct { } // NewManager returns a new Manager. -func NewManager(srv *server.Server) *Manager { +func NewManager(srv bs.Server) *Manager { m := &Manager{ member: &member.Member{}, groups: make(map[string]*ResourceGroup), diff --git a/pkg/swaggerserver/swaggerserver.go b/pkg/swaggerserver/swaggerserver.go index 3c2326ac115..778844adbde 100644 --- a/pkg/swaggerserver/swaggerserver.go +++ b/pkg/swaggerserver/swaggerserver.go @@ -18,13 +18,14 @@ import ( "context" "net/http" + "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" ) const swaggerPrefix = "/swagger/" var ( - swaggerServiceGroup = server.APIServiceGroup{ + swaggerServiceGroup = apiutil.APIServiceGroup{ Name: "swagger", Version: "v1", IsCore: false, @@ -33,7 +34,7 @@ var ( ) // NewHandler creates a HTTP handler for Swagger. -func NewHandler(context.Context, *server.Server) (http.Handler, server.APIServiceGroup, error) { +func NewHandler(context.Context, *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { swaggerHandler := http.NewServeMux() swaggerHandler.Handle(swaggerPrefix, handler()) return swaggerHandler, swaggerServiceGroup, nil diff --git a/pkg/utils/apiutil/apiutil.go b/pkg/utils/apiutil/apiutil.go index 1989df33b2c..4b393eccf37 100644 --- a/pkg/utils/apiutil/apiutil.go +++ b/pkg/utils/apiutil/apiutil.go @@ -23,6 +23,7 @@ import ( "net" "net/http" "net/url" + "path" "strconv" "strings" @@ -32,6 +33,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" "github.com/unrolled/render" + "go.uber.org/zap" ) var ( @@ -322,3 +324,46 @@ func ReadJSONRespondError(rd *render.Render, w http.ResponseWriter, body io.Read ErrorResp(rd, w, errCode) return err } + +const ( + // CorePath the core group, is at REST path `/pd/api/v1`. + CorePath = "/pd/api/v1" + // ExtensionsPath the named groups are REST at `/pd/apis/{GROUP_NAME}/{Version}`. + ExtensionsPath = "/pd/apis" +) + +// APIServiceGroup used to register the HTTP REST API. +type APIServiceGroup struct { + Name string + Version string + IsCore bool + PathPrefix string +} + +// Path returns the path of the service. +func (sg *APIServiceGroup) Path() string { + if len(sg.PathPrefix) > 0 { + return sg.PathPrefix + } + if sg.IsCore { + return CorePath + } + if len(sg.Name) > 0 && len(sg.Version) > 0 { + return path.Join(ExtensionsPath, sg.Name, sg.Version) + } + return "" +} + +// RegisterUserDefinedHandlers register the user defined handlers. +func RegisterUserDefinedHandlers(registerMap map[string]http.Handler, group *APIServiceGroup, handler http.Handler) error { + pathPrefix := group.Path() + if _, ok := registerMap[pathPrefix]; ok { + return errs.ErrServiceRegistered.FastGenByArgs(pathPrefix) + } + if len(pathPrefix) == 0 { + return errs.ErrAPIInformationInvalid.FastGenByArgs(group.Name, group.Version) + } + registerMap[pathPrefix] = handler + log.Info("register REST path", zap.String("path", pathPrefix)) + return nil +} diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go index 29a98aaa63a..0e6f1f38ccb 100644 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/slice" + "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" "github.com/urfave/negroni" "go.uber.org/zap" @@ -40,11 +41,11 @@ const ( type runtimeServiceValidator struct { s *server.Server - group server.APIServiceGroup + group apiutil.APIServiceGroup } // NewRuntimeServiceValidator checks if the path is invalid. -func NewRuntimeServiceValidator(s *server.Server, group server.APIServiceGroup) negroni.Handler { +func NewRuntimeServiceValidator(s *server.Server, group apiutil.APIServiceGroup) negroni.Handler { return &runtimeServiceValidator{s: s, group: group} } @@ -58,7 +59,7 @@ func (h *runtimeServiceValidator) ServeHTTP(w http.ResponseWriter, r *http.Reque } // IsServiceAllowed checks the service through the path. -func IsServiceAllowed(s *server.Server, group server.APIServiceGroup) bool { +func IsServiceAllowed(s *server.Server, group apiutil.APIServiceGroup) bool { // for core path if group.IsCore { return true diff --git a/server/api/server.go b/server/api/server.go index f44a352648a..6d0f5a5aa13 100644 --- a/server/api/server.go +++ b/server/api/server.go @@ -19,6 +19,7 @@ import ( "net/http" "github.com/gorilla/mux" + "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/apiutil/serverapi" "github.com/tikv/pd/server" "github.com/urfave/negroni" @@ -27,8 +28,8 @@ import ( const apiPrefix = "/pd" // NewHandler creates a HTTP handler for API. -func NewHandler(ctx context.Context, svr *server.Server) (http.Handler, server.APIServiceGroup, error) { - group := server.APIServiceGroup{ +func NewHandler(ctx context.Context, svr *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { + group := apiutil.APIServiceGroup{ Name: "core", IsCore: true, } diff --git a/server/apiv2/router.go b/server/apiv2/router.go index 8efc42cf7f0..5307b2b5c0f 100644 --- a/server/apiv2/router.go +++ b/server/apiv2/router.go @@ -21,6 +21,7 @@ import ( "github.com/gin-gonic/gin" "github.com/joho/godotenv" + "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/apiv2/handlers" "github.com/tikv/pd/server/apiv2/middlewares" @@ -28,7 +29,7 @@ import ( var once sync.Once -var group = server.APIServiceGroup{ +var group = apiutil.APIServiceGroup{ Name: "core", IsCore: true, Version: "v2", @@ -47,7 +48,7 @@ const apiV2Prefix = "/pd/api/v2/" // @license.name Apache 2.0 // @license.url http://www.apache.org/licenses/LICENSE-2.0.html // @BasePath /pd/api/v2 -func NewV2Handler(_ context.Context, svr *server.Server) (http.Handler, server.APIServiceGroup, error) { +func NewV2Handler(_ context.Context, svr *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { once.Do(func() { // See https://github.com/pingcap/tidb-dashboard/blob/f8ecb64e3d63f4ed91c3dca7a04362418ade01d8/pkg/apiserver/apiserver.go#L84 // These global modification will be effective only for the first invoke. diff --git a/server/server.go b/server/server.go index fa21790a6f2..19259c2cac8 100644 --- a/server/server.go +++ b/server/server.go @@ -44,6 +44,7 @@ import ( "github.com/tikv/pd/pkg/encryption" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/id" + "github.com/tikv/pd/pkg/mcs/registry" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/storage/endpoint" @@ -183,14 +184,7 @@ type Server struct { } // HandlerBuilder builds a server HTTP handler. -type HandlerBuilder func(context.Context, *Server) (http.Handler, APIServiceGroup, error) - -const ( - // CorePath the core group, is at REST path `/pd/api/v1`. - CorePath = "/pd/api/v1" - // ExtensionsPath the named groups are REST at `/pd/apis/{GROUP_NAME}/{Version}`. - ExtensionsPath = "/pd/apis" -) +type HandlerBuilder func(context.Context, *Server) (http.Handler, apiutil.APIServiceGroup, error) // CreateServer creates the UNINITIALIZED pd server with given configuration. func CreateServer(ctx context.Context, cfg *config.Config, legacyServiceBuilders ...HandlerBuilder) (*Server, error) { @@ -233,7 +227,7 @@ func CreateServer(ctx context.Context, cfg *config.Config, legacyServiceBuilders etcdCfg.UserHandlers = userHandlers } // New way to register services. - registry := NewServiceRegistry() + registry := registry.ServerServiceRegistry // Register the micro services REST path. registry.InstallAllRESTHandler(s, etcdCfg.UserHandlers) diff --git a/server/server_test.go b/server/server_test.go index 954710e0f9a..a9b00077f1c 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -170,7 +170,7 @@ func (suite *leaderServerTestSuite) TestCheckClusterID() { } func (suite *leaderServerTestSuite) TestRegisterServerHandler() { - mokHandler := func(ctx context.Context, s *Server) (http.Handler, APIServiceGroup, error) { + mokHandler := func(ctx context.Context, s *Server) (http.Handler, apiutil.APIServiceGroup, error) { mux := http.NewServeMux() mux.HandleFunc("/pd/apis/mok/v1/hello", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "Hello World") @@ -178,7 +178,7 @@ func (suite *leaderServerTestSuite) TestRegisterServerHandler() { clientIP := apiutil.GetIPAddrFromHTTPRequest(r) suite.Equal("127.0.0.1", clientIP) }) - info := APIServiceGroup{ + info := apiutil.APIServiceGroup{ Name: "mok", Version: "v1", } @@ -209,7 +209,7 @@ func (suite *leaderServerTestSuite) TestRegisterServerHandler() { } func (suite *leaderServerTestSuite) TestSourceIpForHeaderForwarded() { - mokHandler := func(ctx context.Context, s *Server) (http.Handler, APIServiceGroup, error) { + mokHandler := func(ctx context.Context, s *Server) (http.Handler, apiutil.APIServiceGroup, error) { mux := http.NewServeMux() mux.HandleFunc("/pd/apis/mok/v1/hello", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "Hello World") @@ -217,7 +217,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderForwarded() { clientIP := apiutil.GetIPAddrFromHTTPRequest(r) suite.Equal("127.0.0.2", clientIP) }) - info := APIServiceGroup{ + info := apiutil.APIServiceGroup{ Name: "mok", Version: "v1", } @@ -252,7 +252,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderForwarded() { } func (suite *leaderServerTestSuite) TestSourceIpForHeaderXReal() { - mokHandler := func(ctx context.Context, s *Server) (http.Handler, APIServiceGroup, error) { + mokHandler := func(ctx context.Context, s *Server) (http.Handler, apiutil.APIServiceGroup, error) { mux := http.NewServeMux() mux.HandleFunc("/pd/apis/mok/v1/hello", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "Hello World") @@ -260,7 +260,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderXReal() { clientIP := apiutil.GetIPAddrFromHTTPRequest(r) suite.Equal("127.0.0.2", clientIP) }) - info := APIServiceGroup{ + info := apiutil.APIServiceGroup{ Name: "mok", Version: "v1", } @@ -295,7 +295,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderXReal() { } func (suite *leaderServerTestSuite) TestSourceIpForHeaderBoth() { - mokHandler := func(ctx context.Context, s *Server) (http.Handler, APIServiceGroup, error) { + mokHandler := func(ctx context.Context, s *Server) (http.Handler, apiutil.APIServiceGroup, error) { mux := http.NewServeMux() mux.HandleFunc("/pd/apis/mok/v1/hello", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "Hello World") @@ -303,7 +303,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderBoth() { clientIP := apiutil.GetIPAddrFromHTTPRequest(r) suite.Equal("127.0.0.2", clientIP) }) - info := APIServiceGroup{ + info := apiutil.APIServiceGroup{ Name: "mok", Version: "v1", } diff --git a/server/util.go b/server/util.go index 33cbea29b63..12feae46bf5 100644 --- a/server/util.go +++ b/server/util.go @@ -19,7 +19,6 @@ import ( "fmt" "math/rand" "net/http" - "path" "strings" "time" @@ -28,6 +27,7 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" @@ -35,7 +35,6 @@ import ( "github.com/urfave/negroni" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" - "google.golang.org/grpc" ) const ( @@ -164,63 +163,6 @@ func checkBootstrapRequest(clusterID uint64, req *pdpb.BootstrapRequest) error { return nil } -/// REST API and GRPC services relative Utils. - -// ServiceRegistry used to install the registered services, including gRPC and HTTP API. -type ServiceRegistry interface { - InstallAllGRPCServices(srv *Server, g *grpc.Server) - InstallAllRESTHandler(srv *Server, userDefineHandler map[string]http.Handler) -} - -// NewServiceRegistry is a hook for mcs code which implements the micro service. -var NewServiceRegistry = func() ServiceRegistry { - return dummyServiceRegistry{} -} - -type dummyServiceRegistry struct{} - -func (d dummyServiceRegistry) InstallAllGRPCServices(srv *Server, g *grpc.Server) { -} - -func (d dummyServiceRegistry) InstallAllRESTHandler(srv *Server, userDefineHandler map[string]http.Handler) { -} - -// APIServiceGroup used to register the HTTP REST API. -type APIServiceGroup struct { - Name string - Version string - IsCore bool - PathPrefix string -} - -// Path returns the path of the service. -func (sg *APIServiceGroup) Path() string { - if len(sg.PathPrefix) > 0 { - return sg.PathPrefix - } - if sg.IsCore { - return CorePath - } - if len(sg.Name) > 0 && len(sg.Version) > 0 { - return path.Join(ExtensionsPath, sg.Name, sg.Version) - } - return "" -} - -// RegisterUserDefinedHandlers register the user defined handlers. -func RegisterUserDefinedHandlers(registerMap map[string]http.Handler, group *APIServiceGroup, handler http.Handler) error { - pathPrefix := group.Path() - if _, ok := registerMap[pathPrefix]; ok { - return errs.ErrServiceRegistered.FastGenByArgs(pathPrefix) - } - if len(pathPrefix) == 0 { - return errs.ErrAPIInformationInvalid.FastGenByArgs(group.Name, group.Version) - } - registerMap[pathPrefix] = handler - log.Info("register REST path", zap.String("path", pathPrefix)) - return nil -} - func combineBuilderServerHTTPService(ctx context.Context, svr *Server, serviceBuilders ...HandlerBuilder) (map[string]http.Handler, error) { userHandlers := make(map[string]http.Handler) registerMap := make(map[string]http.Handler) @@ -239,16 +181,16 @@ func combineBuilderServerHTTPService(ctx context.Context, svr *Server, serviceBu return nil, errs.ErrAPIInformationInvalid.FastGenByArgs(info.Name, info.Version) } - if err := RegisterUserDefinedHandlers(registerMap, &info, handler); err != nil { + if err := apiutil.RegisterUserDefinedHandlers(registerMap, &info, handler); err != nil { return nil, err } } // Combine the pd service to the router. the extension service will be added to the userHandlers. for pathPrefix, handler := range registerMap { - if strings.Contains(pathPrefix, CorePath) || strings.Contains(pathPrefix, ExtensionsPath) { + if strings.Contains(pathPrefix, apiutil.CorePath) || strings.Contains(pathPrefix, apiutil.ExtensionsPath) { router.PathPrefix(pathPrefix).Handler(handler) - if pathPrefix == CorePath { + if pathPrefix == apiutil.CorePath { // Deprecated router.Path("/pd/health").Handler(handler) // Deprecated diff --git a/tests/pdctl/global_test.go b/tests/pdctl/global_test.go index a57b929c4a4..2575acfb450 100644 --- a/tests/pdctl/global_test.go +++ b/tests/pdctl/global_test.go @@ -32,7 +32,7 @@ import ( func TestSendAndGetComponent(t *testing.T) { re := require.New(t) - handler := func(ctx context.Context, s *server.Server) (http.Handler, server.APIServiceGroup, error) { + handler := func(ctx context.Context, s *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { mux := http.NewServeMux() mux.HandleFunc("/pd/api/v1/health", func(w http.ResponseWriter, r *http.Request) { component := apiutil.GetComponentNameOnHTTP(r) @@ -43,7 +43,7 @@ func TestSendAndGetComponent(t *testing.T) { re.Equal("pdctl", component) fmt.Fprint(w, component) }) - info := server.APIServiceGroup{ + info := apiutil.APIServiceGroup{ IsCore: true, } return mux, info, nil diff --git a/tests/registry/registry_test.go b/tests/registry/registry_test.go index 0e6d84c9fea..296d54972a6 100644 --- a/tests/registry/registry_test.go +++ b/tests/registry/registry_test.go @@ -8,9 +8,10 @@ import ( "testing" "github.com/stretchr/testify/require" + bs "github.com/tikv/pd/pkg/basic_server" "github.com/tikv/pd/pkg/mcs/registry" + "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/testutil" - "github.com/tikv/pd/server" "github.com/tikv/pd/tests" "go.uber.org/goleak" "google.golang.org/grpc" @@ -30,7 +31,7 @@ func (t *testServiceRegistry) RegisterGRPCService(g *grpc.Server) { } func (t *testServiceRegistry) RegisterRESTHandler(userDefineHandlers map[string]http.Handler) { - group := server.APIServiceGroup{ + group := apiutil.APIServiceGroup{ Name: "my-http-service", Version: "v1alpha1", IsCore: false, @@ -40,22 +41,15 @@ func (t *testServiceRegistry) RegisterRESTHandler(userDefineHandlers map[string] w.WriteHeader(http.StatusOK) w.Write([]byte("Hello World!")) }) - server.RegisterUserDefinedHandlers(userDefineHandlers, &group, handler) + apiutil.RegisterUserDefinedHandlers(userDefineHandlers, &group, handler) } -func newTestServiceRegistry(_ *server.Server) registry.RegistrableService { +func newTestServiceRegistry(_ bs.Server) registry.RegistrableService { return &testServiceRegistry{} } -func install(register *registry.ServiceRegistry) { - register.RegisterService("test", newTestServiceRegistry) - server.NewServiceRegistry = func() server.ServiceRegistry { - return register - } -} - func TestRegistryService(t *testing.T) { - install(registry.ServerServiceRegistry) + registry.ServerServiceRegistry.RegisterService("test", newTestServiceRegistry) re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() From 7fca2da5706173f6edc74625fbbdc29769e08774 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Tue, 7 Feb 2023 23:55:09 +0800 Subject: [PATCH 02/14] rm install Signed-off-by: lhy1024 --- cmd/pd-server/main.go | 4 ---- server/server.go | 4 +++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/pd-server/main.go b/cmd/pd-server/main.go index 3292439b370..bc1f2084a65 100644 --- a/cmd/pd-server/main.go +++ b/cmd/pd-server/main.go @@ -40,10 +40,6 @@ import ( // Register schedulers. _ "github.com/tikv/pd/server/schedulers" - - // Register Service - _ "github.com/tikv/pd/pkg/mcs/registry" - _ "github.com/tikv/pd/pkg/mcs/resource_manager/server/install" ) func main() { diff --git a/server/server.go b/server/server.go index 19259c2cac8..70f3a48369b 100644 --- a/server/server.go +++ b/server/server.go @@ -45,6 +45,8 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/id" "github.com/tikv/pd/pkg/mcs/registry" + rm_server "github.com/tikv/pd/pkg/mcs/resource_manager/server" + _ "github.com/tikv/pd/pkg/mcs/resource_manager/server/apis/v1" // init API group "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/storage/endpoint" @@ -228,7 +230,7 @@ func CreateServer(ctx context.Context, cfg *config.Config, legacyServiceBuilders } // New way to register services. registry := registry.ServerServiceRegistry - + registry.RegisterService("ResourceManager", rm_server.NewService) // Register the micro services REST path. registry.InstallAllRESTHandler(s, etcdCfg.UserHandlers) From 1d115f09797390f801145038cc768adf7c6e03ab Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 9 Feb 2023 19:49:15 +0800 Subject: [PATCH 03/14] address comments Signed-off-by: lhy1024 Conflicts: cmd/pd-server/main.go pkg/mcs/tso/server/server.go server/server.go --- cmd/pd-server/main.go | 9 +- .../basic_server.go | 3 +- pkg/mcs/registry/registry.go | 2 +- .../resource_manager/server/grpc_service.go | 2 +- pkg/mcs/resource_manager/server/manager.go | 2 +- pkg/mcs/tso/server/server.go | 119 ++++++++++++++++++ server/server.go | 2 +- tests/registry/registry_test.go | 2 +- 8 files changed, 130 insertions(+), 11 deletions(-) rename pkg/{basic_server => basicserver}/basic_server.go (93%) create mode 100644 pkg/mcs/tso/server/server.go diff --git a/cmd/pd-server/main.go b/cmd/pd-server/main.go index bc1f2084a65..21e6d5ca0e0 100644 --- a/cmd/pd-server/main.go +++ b/cmd/pd-server/main.go @@ -25,7 +25,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/tikv/pd/pkg/autoscaling" - basicsvr "github.com/tikv/pd/pkg/basic_server" + bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/dashboard" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/swaggerserver" @@ -36,10 +36,8 @@ import ( "github.com/tikv/pd/server/apiv2" "github.com/tikv/pd/server/config" "github.com/tikv/pd/server/join" + "github.com/tikv/pd/server/schedulers" "go.uber.org/zap" - - // Register schedulers. - _ "github.com/tikv/pd/server/schedulers" ) func main() { @@ -74,7 +72,8 @@ func main() { } } -func createServerWrapper(args []string) (context.Context, context.CancelFunc, basicsvr.Server) { +func createServerWrapper(args []string) (context.Context, context.CancelFunc, bs.Server) { + schedulers.Register() cfg := config.NewConfig() err := cfg.Parse(args) diff --git a/pkg/basic_server/basic_server.go b/pkg/basicserver/basic_server.go similarity index 93% rename from pkg/basic_server/basic_server.go rename to pkg/basicserver/basic_server.go index d9ca2028018..ab72704429b 100644 --- a/pkg/basic_server/basic_server.go +++ b/pkg/basicserver/basic_server.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package basicsvr +package server import ( "context" @@ -38,6 +38,7 @@ type Server interface { GetHTTPClient() *http.Client // AddStartCallback adds a callback in the startServer phase. AddStartCallback(callbacks ...func()) + // TODO: replace these two methods with `primary` function without etcd server dependency. // GetMember returns the member information. GetMember() *member.Member // AddLeaderCallback adds a callback in the leader campaign phase. diff --git a/pkg/mcs/registry/registry.go b/pkg/mcs/registry/registry.go index e3f299a6c98..857f362a8fc 100644 --- a/pkg/mcs/registry/registry.go +++ b/pkg/mcs/registry/registry.go @@ -21,7 +21,7 @@ import ( "net/http" "github.com/pingcap/log" - bs "github.com/tikv/pd/pkg/basic_server" + bs "github.com/tikv/pd/pkg/basicserver" "go.uber.org/zap" "google.golang.org/grpc" ) diff --git a/pkg/mcs/resource_manager/server/grpc_service.go b/pkg/mcs/resource_manager/server/grpc_service.go index cba2b3f3473..363a79fe841 100644 --- a/pkg/mcs/resource_manager/server/grpc_service.go +++ b/pkg/mcs/resource_manager/server/grpc_service.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/errors" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/log" - bs "github.com/tikv/pd/pkg/basic_server" + bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/mcs/registry" "github.com/tikv/pd/pkg/utils/apiutil" "go.uber.org/zap" diff --git a/pkg/mcs/resource_manager/server/manager.go b/pkg/mcs/resource_manager/server/manager.go index 867fe27f088..0e3e6583b3e 100644 --- a/pkg/mcs/resource_manager/server/manager.go +++ b/pkg/mcs/resource_manager/server/manager.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/errors" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/log" - bs "github.com/tikv/pd/pkg/basic_server" + bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" diff --git a/pkg/mcs/tso/server/server.go b/pkg/mcs/tso/server/server.go new file mode 100644 index 00000000000..4d336a4ef5b --- /dev/null +++ b/pkg/mcs/tso/server/server.go @@ -0,0 +1,119 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "flag" + "net/http" + "os" + + grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/pingcap/errors" + "github.com/pingcap/log" + bs "github.com/tikv/pd/pkg/basicserver" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/tso" + "github.com/tikv/pd/pkg/utils/logutil" + "github.com/tikv/pd/pkg/utils/metricutil" + "go.etcd.io/etcd/clientv3" +) + +// Server is the TSO server, and it implements bs.Server. +// nolint +type Server struct { + ctx context.Context +} + +// TODO: Implement the following methods defined in bs.Server + +// Name returns the unique etcd Name for this server in etcd cluster. +func (s *Server) Name() string { + return "" +} + +// Context returns the context of server. +func (s *Server) Context() context.Context { + return s.ctx +} + +// Run runs the pd server. +func (s *Server) Run() error { + return nil +} + +// Close closes the server. +func (s *Server) Close() { +} + +// GetClient returns builtin etcd client. +func (s *Server) GetClient() *clientv3.Client { + return nil +} + +// GetHTTPClient returns builtin http client. +func (s *Server) GetHTTPClient() *http.Client { + return nil +} + +// CreateServerWrapper encapsulates the configuration/log/metrics initialization and create the server +func CreateServerWrapper(args []string) (context.Context, context.CancelFunc, bs.Server) { + cfg := tso.NewConfig() + err := cfg.Parse(os.Args[1:]) + + if cfg.Version { + printVersionInfo() + exit(0) + } + + defer logutil.LogPanic() + + switch errors.Cause(err) { + case nil: + case flag.ErrHelp: + exit(0) + default: + log.Fatal("parse cmd flags error", errs.ZapError(err)) + } + + if cfg.ConfigCheck { + printConfigCheckMsg(cfg) + exit(0) + } + + // TODO: Initialize logger + + // TODO: Make it configurable if it has big impact on performance. + grpcprometheus.EnableHandlingTimeHistogram() + + metricutil.Push(&cfg.Metric) + + // TODO: Create the server + + return nil, nil, nil +} + +// TODO: implement it +func printVersionInfo() { +} + +// TODO: implement it +func printConfigCheckMsg(cfg *tso.Config) { +} + +func exit(code int) { + log.Sync() + os.Exit(code) +} diff --git a/server/server.go b/server/server.go index 70f3a48369b..920a9639ddd 100644 --- a/server/server.go +++ b/server/server.go @@ -102,7 +102,7 @@ var ( etcdCommittedIndexGauge = etcdStateGauge.WithLabelValues("committedIndex") ) -// Server is the pd server. +// Server is the pd server. It implements bs.Server // nolint type Server struct { diagnosticspb.DiagnosticsServer diff --git a/tests/registry/registry_test.go b/tests/registry/registry_test.go index 296d54972a6..4e9d21158bd 100644 --- a/tests/registry/registry_test.go +++ b/tests/registry/registry_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/stretchr/testify/require" - bs "github.com/tikv/pd/pkg/basic_server" + bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/mcs/registry" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/testutil" From 4308fa2ed0c97a0d064756eab862b47798e7dae5 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 9 Feb 2023 19:56:30 +0800 Subject: [PATCH 04/14] fix conflict Signed-off-by: lhy1024 --- pkg/mcs/tso/server/server.go | 20 -------------------- server/server.go | 1 - 2 files changed, 21 deletions(-) diff --git a/pkg/mcs/tso/server/server.go b/pkg/mcs/tso/server/server.go index 9525832c028..4d336a4ef5b 100644 --- a/pkg/mcs/tso/server/server.go +++ b/pkg/mcs/tso/server/server.go @@ -23,11 +23,7 @@ import ( grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/pingcap/errors" "github.com/pingcap/log" -<<<<<<< HEAD bs "github.com/tikv/pd/pkg/basicserver" -======= - basicsvr "github.com/tikv/pd/pkg/basic_server" ->>>>>>> 2be26ff534692fb3de17b7ce69cff46a67359fc3 "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/logutil" @@ -35,25 +31,13 @@ import ( "go.etcd.io/etcd/clientv3" ) -<<<<<<< HEAD // Server is the TSO server, and it implements bs.Server. -======= -// If server doesn't implement all methods of basicsvr.Server, this line will result in a clear -// error message like "*Server does not implement basicsvr.Server (missing Method method)" -var _ basicsvr.Server = (*Server)(nil) - -// Server is the TSO server, and it implements basicsvr.Server. ->>>>>>> 2be26ff534692fb3de17b7ce69cff46a67359fc3 // nolint type Server struct { ctx context.Context } -<<<<<<< HEAD // TODO: Implement the following methods defined in bs.Server -======= -// TODO: Implement the following methods defined in basicsvr.Server ->>>>>>> 2be26ff534692fb3de17b7ce69cff46a67359fc3 // Name returns the unique etcd Name for this server in etcd cluster. func (s *Server) Name() string { @@ -85,11 +69,7 @@ func (s *Server) GetHTTPClient() *http.Client { } // CreateServerWrapper encapsulates the configuration/log/metrics initialization and create the server -<<<<<<< HEAD func CreateServerWrapper(args []string) (context.Context, context.CancelFunc, bs.Server) { -======= -func CreateServerWrapper(args []string) (context.Context, context.CancelFunc, basicsvr.Server) { ->>>>>>> 2be26ff534692fb3de17b7ce69cff46a67359fc3 cfg := tso.NewConfig() err := cfg.Parse(os.Args[1:]) diff --git a/server/server.go b/server/server.go index e5fad672474..acc802094af 100644 --- a/server/server.go +++ b/server/server.go @@ -40,7 +40,6 @@ import ( "github.com/pingcap/log" "github.com/pingcap/sysutil" "github.com/tikv/pd/pkg/audit" - basicsvr "github.com/tikv/pd/pkg/basic_server" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/encryption" "github.com/tikv/pd/pkg/errs" From e9b82423f8f66f6e10dd4b8f12495a1df90e75f5 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 9 Feb 2023 23:45:33 +0800 Subject: [PATCH 05/14] fix handler nil Signed-off-by: lhy1024 --- pkg/mcs/registry/registry.go | 5 ++- server/server.go | 2 +- server/server_test.go | 64 ++++++++---------------------- tests/server/member/member_test.go | 3 +- 4 files changed, 22 insertions(+), 52 deletions(-) diff --git a/pkg/mcs/registry/registry.go b/pkg/mcs/registry/registry.go index 857f362a8fc..e3aee1552b0 100644 --- a/pkg/mcs/registry/registry.go +++ b/pkg/mcs/registry/registry.go @@ -28,7 +28,7 @@ import ( var ( // ServerServiceRegistry is the global grpc service registry. - ServerServiceRegistry = newServiceRegistry() + ServerServiceRegistry = NewServerServiceRegistry() ) // ServiceBuilder is a function that creates a grpc service. @@ -47,7 +47,8 @@ type ServiceRegistry struct { services map[string]RegistrableService } -func newServiceRegistry() *ServiceRegistry { +// NewServerServiceRegistry creates a new ServiceRegistry. +func NewServerServiceRegistry() *ServiceRegistry { return &ServiceRegistry{ builders: make(map[string]ServiceBuilder), services: make(map[string]RegistrableService), diff --git a/server/server.go b/server/server.go index acc802094af..e72d7c5924f 100644 --- a/server/server.go +++ b/server/server.go @@ -229,7 +229,7 @@ func CreateServer(ctx context.Context, cfg *config.Config, legacyServiceBuilders etcdCfg.UserHandlers = userHandlers } // New way to register services. - registry := registry.ServerServiceRegistry + registry := registry.NewServerServiceRegistry() registry.RegisterService("ResourceManager", rm_server.NewService) // Register the micro services REST path. registry.InstallAllRESTHandler(s, etcdCfg.UserHandlers) diff --git a/server/server_test.go b/server/server_test.go index a9b00077f1c..e37128b088a 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -60,7 +60,8 @@ func (suite *leaderServerTestSuite) SetupSuite() { cfg := cfgs[i] go func() { - svr, err := CreateServer(suite.ctx, cfg) + mokHandler := suite.createMokHandler("127.0.0.1") + svr, err := CreateServer(suite.ctx, cfg, mokHandler) suite.NoError(err) err = svr.Run() suite.NoError(err) @@ -89,7 +90,8 @@ func (suite *leaderServerTestSuite) newTestServersWithCfgs(ctx context.Context, ch := make(chan *Server) for _, cfg := range cfgs { go func(cfg *config.Config) { - svr, err := CreateServer(ctx, cfg) + mokHandler := suite.createMokHandler("127.0.0.1") + svr, err := CreateServer(ctx, cfg, mokHandler) // prevent blocking if Asserts fails failed := true defer func() { @@ -154,7 +156,8 @@ func (suite *leaderServerTestSuite) TestCheckClusterID() { // Start previous cluster, expect an error. cfgA.InitialCluster = originInitial - svr, err := CreateServer(ctx, cfgA) + mokHandler := suite.createMokHandler("127.0.0.1") + svr, err := CreateServer(ctx, cfgA, mokHandler) suite.NoError(err) etcd, err := embed.StartEtcd(svr.etcdCfg) @@ -169,14 +172,14 @@ func (suite *leaderServerTestSuite) TestCheckClusterID() { testutil.CleanServer(cfgA.DataDir) } -func (suite *leaderServerTestSuite) TestRegisterServerHandler() { - mokHandler := func(ctx context.Context, s *Server) (http.Handler, apiutil.APIServiceGroup, error) { +func (suite *leaderServerTestSuite) createMokHandler(ip string) HandlerBuilder { + return func(ctx context.Context, s *Server) (http.Handler, apiutil.APIServiceGroup, error) { mux := http.NewServeMux() mux.HandleFunc("/pd/apis/mok/v1/hello", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "Hello World") // test getting ip clientIP := apiutil.GetIPAddrFromHTTPRequest(r) - suite.Equal("127.0.0.1", clientIP) + suite.Equal(ip, clientIP) }) info := apiutil.APIServiceGroup{ Name: "mok", @@ -184,8 +187,12 @@ func (suite *leaderServerTestSuite) TestRegisterServerHandler() { } return mux, info, nil } +} + +func (suite *leaderServerTestSuite) TestRegisterServerHandler() { cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) ctx, cancel := context.WithCancel(context.Background()) + mokHandler := suite.createMokHandler("127.0.0.1") svr, err := CreateServer(ctx, cfg, mokHandler) suite.NoError(err) _, err = CreateServer(ctx, cfg, mokHandler, mokHandler) @@ -209,20 +216,7 @@ func (suite *leaderServerTestSuite) TestRegisterServerHandler() { } func (suite *leaderServerTestSuite) TestSourceIpForHeaderForwarded() { - mokHandler := func(ctx context.Context, s *Server) (http.Handler, apiutil.APIServiceGroup, error) { - mux := http.NewServeMux() - mux.HandleFunc("/pd/apis/mok/v1/hello", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, "Hello World") - // test getting ip - clientIP := apiutil.GetIPAddrFromHTTPRequest(r) - suite.Equal("127.0.0.2", clientIP) - }) - info := apiutil.APIServiceGroup{ - Name: "mok", - Version: "v1", - } - return mux, info, nil - } + mokHandler := suite.createMokHandler("127.0.0.2") cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) ctx, cancel := context.WithCancel(context.Background()) svr, err := CreateServer(ctx, cfg, mokHandler) @@ -252,20 +246,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderForwarded() { } func (suite *leaderServerTestSuite) TestSourceIpForHeaderXReal() { - mokHandler := func(ctx context.Context, s *Server) (http.Handler, apiutil.APIServiceGroup, error) { - mux := http.NewServeMux() - mux.HandleFunc("/pd/apis/mok/v1/hello", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, "Hello World") - // test getting ip - clientIP := apiutil.GetIPAddrFromHTTPRequest(r) - suite.Equal("127.0.0.2", clientIP) - }) - info := apiutil.APIServiceGroup{ - Name: "mok", - Version: "v1", - } - return mux, info, nil - } + mokHandler := suite.createMokHandler("127.0.0.2") cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) ctx, cancel := context.WithCancel(context.Background()) svr, err := CreateServer(ctx, cfg, mokHandler) @@ -295,20 +276,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderXReal() { } func (suite *leaderServerTestSuite) TestSourceIpForHeaderBoth() { - mokHandler := func(ctx context.Context, s *Server) (http.Handler, apiutil.APIServiceGroup, error) { - mux := http.NewServeMux() - mux.HandleFunc("/pd/apis/mok/v1/hello", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, "Hello World") - // test getting ip - clientIP := apiutil.GetIPAddrFromHTTPRequest(r) - suite.Equal("127.0.0.2", clientIP) - }) - info := apiutil.APIServiceGroup{ - Name: "mok", - Version: "v1", - } - return mux, info, nil - } + mokHandler := suite.createMokHandler("127.0.0.2") cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) ctx, cancel := context.WithCancel(context.Background()) svr, err := CreateServer(ctx, cfg, mokHandler) diff --git a/tests/server/member/member_test.go b/tests/server/member/member_test.go index d4d92c193f3..27cbe7f3fec 100644 --- a/tests/server/member/member_test.go +++ b/tests/server/member/member_test.go @@ -33,6 +33,7 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" + "github.com/tikv/pd/server/api" "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" "go.uber.org/goleak" @@ -307,7 +308,7 @@ func TestGetLeader(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(1) done := make(chan bool) - svr, err := server.CreateServer(ctx, cfg) + svr, err := server.CreateServer(ctx, cfg, api.NewHandler) re.NoError(err) defer svr.Close() re.NoError(svr.Run()) From 268b24dc1843c6b998c7cef5bb242d0a941e19bb Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 10 Feb 2023 12:08:27 +0800 Subject: [PATCH 06/14] add failpoint Signed-off-by: lhy1024 --- server/server.go | 13 +++++++++---- tests/registry/registry_test.go | 5 ++++- 2 files changed, 13 insertions(+), 5 deletions(-) mode change 100644 => 100755 server/server.go diff --git a/server/server.go b/server/server.go old mode 100644 new mode 100755 index e72d7c5924f..a5cf9629edf --- a/server/server.go +++ b/server/server.go @@ -183,6 +183,8 @@ type Server struct { serviceAuditBackendLabels map[string]*audit.BackendLabels auditBackends []audit.Backend + + registry *registry.ServiceRegistry } // HandlerBuilder builds a server HTTP handler. @@ -229,10 +231,13 @@ func CreateServer(ctx context.Context, cfg *config.Config, legacyServiceBuilders etcdCfg.UserHandlers = userHandlers } // New way to register services. - registry := registry.NewServerServiceRegistry() - registry.RegisterService("ResourceManager", rm_server.NewService) + s.registry = registry.NewServerServiceRegistry() + failpoint.Inject("testRegistry", func() { + s.registry = registry.ServerServiceRegistry + }) + s.registry.RegisterService("ResourceManager", rm_server.NewService) // Register the micro services REST path. - registry.InstallAllRESTHandler(s, etcdCfg.UserHandlers) + s.registry.InstallAllRESTHandler(s, etcdCfg.UserHandlers) etcdCfg.ServiceRegister = func(gs *grpc.Server) { grpcServer := &GrpcServer{Server: s} @@ -240,7 +245,7 @@ func CreateServer(ctx context.Context, cfg *config.Config, legacyServiceBuilders keyspacepb.RegisterKeyspaceServer(gs, &KeyspaceServer{GrpcServer: grpcServer}) diagnosticspb.RegisterDiagnosticsServer(gs, s) // Register the micro services GRPC service. - registry.InstallAllGRPCServices(s, gs) + s.registry.InstallAllGRPCServices(s, gs) } s.etcdCfg = etcdCfg diff --git a/tests/registry/registry_test.go b/tests/registry/registry_test.go index 4e9d21158bd..b33f5552871 100644 --- a/tests/registry/registry_test.go +++ b/tests/registry/registry_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/pingcap/failpoint" "github.com/stretchr/testify/require" bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/mcs/registry" @@ -49,8 +50,10 @@ func newTestServiceRegistry(_ bs.Server) registry.RegistrableService { } func TestRegistryService(t *testing.T) { - registry.ServerServiceRegistry.RegisterService("test", newTestServiceRegistry) re := require.New(t) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/testRegistry", `return(true)`)) + defer re.NoError(failpoint.Disable("github.com/tikv/pd/server/testRegistry")) + registry.ServerServiceRegistry.RegisterService("test", newTestServiceRegistry) ctx, cancel := context.WithCancel(context.Background()) defer cancel() cluster, err := tests.NewTestCluster(ctx, 1) From 8e199bec5d48b4fc7049945d46c00fba8d79d550 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 10 Feb 2023 13:43:40 +0800 Subject: [PATCH 07/14] use mok Signed-off-by: lhy1024 --- server/server.go | 2 +- server/server_test.go | 32 +++++++----------------------- server/testutil.go | 20 +++++++++++++++++++ tests/registry/registry_test.go | 4 ++-- tests/server/member/member_test.go | 3 +-- 5 files changed, 31 insertions(+), 30 deletions(-) diff --git a/server/server.go b/server/server.go index a5cf9629edf..22cc955c44d 100755 --- a/server/server.go +++ b/server/server.go @@ -232,7 +232,7 @@ func CreateServer(ctx context.Context, cfg *config.Config, legacyServiceBuilders } // New way to register services. s.registry = registry.NewServerServiceRegistry() - failpoint.Inject("testRegistry", func() { + failpoint.Inject("useGlobalRegistry", func() { s.registry = registry.ServerServiceRegistry }) s.registry.RegisterService("ResourceManager", rm_server.NewService) diff --git a/server/server_test.go b/server/server_test.go index e37128b088a..d51af2500b9 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -22,7 +22,6 @@ import ( "testing" "github.com/stretchr/testify/suite" - "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/assertutil" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/testutil" @@ -60,7 +59,7 @@ func (suite *leaderServerTestSuite) SetupSuite() { cfg := cfgs[i] go func() { - mokHandler := suite.createMokHandler("127.0.0.1") + mokHandler := CreateMokHandler(suite.Require(), "127.0.0.1") svr, err := CreateServer(suite.ctx, cfg, mokHandler) suite.NoError(err) err = svr.Run() @@ -90,7 +89,7 @@ func (suite *leaderServerTestSuite) newTestServersWithCfgs(ctx context.Context, ch := make(chan *Server) for _, cfg := range cfgs { go func(cfg *config.Config) { - mokHandler := suite.createMokHandler("127.0.0.1") + mokHandler := CreateMokHandler(suite.Require(), "127.0.0.1") svr, err := CreateServer(ctx, cfg, mokHandler) // prevent blocking if Asserts fails failed := true @@ -156,7 +155,7 @@ func (suite *leaderServerTestSuite) TestCheckClusterID() { // Start previous cluster, expect an error. cfgA.InitialCluster = originInitial - mokHandler := suite.createMokHandler("127.0.0.1") + mokHandler := CreateMokHandler(suite.Require(), "127.0.0.1") svr, err := CreateServer(ctx, cfgA, mokHandler) suite.NoError(err) @@ -172,27 +171,10 @@ func (suite *leaderServerTestSuite) TestCheckClusterID() { testutil.CleanServer(cfgA.DataDir) } -func (suite *leaderServerTestSuite) createMokHandler(ip string) HandlerBuilder { - return func(ctx context.Context, s *Server) (http.Handler, apiutil.APIServiceGroup, error) { - mux := http.NewServeMux() - mux.HandleFunc("/pd/apis/mok/v1/hello", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, "Hello World") - // test getting ip - clientIP := apiutil.GetIPAddrFromHTTPRequest(r) - suite.Equal(ip, clientIP) - }) - info := apiutil.APIServiceGroup{ - Name: "mok", - Version: "v1", - } - return mux, info, nil - } -} - func (suite *leaderServerTestSuite) TestRegisterServerHandler() { cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) ctx, cancel := context.WithCancel(context.Background()) - mokHandler := suite.createMokHandler("127.0.0.1") + mokHandler := CreateMokHandler(suite.Require(), "127.0.0.1") svr, err := CreateServer(ctx, cfg, mokHandler) suite.NoError(err) _, err = CreateServer(ctx, cfg, mokHandler, mokHandler) @@ -216,7 +198,7 @@ func (suite *leaderServerTestSuite) TestRegisterServerHandler() { } func (suite *leaderServerTestSuite) TestSourceIpForHeaderForwarded() { - mokHandler := suite.createMokHandler("127.0.0.2") + mokHandler := CreateMokHandler(suite.Require(), "127.0.0.2") cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) ctx, cancel := context.WithCancel(context.Background()) svr, err := CreateServer(ctx, cfg, mokHandler) @@ -246,7 +228,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderForwarded() { } func (suite *leaderServerTestSuite) TestSourceIpForHeaderXReal() { - mokHandler := suite.createMokHandler("127.0.0.2") + mokHandler := CreateMokHandler(suite.Require(), "127.0.0.2") cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) ctx, cancel := context.WithCancel(context.Background()) svr, err := CreateServer(ctx, cfg, mokHandler) @@ -276,7 +258,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderXReal() { } func (suite *leaderServerTestSuite) TestSourceIpForHeaderBoth() { - mokHandler := suite.createMokHandler("127.0.0.2") + mokHandler := CreateMokHandler(suite.Require(), "127.0.0.2") cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) ctx, cancel := context.WithCancel(context.Background()) svr, err := CreateServer(ctx, cfg, mokHandler) diff --git a/server/testutil.go b/server/testutil.go index f0fc9069904..7694c9dfbd6 100644 --- a/server/testutil.go +++ b/server/testutil.go @@ -17,6 +17,7 @@ package server import ( "context" "fmt" + "net/http" "os" "strings" "sync" @@ -24,6 +25,7 @@ import ( "github.com/pingcap/log" "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/assertutil" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" @@ -137,3 +139,21 @@ func MustWaitLeader(re *require.Assertions, svrs []*Server) *Server { }) return leader } + +// CreateMokHandler creates a mock handler for test. +func CreateMokHandler(re *require.Assertions, ip string) HandlerBuilder { + return func(ctx context.Context, s *Server) (http.Handler, apiutil.APIServiceGroup, error) { + mux := http.NewServeMux() + mux.HandleFunc("/pd/apis/mok/v1/hello", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello World") + // test getting ip + clientIP := apiutil.GetIPAddrFromHTTPRequest(r) + re.Equal(ip, clientIP) + }) + info := apiutil.APIServiceGroup{ + Name: "mok", + Version: "v1", + } + return mux, info, nil + } +} diff --git a/tests/registry/registry_test.go b/tests/registry/registry_test.go index b33f5552871..9f99e8e942d 100644 --- a/tests/registry/registry_test.go +++ b/tests/registry/registry_test.go @@ -51,8 +51,8 @@ func newTestServiceRegistry(_ bs.Server) registry.RegistrableService { func TestRegistryService(t *testing.T) { re := require.New(t) - re.NoError(failpoint.Enable("github.com/tikv/pd/server/testRegistry", `return(true)`)) - defer re.NoError(failpoint.Disable("github.com/tikv/pd/server/testRegistry")) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/useGlobalRegistry", `return(true)`)) + defer re.NoError(failpoint.Disable("github.com/tikv/pd/server/useGlobalRegistry")) registry.ServerServiceRegistry.RegisterService("test", newTestServiceRegistry) ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/tests/server/member/member_test.go b/tests/server/member/member_test.go index 27cbe7f3fec..0a97e83d38b 100644 --- a/tests/server/member/member_test.go +++ b/tests/server/member/member_test.go @@ -33,7 +33,6 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/api" "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" "go.uber.org/goleak" @@ -308,7 +307,7 @@ func TestGetLeader(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(1) done := make(chan bool) - svr, err := server.CreateServer(ctx, cfg, api.NewHandler) + svr, err := server.CreateServer(ctx, cfg, server.CreateMokHandler(re, "127.0.0.1")) re.NoError(err) defer svr.Close() re.NoError(svr.Run()) From d33dd37a13cc58806614d4ac43abb866d0f6ea6b Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 10 Feb 2023 14:25:42 +0800 Subject: [PATCH 08/14] address comments Signed-off-by: lhy1024 --- server/server.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 server/server.go diff --git a/server/server.go b/server/server.go old mode 100755 new mode 100644 From 69d0b1ad33f30f4adbc25d2b33fd8e77493639cf Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 10 Feb 2023 14:45:50 +0800 Subject: [PATCH 09/14] move test Signed-off-by: lhy1024 --- tests/registry/registry_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/registry/registry_test.go b/tests/registry/registry_test.go index 9f99e8e942d..01f036d5f4c 100644 --- a/tests/registry/registry_test.go +++ b/tests/registry/registry_test.go @@ -51,8 +51,7 @@ func newTestServiceRegistry(_ bs.Server) registry.RegistrableService { func TestRegistryService(t *testing.T) { re := require.New(t) - re.NoError(failpoint.Enable("github.com/tikv/pd/server/useGlobalRegistry", `return(true)`)) - defer re.NoError(failpoint.Disable("github.com/tikv/pd/server/useGlobalRegistry")) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/useGlobalRegistry", "return(true)")) registry.ServerServiceRegistry.RegisterService("test", newTestServiceRegistry) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -83,4 +82,5 @@ func TestRegistryService(t *testing.T) { respString, err := io.ReadAll(resp1.Body) re.NoError(err) re.Equal("Hello World!", string(respString)) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/useGlobalRegistry")) } From 7f17cb9f387130f6b0792d2593ab617e4c4c4a24 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 10 Feb 2023 15:14:45 +0800 Subject: [PATCH 10/14] fix client test Signed-off-by: lhy1024 --- server/testutil.go | 5 +++-- tests/client/client_test.go | 2 +- tests/client/global_config_test.go | 2 +- tests/client/go.mod | 1 + tests/client/go.sum | 2 ++ 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/testutil.go b/server/testutil.go index 7694c9dfbd6..01c02bad4ec 100644 --- a/server/testutil.go +++ b/server/testutil.go @@ -43,10 +43,11 @@ func init() { type CleanupFunc func() // NewTestServer creates a pd server for testing. -func NewTestServer(c *assertutil.Checker) (*Server, CleanupFunc, error) { +func NewTestServer(re *require.Assertions, c *assertutil.Checker) (*Server, CleanupFunc, error) { ctx, cancel := context.WithCancel(context.Background()) cfg := NewTestSingleConfig(c) - s, err := CreateServer(ctx, cfg) + mokHandler := CreateMokHandler(re, "127.0.0.1") + s, err := CreateServer(ctx, cfg, mokHandler) if err != nil { cancel() return nil, nil, err diff --git a/tests/client/client_test.go b/tests/client/client_test.go index 4800ed59b01..ca1f5f9ac82 100644 --- a/tests/client/client_test.go +++ b/tests/client/client_test.go @@ -786,7 +786,7 @@ func TestClientTestSuite(t *testing.T) { func (suite *clientTestSuite) SetupSuite() { var err error re := suite.Require() - suite.srv, suite.cleanup, err = server.NewTestServer(assertutil.CheckerWithNilAssert(re)) + suite.srv, suite.cleanup, err = server.NewTestServer(re, assertutil.CheckerWithNilAssert(re)) suite.NoError(err) suite.grpcPDClient = testutil.MustNewGrpcClient(re, suite.srv.GetAddr()) suite.grpcSvr = &server.GrpcServer{Server: suite.srv} diff --git a/tests/client/global_config_test.go b/tests/client/global_config_test.go index 0958f907385..17a8f4b71bc 100644 --- a/tests/client/global_config_test.go +++ b/tests/client/global_config_test.go @@ -63,7 +63,7 @@ func (suite *globalConfigTestSuite) SetupSuite() { var gsi *server.Server checker := assertutil.NewChecker() checker.FailNow = func() {} - gsi, suite.cleanup, err = server.NewTestServer(checker) + gsi, suite.cleanup, err = server.NewTestServer(suite.Require(), checker) suite.server = &server.GrpcServer{Server: gsi} suite.NoError(err) addr := suite.server.GetAddr() diff --git a/tests/client/go.mod b/tests/client/go.mod index 616b8f5f414..06655b8ef89 100644 --- a/tests/client/go.mod +++ b/tests/client/go.mod @@ -41,6 +41,7 @@ require ( github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect + github.com/gin-contrib/cors v1.4.0 // indirect github.com/gin-contrib/gzip v0.0.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.8.1 // indirect diff --git a/tests/client/go.sum b/tests/client/go.sum index 8039b79563c..a7c5234c11d 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -89,6 +89,8 @@ github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= +github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= github.com/gin-contrib/gzip v0.0.1 h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc= github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= From 6b9824ef7a7ecd8421eac039f3cca6a040c85d52 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 10 Feb 2023 15:21:19 +0800 Subject: [PATCH 11/14] go mod tidy Signed-off-by: lhy1024 --- tests/client/go.sum | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/client/go.sum b/tests/client/go.sum index a7c5234c11d..c0dab0d248d 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -693,6 +693,7 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= From 730b9d0bdf3c16fbd8a3a64dcb34bfd52d185817 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 10 Feb 2023 17:22:53 +0800 Subject: [PATCH 12/14] address comments Signed-off-by: lhy1024 --- pkg/mcs/registry/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mcs/registry/registry.go b/pkg/mcs/registry/registry.go index e3aee1552b0..02e49f63d75 100644 --- a/pkg/mcs/registry/registry.go +++ b/pkg/mcs/registry/registry.go @@ -94,6 +94,6 @@ func (r *ServiceRegistry) InstallAllRESTHandler(srv bs.Server, h map[string]http } // RegisterService registers a grpc service. -func (r ServiceRegistry) RegisterService(name string, service ServiceBuilder) { +func (r *ServiceRegistry) RegisterService(name string, service ServiceBuilder) { r.builders[name] = service } From c80a2a561ee12865a38f3f79cf64bb573fec1b8f Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 10 Feb 2023 17:39:40 +0800 Subject: [PATCH 13/14] address comments: rename Signed-off-by: lhy1024 --- server/server_test.go | 36 +++++++++++++++--------------- server/testutil.go | 8 +++---- tests/server/member/member_test.go | 2 +- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/server/server_test.go b/server/server_test.go index d51af2500b9..2a79974ded6 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -59,8 +59,8 @@ func (suite *leaderServerTestSuite) SetupSuite() { cfg := cfgs[i] go func() { - mokHandler := CreateMokHandler(suite.Require(), "127.0.0.1") - svr, err := CreateServer(suite.ctx, cfg, mokHandler) + MockHandler := CreateMockHandler(suite.Require(), "127.0.0.1") + svr, err := CreateServer(suite.ctx, cfg, MockHandler) suite.NoError(err) err = svr.Run() suite.NoError(err) @@ -89,8 +89,8 @@ func (suite *leaderServerTestSuite) newTestServersWithCfgs(ctx context.Context, ch := make(chan *Server) for _, cfg := range cfgs { go func(cfg *config.Config) { - mokHandler := CreateMokHandler(suite.Require(), "127.0.0.1") - svr, err := CreateServer(ctx, cfg, mokHandler) + MockHandler := CreateMockHandler(suite.Require(), "127.0.0.1") + svr, err := CreateServer(ctx, cfg, MockHandler) // prevent blocking if Asserts fails failed := true defer func() { @@ -155,8 +155,8 @@ func (suite *leaderServerTestSuite) TestCheckClusterID() { // Start previous cluster, expect an error. cfgA.InitialCluster = originInitial - mokHandler := CreateMokHandler(suite.Require(), "127.0.0.1") - svr, err := CreateServer(ctx, cfgA, mokHandler) + MockHandler := CreateMockHandler(suite.Require(), "127.0.0.1") + svr, err := CreateServer(ctx, cfgA, MockHandler) suite.NoError(err) etcd, err := embed.StartEtcd(svr.etcdCfg) @@ -174,10 +174,10 @@ func (suite *leaderServerTestSuite) TestCheckClusterID() { func (suite *leaderServerTestSuite) TestRegisterServerHandler() { cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) ctx, cancel := context.WithCancel(context.Background()) - mokHandler := CreateMokHandler(suite.Require(), "127.0.0.1") - svr, err := CreateServer(ctx, cfg, mokHandler) + MockHandler := CreateMockHandler(suite.Require(), "127.0.0.1") + svr, err := CreateServer(ctx, cfg, MockHandler) suite.NoError(err) - _, err = CreateServer(ctx, cfg, mokHandler, mokHandler) + _, err = CreateServer(ctx, cfg, MockHandler, MockHandler) // Repeat register. suite.Error(err) defer func() { @@ -198,12 +198,12 @@ func (suite *leaderServerTestSuite) TestRegisterServerHandler() { } func (suite *leaderServerTestSuite) TestSourceIpForHeaderForwarded() { - mokHandler := CreateMokHandler(suite.Require(), "127.0.0.2") + MockHandler := CreateMockHandler(suite.Require(), "127.0.0.2") cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) ctx, cancel := context.WithCancel(context.Background()) - svr, err := CreateServer(ctx, cfg, mokHandler) + svr, err := CreateServer(ctx, cfg, MockHandler) suite.NoError(err) - _, err = CreateServer(ctx, cfg, mokHandler, mokHandler) + _, err = CreateServer(ctx, cfg, MockHandler, MockHandler) // Repeat register. suite.Error(err) defer func() { @@ -228,12 +228,12 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderForwarded() { } func (suite *leaderServerTestSuite) TestSourceIpForHeaderXReal() { - mokHandler := CreateMokHandler(suite.Require(), "127.0.0.2") + MockHandler := CreateMockHandler(suite.Require(), "127.0.0.2") cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) ctx, cancel := context.WithCancel(context.Background()) - svr, err := CreateServer(ctx, cfg, mokHandler) + svr, err := CreateServer(ctx, cfg, MockHandler) suite.NoError(err) - _, err = CreateServer(ctx, cfg, mokHandler, mokHandler) + _, err = CreateServer(ctx, cfg, MockHandler, MockHandler) // Repeat register. suite.Error(err) defer func() { @@ -258,12 +258,12 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderXReal() { } func (suite *leaderServerTestSuite) TestSourceIpForHeaderBoth() { - mokHandler := CreateMokHandler(suite.Require(), "127.0.0.2") + MockHandler := CreateMockHandler(suite.Require(), "127.0.0.2") cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) ctx, cancel := context.WithCancel(context.Background()) - svr, err := CreateServer(ctx, cfg, mokHandler) + svr, err := CreateServer(ctx, cfg, MockHandler) suite.NoError(err) - _, err = CreateServer(ctx, cfg, mokHandler, mokHandler) + _, err = CreateServer(ctx, cfg, MockHandler, MockHandler) // Repeat register. suite.Error(err) defer func() { diff --git a/server/testutil.go b/server/testutil.go index 01c02bad4ec..908640b5a65 100644 --- a/server/testutil.go +++ b/server/testutil.go @@ -46,8 +46,8 @@ type CleanupFunc func() func NewTestServer(re *require.Assertions, c *assertutil.Checker) (*Server, CleanupFunc, error) { ctx, cancel := context.WithCancel(context.Background()) cfg := NewTestSingleConfig(c) - mokHandler := CreateMokHandler(re, "127.0.0.1") - s, err := CreateServer(ctx, cfg, mokHandler) + MockHandler := CreateMockHandler(re, "127.0.0.1") + s, err := CreateServer(ctx, cfg, MockHandler) if err != nil { cancel() return nil, nil, err @@ -141,8 +141,8 @@ func MustWaitLeader(re *require.Assertions, svrs []*Server) *Server { return leader } -// CreateMokHandler creates a mock handler for test. -func CreateMokHandler(re *require.Assertions, ip string) HandlerBuilder { +// CreateMockHandler creates a mock handler for test. +func CreateMockHandler(re *require.Assertions, ip string) HandlerBuilder { return func(ctx context.Context, s *Server) (http.Handler, apiutil.APIServiceGroup, error) { mux := http.NewServeMux() mux.HandleFunc("/pd/apis/mok/v1/hello", func(w http.ResponseWriter, r *http.Request) { diff --git a/tests/server/member/member_test.go b/tests/server/member/member_test.go index 0a97e83d38b..c8b953e64c1 100644 --- a/tests/server/member/member_test.go +++ b/tests/server/member/member_test.go @@ -307,7 +307,7 @@ func TestGetLeader(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(1) done := make(chan bool) - svr, err := server.CreateServer(ctx, cfg, server.CreateMokHandler(re, "127.0.0.1")) + svr, err := server.CreateServer(ctx, cfg, server.CreateMockHandler(re, "127.0.0.1")) re.NoError(err) defer svr.Close() re.NoError(svr.Run()) From f526604bd4622500e5dd415868bb2f30f1ef12c4 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 10 Feb 2023 18:26:13 +0800 Subject: [PATCH 14/14] address comments: fix url Signed-off-by: lhy1024 --- server/server_test.go | 8 ++++---- server/testutil.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/server_test.go b/server/server_test.go index 2a79974ded6..2ff47348a80 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -187,7 +187,7 @@ func (suite *leaderServerTestSuite) TestRegisterServerHandler() { }() err = svr.Run() suite.NoError(err) - resp, err := http.Get(fmt.Sprintf("%s/pd/apis/mok/v1/hello", svr.GetAddr())) + resp, err := http.Get(fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr())) suite.NoError(err) suite.Equal(http.StatusOK, resp.StatusCode) defer resp.Body.Close() @@ -214,7 +214,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderForwarded() { err = svr.Run() suite.NoError(err) - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mok/v1/hello", svr.GetAddr()), nil) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr()), nil) suite.NoError(err) req.Header.Add("X-Forwarded-For", "127.0.0.2") resp, err := http.DefaultClient.Do(req) @@ -244,7 +244,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderXReal() { err = svr.Run() suite.NoError(err) - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mok/v1/hello", svr.GetAddr()), nil) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr()), nil) suite.NoError(err) req.Header.Add("X-Real-Ip", "127.0.0.2") resp, err := http.DefaultClient.Do(req) @@ -274,7 +274,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderBoth() { err = svr.Run() suite.NoError(err) - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mok/v1/hello", svr.GetAddr()), nil) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr()), nil) suite.NoError(err) req.Header.Add("X-Forwarded-For", "127.0.0.2") req.Header.Add("X-Real-Ip", "127.0.0.3") diff --git a/server/testutil.go b/server/testutil.go index 908640b5a65..d64fbbf8a37 100644 --- a/server/testutil.go +++ b/server/testutil.go @@ -145,14 +145,14 @@ func MustWaitLeader(re *require.Assertions, svrs []*Server) *Server { func CreateMockHandler(re *require.Assertions, ip string) HandlerBuilder { return func(ctx context.Context, s *Server) (http.Handler, apiutil.APIServiceGroup, error) { mux := http.NewServeMux() - mux.HandleFunc("/pd/apis/mok/v1/hello", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/pd/apis/mock/v1/hello", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "Hello World") // test getting ip clientIP := apiutil.GetIPAddrFromHTTPRequest(r) re.Equal(ip, clientIP) }) info := apiutil.APIServiceGroup{ - Name: "mok", + Name: "mock", Version: "v1", } return mux, info, nil