From 0bebe9626256ce53ef7f5106340e13e5cc3bafa6 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 16 Feb 2023 22:09:50 +0800 Subject: [PATCH 01/24] resource_manager: add service mode server Signed-off-by: lhy1024 --- client/resource_manager_client.go | 6 +- go.mod | 2 +- pkg/basicserver/basic_server.go | 10 +- pkg/mcs/resource_manager/server/config.go | 92 ++++++ .../resource_manager/server/grpc_service.go | 22 +- pkg/mcs/resource_manager/server/manager.go | 10 +- pkg/mcs/resource_manager/server/server.go | 304 ++++++++++++++++++ pkg/mcs/tso/server/server.go | 14 +- server/server.go | 9 +- tests/mcs/go.mod | 4 +- tests/mcs/go.sum | 44 ++- tests/mcs/resource_manager/server_test.go | 100 ++++++ tests/registry/registry_test.go | 18 +- 13 files changed, 595 insertions(+), 40 deletions(-) create mode 100644 pkg/mcs/resource_manager/server/config.go create mode 100644 pkg/mcs/resource_manager/server/server.go create mode 100644 tests/mcs/resource_manager/server_test.go diff --git a/client/resource_manager_client.go b/client/resource_manager_client.go index 81dc42afd3d..a3b85cd67b7 100644 --- a/client/resource_manager_client.go +++ b/client/resource_manager_client.go @@ -33,8 +33,8 @@ const ( add actionType = 0 modify actionType = 1 groupSettingsPathPrefix = "resource_group/settings" - // errNotLeaderMsg is returned when the requested server is not the leader. - errNotLeaderMsg = "not leader" + // errNotPrimary is returned when the requested server is not primary. + errNotPrimary = "not primary" ) // ResourceManagerClient manages resource group info and token request. @@ -58,7 +58,7 @@ func (c *client) resourceManagerClient() rmpb.ResourceManagerClient { // gRPCErrorHandler is used to handle the gRPC error returned by the resource manager service. func (c *client) gRPCErrorHandler(err error) { - if strings.Contains(err.Error(), errNotLeaderMsg) { + if strings.Contains(err.Error(), errNotPrimary) { c.ScheduleCheckLeader() } } diff --git a/go.mod b/go.mod index 3baeba2cb3c..c0220174f99 100644 --- a/go.mod +++ b/go.mod @@ -146,7 +146,7 @@ require ( github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/sirupsen/logrus v1.6.0 // indirect - github.com/soheilhy/cmux v0.1.4 // indirect + github.com/soheilhy/cmux v0.1.4 github.com/stretchr/objx v0.5.0 // indirect github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 // indirect github.com/tidwall/gjson v1.9.3 // indirect diff --git a/pkg/basicserver/basic_server.go b/pkg/basicserver/basic_server.go index d83296961d4..187cd6bf82c 100644 --- a/pkg/basicserver/basic_server.go +++ b/pkg/basicserver/basic_server.go @@ -18,7 +18,6 @@ import ( "context" "net/http" - "github.com/tikv/pd/pkg/member" "go.etcd.io/etcd/clientv3" ) @@ -38,9 +37,8 @@ type Server interface { GetHTTPClient() *http.Client // AddStartCallback adds a callback in the startServer phase. AddStartCallback(callbacks ...func()) - // TODO: replace these two methods with `primary` function without etcd server dependency. - // GetMember returns the member information. - GetMember() *member.Member - // AddLeaderCallback adds a callback in the leader campaign phase. - AddLeaderCallback(callbacks ...func(context.Context)) + // IsPrimary returns whether the server is primary, if there is etcd server in the server, it will return whether it is leader. + IsPrimary() bool + // AddPrimaryCallback adds a callback when the server becomes primary, if there is etcd server in the server, it means the server becomes leader. + AddPrimaryCallback(callbacks ...func(context.Context)) } diff --git a/pkg/mcs/resource_manager/server/config.go b/pkg/mcs/resource_manager/server/config.go new file mode 100644 index 00000000000..dcacbce877d --- /dev/null +++ b/pkg/mcs/resource_manager/server/config.go @@ -0,0 +1,92 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "github.com/BurntSushi/toml" + "github.com/pingcap/errors" + "github.com/pingcap/log" + "github.com/spf13/pflag" + "github.com/tikv/pd/pkg/encryption" + "github.com/tikv/pd/pkg/utils/grpcutil" + "github.com/tikv/pd/pkg/utils/metricutil" + + "go.uber.org/zap" +) + +// Config is the configuration for the TSO. +type Config struct { + BackendEndpoints string `toml:"backend-endpoints" json:"backend-endpoints"` + ListenAddr string `toml:"listen-addr" json:"listen-addr"` + + Metric metricutil.MetricConfig `toml:"metric" json:"metric"` + + // Log related config. + Log log.Config `toml:"log" json:"log"` + + Logger *zap.Logger + LogProps *log.ZapProperties + + Security SecurityConfig `toml:"security" json:"security"` +} + +// NewConfig creates a new config. +func NewConfig() *Config { + return &Config{} +} + +// Parse parses flag definitions from the argument list. +func (c *Config) Parse(flagSet *pflag.FlagSet) error { + // Load config file if specified. + if configFile, _ := flagSet.GetString("config"); configFile != "" { + _, err := c.configFromFile(configFile) + if err != nil { + return err + } + } + + // ignore the error check here + adjustCommandlineString(flagSet, &c.Log.Level, "log-level") + adjustCommandlineString(flagSet, &c.Log.File.Filename, "log-file") + adjustCommandlineString(flagSet, &c.Metric.PushAddress, "metrics-addr") + adjustCommandlineString(flagSet, &c.Security.CAPath, "cacert") + adjustCommandlineString(flagSet, &c.Security.CertPath, "cert") + adjustCommandlineString(flagSet, &c.Security.KeyPath, "key") + adjustCommandlineString(flagSet, &c.BackendEndpoints, "backend-endpoints") + adjustCommandlineString(flagSet, &c.ListenAddr, "listen-addr") + + // TODO: Implement the main function body + return nil +} + +// configFromFile loads config from file. +func (c *Config) configFromFile(path string) (*toml.MetaData, error) { + meta, err := toml.DecodeFile(path, c) + return &meta, errors.WithStack(err) +} + +// SecurityConfig indicates the security configuration for pd server +type SecurityConfig struct { + grpcutil.TLSConfig + // RedactInfoLog indicates that whether enabling redact log + RedactInfoLog bool `toml:"redact-info-log" json:"redact-info-log"` + Encryption encryption.Config `toml:"encryption" json:"encryption"` +} + +func adjustCommandlineString(flagSet *pflag.FlagSet, v *string, name string) { + if value, _ := flagSet.GetString(name); value != "" { + *v = value + } +} diff --git a/pkg/mcs/resource_manager/server/grpc_service.go b/pkg/mcs/resource_manager/server/grpc_service.go index 363a79fe841..1a8863f5276 100644 --- a/pkg/mcs/resource_manager/server/grpc_service.go +++ b/pkg/mcs/resource_manager/server/grpc_service.go @@ -33,8 +33,8 @@ import ( ) var ( - // errNotLeader is returned when current server is not the leader. - errNotLeader = status.Errorf(codes.Unavailable, "not leader") + // errNotPrimary is returned when current server is not primary. + errNotPrimary = status.Errorf(codes.Unavailable, "not primary") ) var _ rmpb.ResourceManagerServer = (*Service)(nil) @@ -84,16 +84,16 @@ func (s *Service) GetManager() *Manager { return s.manager } -func (s *Service) checkLeader() error { - if !s.manager.member.IsLeader() { - return errNotLeader +func (s *Service) checkPrimary() error { + if !s.manager.srv.IsPrimary() { + return errNotPrimary } return nil } // GetResourceGroup implements ResourceManagerServer.GetResourceGroup. func (s *Service) GetResourceGroup(ctx context.Context, req *rmpb.GetResourceGroupRequest) (*rmpb.GetResourceGroupResponse, error) { - if err := s.checkLeader(); err != nil { + if err := s.checkPrimary(); err != nil { return nil, err } rg := s.manager.GetResourceGroup(req.ResourceGroupName) @@ -107,7 +107,7 @@ func (s *Service) GetResourceGroup(ctx context.Context, req *rmpb.GetResourceGro // ListResourceGroups implements ResourceManagerServer.ListResourceGroups. func (s *Service) ListResourceGroups(ctx context.Context, req *rmpb.ListResourceGroupsRequest) (*rmpb.ListResourceGroupsResponse, error) { - if err := s.checkLeader(); err != nil { + if err := s.checkPrimary(); err != nil { return nil, err } groups := s.manager.GetResourceGroupList() @@ -122,7 +122,7 @@ func (s *Service) ListResourceGroups(ctx context.Context, req *rmpb.ListResource // AddResourceGroup implements ResourceManagerServer.AddResourceGroup. func (s *Service) AddResourceGroup(ctx context.Context, req *rmpb.PutResourceGroupRequest) (*rmpb.PutResourceGroupResponse, error) { - if err := s.checkLeader(); err != nil { + if err := s.checkPrimary(); err != nil { return nil, err } rg := FromProtoResourceGroup(req.GetGroup()) @@ -135,7 +135,7 @@ func (s *Service) AddResourceGroup(ctx context.Context, req *rmpb.PutResourceGro // DeleteResourceGroup implements ResourceManagerServer.DeleteResourceGroup. func (s *Service) DeleteResourceGroup(ctx context.Context, req *rmpb.DeleteResourceGroupRequest) (*rmpb.DeleteResourceGroupResponse, error) { - if err := s.checkLeader(); err != nil { + if err := s.checkPrimary(); err != nil { return nil, err } err := s.manager.DeleteResourceGroup(req.ResourceGroupName) @@ -147,7 +147,7 @@ func (s *Service) DeleteResourceGroup(ctx context.Context, req *rmpb.DeleteResou // ModifyResourceGroup implements ResourceManagerServer.ModifyResourceGroup. func (s *Service) ModifyResourceGroup(ctx context.Context, req *rmpb.PutResourceGroupRequest) (*rmpb.PutResourceGroupResponse, error) { - if err := s.checkLeader(); err != nil { + if err := s.checkPrimary(); err != nil { return nil, err } err := s.manager.ModifyResourceGroup(req.GetGroup()) @@ -172,7 +172,7 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu if err != nil { return errors.WithStack(err) } - if err := s.checkLeader(); err != nil { + if err := s.checkPrimary(); err != nil { return err } targetPeriodMs := request.GetTargetRequestPeriodMs() diff --git a/pkg/mcs/resource_manager/server/manager.go b/pkg/mcs/resource_manager/server/manager.go index 0c6698133d2..773a49fd057 100644 --- a/pkg/mcs/resource_manager/server/manager.go +++ b/pkg/mcs/resource_manager/server/manager.go @@ -27,7 +27,6 @@ import ( rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/log" bs "github.com/tikv/pd/pkg/basicserver" - "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "go.uber.org/zap" @@ -38,7 +37,7 @@ const defaultConsumptionChanSize = 1024 // Manager is the manager of resource group. type Manager struct { sync.RWMutex - member *member.Member + srv bs.Server groups map[string]*ResourceGroup storage endpoint.ResourceGroupStorage // consumptionChan is used to send the consumption @@ -52,7 +51,6 @@ type Manager struct { // NewManager returns a new Manager. func NewManager(srv bs.Server) *Manager { m := &Manager{ - member: &member.Member{}, groups: make(map[string]*ResourceGroup), consumptionDispatcher: make(chan struct { resourceGroupName string @@ -66,10 +64,10 @@ func NewManager(srv bs.Server) *Manager { kv.NewEtcdKVBase(srv.GetClient(), "resource_group"), nil, ) - m.member = srv.GetMember() + m.srv = srv }) - // The second initialization after the leader is elected. - srv.AddLeaderCallback(m.Init) + // The second initialization after becoming primary. + srv.AddPrimaryCallback(m.Init) return m } diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go new file mode 100644 index 00000000000..b634391def4 --- /dev/null +++ b/pkg/mcs/resource_manager/server/server.go @@ -0,0 +1,304 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "crypto/tls" + "flag" + "net" + "net/http" + "net/url" + "os" + "os/signal" + "sync" + "sync/atomic" + "syscall" + "time" + + grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/pingcap/errors" + "github.com/pingcap/log" + "github.com/soheilhy/cmux" + "github.com/spf13/cobra" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/logutil" + "github.com/tikv/pd/pkg/utils/metricutil" + "go.etcd.io/etcd/clientv3" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +// Server is the resource manager server, and it implements bs.Server. +// nolint +type Server struct { + cfg *Config + mux cmux.CMux + // Server state. 0 is not serving, 1 is serving. + isServing int64 + ctx context.Context + name string + + etcdClient *clientv3.Client + httpClient *http.Client + + grpcServer *grpc.Server + httpServer *http.Server + service *Service + + serverLoopWg sync.WaitGroup + // Callback functions for different stages + // startCallbacks will be called after the server is started. + startCallbacks []func() + // primaryCallbacks will be called after the server becomes leader. + primaryCallbacks []func(context.Context) +} + +// Name returns the unique etcd Name for this server in etcd cluster. +func (s *Server) Name() string { + return s.name +} + +// Context returns the context. +func (s *Server) Context() context.Context { + return s.ctx +} + +// Run runs the pd server. +func (s *Server) Run() error { + if err := s.initClient(); err != nil { + return err + } + if err := s.startServer(); err != nil { + return err + } + return s.mux.Serve() +} + +// Close closes the server. +func (s *Server) Close() { + if !atomic.CompareAndSwapInt64(&s.isServing, 1, 0) { + // server is already closed + return + } + + log.Info("closing server") + + s.grpcServer.Stop() + s.httpServer.Shutdown(s.ctx) + s.serverLoopWg.Wait() + + if s.etcdClient != nil { + if err := s.etcdClient.Close(); err != nil { + log.Error("close etcd client meet error", errs.ZapError(errs.ErrCloseEtcdClient, err)) + } + } + + if s.httpClient != nil { + s.httpClient.CloseIdleConnections() + } +} + +// GetClient returns builtin etcd client. +func (s *Server) GetClient() *clientv3.Client { + return s.etcdClient +} + +// GetHTTPClient returns builtin http client. +func (s *Server) GetHTTPClient() *http.Client { + return s.httpClient +} + +// AddStartCallback adds a callback in the startServer phase. +func (s *Server) AddStartCallback(callbacks ...func()) { + s.startCallbacks = append(s.startCallbacks, callbacks...) +} + +// IsPrimary returns whether the server is primary, if there is etcd server in the server, it will return whether it is leader. +func (s *Server) IsPrimary() bool { + // TODO: implement this + return true +} + +// AddPrimaryCallback adds a callback when the server becomes primary, if there is etcd server in the server, it means the server becomes leader. +func (s *Server) AddPrimaryCallback(callbacks ...func(context.Context)) { + s.primaryCallbacks = append(s.primaryCallbacks, callbacks...) +} + +func (s *Server) initClient() error { + tlsConfig, err := s.cfg.Security.ToTLSConfig() + if err != nil { + return err + } + endpoint, err := url.Parse(s.cfg.BackendEndpoints) + if err != nil { + return err + } + s.etcdClient, s.httpClient, err = etcdutil.CreateClients(tlsConfig, []url.URL{*endpoint}) + return err +} + +func (s *Server) startServer() error { + tlsConfig, err := s.cfg.Security.ToTLSConfig() + if err != nil { + return err + } + if tlsConfig != nil { + l, err := tls.Listen("tcp", s.cfg.ListenAddr, tlsConfig) + if err != nil { + return err + } + s.mux = cmux.New(l) + } else { + l, err := net.Listen("tcp", s.cfg.ListenAddr) + if err != nil { + return err + } + s.mux = cmux.New(l) + } + + grpcL := s.mux.MatchWithWriters(cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc")) + httpL := s.mux.Match(cmux.Any()) + + manager := NewManager(s) + s.service = &Service{ + ctx: s.ctx, + manager: manager, + } + + s.grpcServer = grpc.NewServer() + s.service.RegisterGRPCService(s.grpcServer) + + handler, _ := SetUpRestHandler(s.service) + s.httpServer = &http.Server{ + Handler: handler, + ReadTimeout: 5 * time.Minute, + } + + go func() { + defer s.serverLoopWg.Done() + s.grpcServer.Serve(grpcL) + }() + go func() { + defer s.serverLoopWg.Done() + s.httpServer.Serve(httpL) + }() + s.serverLoopWg.Add(2) + + // Run callbacks + log.Info("triggering the start callback functions") + for _, cb := range s.startCallbacks { + cb() + } + // Server has started. + atomic.StoreInt64(&s.isServing, 1) + return nil +} + +// NewServer creates a new resource manager server. +func NewServer(name string, ctx context.Context, cfg *Config) *Server { + return &Server{ + name: name, + ctx: ctx, + cfg: cfg, + } +} + +// CreateServerWrapper encapsulates the configuration/log/metrics initialization and create the server +func CreateServerWrapper(cmd *cobra.Command, args []string) { + cmd.Flags().Parse(args) + cfg := NewConfig() + flagSet := cmd.Flags() + err := cfg.Parse(flagSet) + if err != nil { + cmd.Println(err) + return + } + + printVersion, err := flagSet.GetBool("version") + if err != nil { + cmd.Println(err) + return + } + if printVersion { + // TODO: support printing resource manager server info + // server.PrintPDInfo() + exit(0) + } + + defer logutil.LogPanic() + + switch errors.Cause(err) { + case nil: + case flag.ErrHelp: + exit(0) + default: + log.Fatal("parse cmd flags error", errs.ZapError(err)) + } + + // New zap logger + err = logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) + if err == nil { + log.ReplaceGlobals(cfg.Logger, cfg.LogProps) + } else { + log.Fatal("initialize logger error", errs.ZapError(err)) + } + // Flushing any buffered log entries + defer log.Sync() + + // TODO: support printing resource manager server info + // server.LogPDInfo() + + grpcprometheus.EnableHandlingTimeHistogram() + + metricutil.Push(&cfg.Metric) + + ctx, cancel := context.WithCancel(context.Background()) + svr := NewServer("ResourceManager", ctx, cfg) + + sc := make(chan os.Signal, 1) + signal.Notify(sc, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT) + + var sig os.Signal + go func() { + sig = <-sc + cancel() + }() + + if err := svr.Run(); err != nil { + log.Fatal("run server failed", errs.ZapError(err)) + } + + <-ctx.Done() + log.Info("Got signal to exit", zap.String("signal", sig.String())) + + svr.Close() + switch sig { + case syscall.SIGTERM: + exit(0) + default: + exit(1) + } +} + +func exit(code int) { + log.Sync() + os.Exit(code) +} diff --git a/pkg/mcs/tso/server/server.go b/pkg/mcs/tso/server/server.go index 0f8ed822407..36b375172b0 100644 --- a/pkg/mcs/tso/server/server.go +++ b/pkg/mcs/tso/server/server.go @@ -32,7 +32,6 @@ import ( "github.com/spf13/cobra" bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/logutil" @@ -122,14 +121,15 @@ func (s *Server) AddStartCallback(callbacks ...func()) { s.startCallbacks = append(s.startCallbacks, callbacks...) } -// GetMember returns the member. -func (s *Server) GetMember() *member.Member { - return nil +// IsPrimary returns whether the server is primary, if there is etcd server in the server, it will return whether it is leader. +func (s *Server) IsPrimary() bool { + // TODO: implement this + return true } -// AddLeaderCallback adds the callback function when the server becomes +// AddPrimaryCallback adds a callback when the server becomes primary, if there is etcd server in the server, it means the server becomes leader. // the global TSO allocator after the flag 'enable-local-tso' is set to true. -func (s *Server) AddLeaderCallback(callbacks ...func(context.Context)) { +func (s *Server) AddPrimaryCallback(callbacks ...func(context.Context)) { // Leave it empty // TODO: implment it when integerating with the Local/Global TSO Allocator. } @@ -309,7 +309,7 @@ func CreateServerWrapper(cmd *cobra.Command, args []string) { // TODO: Create the server ctx, cancel := context.WithCancel(context.Background()) svr := &Server{} - + // TODO: tso server needs timeJumpBackCounter. sc := make(chan os.Signal, 1) signal.Notify(sc, syscall.SIGHUP, diff --git a/server/server.go b/server/server.go index 3830fd574af..e8faf20d9f3 100644 --- a/server/server.go +++ b/server/server.go @@ -1318,8 +1318,13 @@ func (s *Server) SetReplicationModeConfig(cfg config.ReplicationModeConfig) erro return nil } -// AddLeaderCallback adds a callback in the leader campaign phase. -func (s *Server) AddLeaderCallback(callbacks ...func(context.Context)) { +// IsPrimary returns whether the server is primary, if there is etcd server in the server, it will return whether it is leader. +func (s *Server) IsPrimary() bool { + return s.member.IsLeader() +} + +// AddPrimaryCallback adds a callback when the server becomes primary, if there is etcd server in the server, it means the server becomes leader. +func (s *Server) AddPrimaryCallback(callbacks ...func(context.Context)) { s.leaderCallbacks = append(s.leaderCallbacks, callbacks...) } diff --git a/tests/mcs/go.mod b/tests/mcs/go.mod index c9935961450..fda975aaab6 100644 --- a/tests/mcs/go.mod +++ b/tests/mcs/go.mod @@ -9,6 +9,7 @@ require ( github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 go.uber.org/goleak v1.1.12 + google.golang.org/grpc v1.51.0 ) require ( @@ -68,6 +69,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 // indirect github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -113,6 +115,7 @@ require ( github.com/sirupsen/logrus v1.6.0 // indirect github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 // indirect github.com/soheilhy/cmux v0.1.4 // indirect + github.com/spf13/cobra v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 // indirect @@ -148,7 +151,6 @@ require ( golang.org/x/tools v0.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd // indirect - google.golang.org/grpc v1.51.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/tests/mcs/go.sum b/tests/mcs/go.sum index ff06a19d1c8..6e32c4c2255 100644 --- a/tests/mcs/go.sum +++ b/tests/mcs/go.sum @@ -6,6 +6,7 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -28,6 +29,7 @@ github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4= github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54= github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -47,6 +49,7 @@ github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfV github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -57,10 +60,13 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -68,11 +74,14 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= @@ -164,8 +173,9 @@ github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF0 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -209,21 +219,26 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d h1:uGg2frlt3IcT7kbV6LEp5ONv4vmoO2FW4qSO+my/aoM= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys= @@ -286,6 +301,7 @@ github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -306,6 +322,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/microsoft/go-mssqldb v0.17.0 h1:Fto83dMZPnYv1Zwx5vHHxpNraeEaUlQ/hhHLgZiaenE= github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -319,6 +337,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/oleiade/reflections v1.0.1 h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0HHQM= github.com/oleiade/reflections v1.0.1/go.mod h1:rdFxbxq4QXVZWj0F+e9jqjDkc7dbp97vkRixKo2JR60= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -330,6 +349,7 @@ github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo= @@ -371,6 +391,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= @@ -380,15 +401,19 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -397,6 +422,7 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.37.0 h1:XjVcB8g6tgUp8rsPsJ2CvhClfImrpL04YpQHXeHPhRw= github.com/samber/lo v1.37.0/go.mod h1:9vaz2O4o8oOnK23pd2TrXufcbdbJIa3b6cstBWKpopA= github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= @@ -408,6 +434,7 @@ github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 h1:mj/nMDAwTBiaCqMEs4cYCqF7pO6Np7vhy1D1wcQGz+E= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= @@ -416,10 +443,18 @@ github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 h1:Txo4SXVJq/OgEjw github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072/go.mod h1:+4nWMF0+CqEcU74SnX2NxaGqZ8zX4pcQ8Jcs77DbX5A= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -458,6 +493,7 @@ github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7Am github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= @@ -480,16 +516,19 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 h1:fqmtdYQlwZ/vKWSz5amW+a4cnjg23ojz5iL7rjf08Wg= go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793/go.mod h1:eBhtbxXP1qpW0F6+WxoJ64DM1Mrfx46PHtVxEdkLe0I= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -559,6 +598,7 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -635,6 +675,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -678,6 +719,7 @@ google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd h1:OjndDrsik+Gt+e6 google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= diff --git a/tests/mcs/resource_manager/server_test.go b/tests/mcs/resource_manager/server_test.go new file mode 100644 index 00000000000..6effe57e418 --- /dev/null +++ b/tests/mcs/resource_manager/server_test.go @@ -0,0 +1,100 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resourcemanager_test + +import ( + "context" + "encoding/json" + "io" + "net/http" + "strings" + "testing" + "time" + + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/stretchr/testify/require" + rm "github.com/tikv/pd/pkg/mcs/resource_manager/server" + "github.com/tikv/pd/tests" + "google.golang.org/grpc" + "google.golang.org/grpc/test/grpc_testing" +) + +func TestResourceManagerServer(t *testing.T) { + re := require.New(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cluster, err := tests.NewTestCluster(ctx, 1) + defer cluster.Destroy() + re.NoError(err) + + err = cluster.RunInitialServers() + re.NoError(err) + + leaderName := cluster.WaitLeader() + leader := cluster.GetServer(leaderName) + + cfg := rm.NewConfig() + cfg.BackendEndpoints = leader.GetAddr() + cfg.ListenAddr = "127.0.0.1:8086" + + svr := rm.NewServer("ResourceManager", ctx, cfg) + go svr.Run() + time.Sleep(2 * time.Second) // wait for server start + defer svr.Close() + + // Test registered GRPC Service + cc, err := grpc.DialContext(ctx, cfg.ListenAddr, grpc.WithInsecure()) + re.NoError(err) + defer cc.Close() + grpcClient := grpc_testing.NewTestServiceClient(cc) + resp, err := grpcClient.EmptyCall(context.Background(), &grpc_testing.Empty{}) + re.ErrorContains(err, "Unimplemented") + re.Nil(resp) + + // Test registered REST HTTP Handler + url := "http://" + cfg.ListenAddr + "/resource-manager/api/v1/config" + { + resp, err := http.Get(url + "/groups") + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + respString, err := io.ReadAll(resp.Body) + re.NoError(err) + re.Equal("[]", string(respString)) + + } + { + group := &rmpb.ResourceGroup{ + Name: "pingcap", + Mode: 1, + } + createJSON, err := json.Marshal(group) + re.NoError(err) + resp, err := http.Post(url+"/group", "application/json", strings.NewReader(string(createJSON))) + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + } + { + resp, err := http.Get(url + "/group/pingcap") + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + respString, err := io.ReadAll(resp.Body) + re.NoError(err) + re.Equal("{\"name\":\"pingcap\",\"mode\":1,\"r_u_settings\":{\"ru\":{\"state\":{\"initialized\":false}}}}", string(respString)) + } +} diff --git a/tests/registry/registry_test.go b/tests/registry/registry_test.go index 01f036d5f4c..da68bddd354 100644 --- a/tests/registry/registry_test.go +++ b/tests/registry/registry_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package registry_test import ( @@ -69,8 +83,8 @@ func TestRegistryService(t *testing.T) { cc, err := grpc.DialContext(ctx, strings.TrimPrefix(leader.GetAddr(), "http://"), grpc.WithInsecure()) re.NoError(err) defer cc.Close() - grpclient := grpc_testing.NewTestServiceClient(cc) - resp, err := grpclient.EmptyCall(context.Background(), &grpc_testing.Empty{}) + grpcClient := grpc_testing.NewTestServiceClient(cc) + resp, err := grpcClient.EmptyCall(context.Background(), &grpc_testing.Empty{}) re.ErrorContains(err, "Unimplemented") re.Nil(resp) From e12f074eb68712f738f47bc7f4b3f494582adac2 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 16 Feb 2023 22:15:24 +0800 Subject: [PATCH 02/24] fix lint Signed-off-by: lhy1024 --- tests/client/go.mod | 2 ++ tests/client/go.sum | 44 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/tests/client/go.mod b/tests/client/go.mod index 7db99fb1b6a..37e64c196fa 100644 --- a/tests/client/go.mod +++ b/tests/client/go.mod @@ -72,6 +72,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 // indirect github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -117,6 +118,7 @@ require ( github.com/sirupsen/logrus v1.6.0 // indirect github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 // indirect github.com/soheilhy/cmux v0.1.4 // indirect + github.com/spf13/cobra v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 // indirect diff --git a/tests/client/go.sum b/tests/client/go.sum index b7484fb1119..decdc8941cd 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -6,6 +6,7 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -28,6 +29,7 @@ github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4= github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54= github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -47,6 +49,7 @@ github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfV github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -57,10 +60,13 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -68,11 +74,14 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= @@ -164,8 +173,9 @@ github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF0 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -209,21 +219,26 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d h1:uGg2frlt3IcT7kbV6LEp5ONv4vmoO2FW4qSO+my/aoM= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys= @@ -286,6 +301,7 @@ github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -306,6 +322,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/microsoft/go-mssqldb v0.17.0 h1:Fto83dMZPnYv1Zwx5vHHxpNraeEaUlQ/hhHLgZiaenE= github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -319,6 +337,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/oleiade/reflections v1.0.1 h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0HHQM= github.com/oleiade/reflections v1.0.1/go.mod h1:rdFxbxq4QXVZWj0F+e9jqjDkc7dbp97vkRixKo2JR60= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -330,6 +349,7 @@ github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo= @@ -372,6 +392,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= @@ -381,15 +402,19 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -398,6 +423,7 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.37.0 h1:XjVcB8g6tgUp8rsPsJ2CvhClfImrpL04YpQHXeHPhRw= github.com/samber/lo v1.37.0/go.mod h1:9vaz2O4o8oOnK23pd2TrXufcbdbJIa3b6cstBWKpopA= github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= @@ -409,6 +435,7 @@ github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 h1:mj/nMDAwTBiaCqMEs4cYCqF7pO6Np7vhy1D1wcQGz+E= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= @@ -417,10 +444,18 @@ github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 h1:Txo4SXVJq/OgEjw github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072/go.mod h1:+4nWMF0+CqEcU74SnX2NxaGqZ8zX4pcQ8Jcs77DbX5A= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -459,6 +494,7 @@ github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7Am github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= @@ -481,16 +517,19 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 h1:fqmtdYQlwZ/vKWSz5amW+a4cnjg23ojz5iL7rjf08Wg= go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793/go.mod h1:eBhtbxXP1qpW0F6+WxoJ64DM1Mrfx46PHtVxEdkLe0I= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -560,6 +599,7 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -636,6 +676,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -679,6 +720,7 @@ google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd h1:OjndDrsik+Gt+e6 google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= From 00eced338764d6b1615ae8ab4e6aea838ca6e23a Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 16 Feb 2023 22:35:24 +0800 Subject: [PATCH 03/24] Add ReadHeaderTimeout to http.Server Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/server.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index b634391def4..8bfc323ffca 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -184,8 +184,9 @@ func (s *Server) startServer() error { handler, _ := SetUpRestHandler(s.service) s.httpServer = &http.Server{ - Handler: handler, - ReadTimeout: 5 * time.Minute, + Handler: handler, + ReadTimeout: 5 * time.Minute, + ReadHeaderTimeout: 5 * time.Second, } go func() { From f90a9884334dabc6cba7f0636da4b100be25a468 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 16 Feb 2023 22:51:55 +0800 Subject: [PATCH 04/24] fix lint Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index 8bfc323ffca..a6519fc4870 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -210,7 +210,7 @@ func (s *Server) startServer() error { } // NewServer creates a new resource manager server. -func NewServer(name string, ctx context.Context, cfg *Config) *Server { +func NewServer(ctx context.Context, cfg *Config, name string) *Server { return &Server{ name: name, ctx: ctx, @@ -268,7 +268,7 @@ func CreateServerWrapper(cmd *cobra.Command, args []string) { metricutil.Push(&cfg.Metric) ctx, cancel := context.WithCancel(context.Background()) - svr := NewServer("ResourceManager", ctx, cfg) + svr := NewServer(ctx, cfg, "ResourceManager") sc := make(chan os.Signal, 1) signal.Notify(sc, From 216b9e4f1a2ebba7ad45cf3aa9e27c893e068ab5 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Feb 2023 01:20:15 +0800 Subject: [PATCH 05/24] fix test Signed-off-by: lhy1024 --- tests/mcs/resource_manager/server_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mcs/resource_manager/server_test.go b/tests/mcs/resource_manager/server_test.go index 6effe57e418..f4855889dde 100644 --- a/tests/mcs/resource_manager/server_test.go +++ b/tests/mcs/resource_manager/server_test.go @@ -50,7 +50,7 @@ func TestResourceManagerServer(t *testing.T) { cfg.BackendEndpoints = leader.GetAddr() cfg.ListenAddr = "127.0.0.1:8086" - svr := rm.NewServer("ResourceManager", ctx, cfg) + svr := rm.NewServer(ctx, cfg, "ResourceManager") go svr.Run() time.Sleep(2 * time.Second) // wait for server start defer svr.Close() From 34952301dd49e0e52611c4f63e8bfc4f75d716ef Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Feb 2023 11:06:23 +0800 Subject: [PATCH 06/24] address comments Signed-off-by: lhy1024 --- pkg/basicserver/basic_server.go | 8 ++++---- pkg/mcs/resource_manager/server/grpc_service.go | 2 +- pkg/mcs/resource_manager/server/manager.go | 2 +- pkg/mcs/resource_manager/server/server.go | 8 ++++---- pkg/mcs/tso/server/server.go | 8 ++++---- server/server.go | 8 ++++---- tests/mcs/resource_manager/server_test.go | 1 - 7 files changed, 18 insertions(+), 19 deletions(-) diff --git a/pkg/basicserver/basic_server.go b/pkg/basicserver/basic_server.go index 187cd6bf82c..492946dfa88 100644 --- a/pkg/basicserver/basic_server.go +++ b/pkg/basicserver/basic_server.go @@ -37,8 +37,8 @@ type Server interface { GetHTTPClient() *http.Client // AddStartCallback adds a callback in the startServer phase. AddStartCallback(callbacks ...func()) - // IsPrimary returns whether the server is primary, if there is etcd server in the server, it will return whether it is leader. - IsPrimary() bool - // AddPrimaryCallback adds a callback when the server becomes primary, if there is etcd server in the server, it means the server becomes leader. - AddPrimaryCallback(callbacks ...func(context.Context)) + // IsPrimaryOrLeader returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. + IsPrimaryOrLeader() bool + // AddPrimaryOrLeaderCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. + AddPrimaryOrLeaderCallback(callbacks ...func(context.Context)) } diff --git a/pkg/mcs/resource_manager/server/grpc_service.go b/pkg/mcs/resource_manager/server/grpc_service.go index 1a8863f5276..88c061ea767 100644 --- a/pkg/mcs/resource_manager/server/grpc_service.go +++ b/pkg/mcs/resource_manager/server/grpc_service.go @@ -85,7 +85,7 @@ func (s *Service) GetManager() *Manager { } func (s *Service) checkPrimary() error { - if !s.manager.srv.IsPrimary() { + if !s.manager.srv.IsPrimaryOrLeader() { return errNotPrimary } return nil diff --git a/pkg/mcs/resource_manager/server/manager.go b/pkg/mcs/resource_manager/server/manager.go index 773a49fd057..60f6bc82d05 100644 --- a/pkg/mcs/resource_manager/server/manager.go +++ b/pkg/mcs/resource_manager/server/manager.go @@ -67,7 +67,7 @@ func NewManager(srv bs.Server) *Manager { m.srv = srv }) // The second initialization after becoming primary. - srv.AddPrimaryCallback(m.Init) + srv.AddPrimaryOrLeaderCallback(m.Init) return m } diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index a6519fc4870..d8f1d3f2ec4 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -127,14 +127,14 @@ func (s *Server) AddStartCallback(callbacks ...func()) { s.startCallbacks = append(s.startCallbacks, callbacks...) } -// IsPrimary returns whether the server is primary, if there is etcd server in the server, it will return whether it is leader. -func (s *Server) IsPrimary() bool { +// IsPrimaryOrLeader returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. +func (s *Server) IsPrimaryOrLeader() bool { // TODO: implement this return true } -// AddPrimaryCallback adds a callback when the server becomes primary, if there is etcd server in the server, it means the server becomes leader. -func (s *Server) AddPrimaryCallback(callbacks ...func(context.Context)) { +// AddPrimaryOrLeaderCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. +func (s *Server) AddPrimaryOrLeaderCallback(callbacks ...func(context.Context)) { s.primaryCallbacks = append(s.primaryCallbacks, callbacks...) } diff --git a/pkg/mcs/tso/server/server.go b/pkg/mcs/tso/server/server.go index 36b375172b0..6d294638cc6 100644 --- a/pkg/mcs/tso/server/server.go +++ b/pkg/mcs/tso/server/server.go @@ -121,15 +121,15 @@ func (s *Server) AddStartCallback(callbacks ...func()) { s.startCallbacks = append(s.startCallbacks, callbacks...) } -// IsPrimary returns whether the server is primary, if there is etcd server in the server, it will return whether it is leader. -func (s *Server) IsPrimary() bool { +// IsPrimaryOrLeader returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. +func (s *Server) IsPrimaryOrLeader() bool { // TODO: implement this return true } -// AddPrimaryCallback adds a callback when the server becomes primary, if there is etcd server in the server, it means the server becomes leader. +// AddPrimaryOrLeaderCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. // the global TSO allocator after the flag 'enable-local-tso' is set to true. -func (s *Server) AddPrimaryCallback(callbacks ...func(context.Context)) { +func (s *Server) AddPrimaryOrLeaderCallback(callbacks ...func(context.Context)) { // Leave it empty // TODO: implment it when integerating with the Local/Global TSO Allocator. } diff --git a/server/server.go b/server/server.go index e8faf20d9f3..8c691de1934 100644 --- a/server/server.go +++ b/server/server.go @@ -1318,13 +1318,13 @@ func (s *Server) SetReplicationModeConfig(cfg config.ReplicationModeConfig) erro return nil } -// IsPrimary returns whether the server is primary, if there is etcd server in the server, it will return whether it is leader. -func (s *Server) IsPrimary() bool { +// IsPrimaryOrLeader returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. +func (s *Server) IsPrimaryOrLeader() bool { return s.member.IsLeader() } -// AddPrimaryCallback adds a callback when the server becomes primary, if there is etcd server in the server, it means the server becomes leader. -func (s *Server) AddPrimaryCallback(callbacks ...func(context.Context)) { +// AddPrimaryOrLeaderCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. +func (s *Server) AddPrimaryOrLeaderCallback(callbacks ...func(context.Context)) { s.leaderCallbacks = append(s.leaderCallbacks, callbacks...) } diff --git a/tests/mcs/resource_manager/server_test.go b/tests/mcs/resource_manager/server_test.go index f4855889dde..5da769e0cfc 100644 --- a/tests/mcs/resource_manager/server_test.go +++ b/tests/mcs/resource_manager/server_test.go @@ -74,7 +74,6 @@ func TestResourceManagerServer(t *testing.T) { respString, err := io.ReadAll(resp.Body) re.NoError(err) re.Equal("[]", string(respString)) - } { group := &rmpb.ResourceGroup{ From fb5ef97221d1778c2c52f0cd6caa8dd277ec87a5 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Feb 2023 11:16:27 +0800 Subject: [PATCH 07/24] address comments Signed-off-by: lhy1024 --- pkg/basicserver/basic_server.go | 8 ++++---- pkg/mcs/resource_manager/server/grpc_service.go | 2 +- pkg/mcs/resource_manager/server/manager.go | 2 +- pkg/mcs/resource_manager/server/server.go | 10 +++++----- pkg/mcs/tso/server/server.go | 8 ++++---- server/server.go | 8 ++++---- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/pkg/basicserver/basic_server.go b/pkg/basicserver/basic_server.go index 492946dfa88..8b0add6df06 100644 --- a/pkg/basicserver/basic_server.go +++ b/pkg/basicserver/basic_server.go @@ -37,8 +37,8 @@ type Server interface { GetHTTPClient() *http.Client // AddStartCallback adds a callback in the startServer phase. AddStartCallback(callbacks ...func()) - // IsPrimaryOrLeader returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. - IsPrimaryOrLeader() bool - // AddPrimaryOrLeaderCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. - AddPrimaryOrLeaderCallback(callbacks ...func(context.Context)) + // IsServing returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. + IsServing() bool + // AddServiceReadyCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. + AddServiceReadyCallback(callbacks ...func(context.Context)) } diff --git a/pkg/mcs/resource_manager/server/grpc_service.go b/pkg/mcs/resource_manager/server/grpc_service.go index 88c061ea767..f4395597974 100644 --- a/pkg/mcs/resource_manager/server/grpc_service.go +++ b/pkg/mcs/resource_manager/server/grpc_service.go @@ -85,7 +85,7 @@ func (s *Service) GetManager() *Manager { } func (s *Service) checkPrimary() error { - if !s.manager.srv.IsPrimaryOrLeader() { + if !s.manager.srv.IsServing() { return errNotPrimary } return nil diff --git a/pkg/mcs/resource_manager/server/manager.go b/pkg/mcs/resource_manager/server/manager.go index 60f6bc82d05..84245ba34ff 100644 --- a/pkg/mcs/resource_manager/server/manager.go +++ b/pkg/mcs/resource_manager/server/manager.go @@ -67,7 +67,7 @@ func NewManager(srv bs.Server) *Manager { m.srv = srv }) // The second initialization after becoming primary. - srv.AddPrimaryOrLeaderCallback(m.Init) + srv.AddServiceReadyCallback(m.Init) return m } diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index d8f1d3f2ec4..a759916241f 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -127,14 +127,14 @@ func (s *Server) AddStartCallback(callbacks ...func()) { s.startCallbacks = append(s.startCallbacks, callbacks...) } -// IsPrimaryOrLeader returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. -func (s *Server) IsPrimaryOrLeader() bool { +// IsServing returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. +func (s *Server) IsServing() bool { // TODO: implement this return true } -// AddPrimaryOrLeaderCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. -func (s *Server) AddPrimaryOrLeaderCallback(callbacks ...func(context.Context)) { +// AddServiceReadyCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. +func (s *Server) AddServiceReadyCallback(callbacks ...func(context.Context)) { s.primaryCallbacks = append(s.primaryCallbacks, callbacks...) } @@ -189,6 +189,7 @@ func (s *Server) startServer() error { ReadHeaderTimeout: 5 * time.Second, } + s.serverLoopWg.Add(2) go func() { defer s.serverLoopWg.Done() s.grpcServer.Serve(grpcL) @@ -197,7 +198,6 @@ func (s *Server) startServer() error { defer s.serverLoopWg.Done() s.httpServer.Serve(httpL) }() - s.serverLoopWg.Add(2) // Run callbacks log.Info("triggering the start callback functions") diff --git a/pkg/mcs/tso/server/server.go b/pkg/mcs/tso/server/server.go index 6d294638cc6..6cb407bf424 100644 --- a/pkg/mcs/tso/server/server.go +++ b/pkg/mcs/tso/server/server.go @@ -121,15 +121,15 @@ func (s *Server) AddStartCallback(callbacks ...func()) { s.startCallbacks = append(s.startCallbacks, callbacks...) } -// IsPrimaryOrLeader returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. -func (s *Server) IsPrimaryOrLeader() bool { +// IsServing returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. +func (s *Server) IsServing() bool { // TODO: implement this return true } -// AddPrimaryOrLeaderCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. +// AddServiceReadyCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. // the global TSO allocator after the flag 'enable-local-tso' is set to true. -func (s *Server) AddPrimaryOrLeaderCallback(callbacks ...func(context.Context)) { +func (s *Server) AddServiceReadyCallback(callbacks ...func(context.Context)) { // Leave it empty // TODO: implment it when integerating with the Local/Global TSO Allocator. } diff --git a/server/server.go b/server/server.go index 8c691de1934..2fc6f682077 100644 --- a/server/server.go +++ b/server/server.go @@ -1318,13 +1318,13 @@ func (s *Server) SetReplicationModeConfig(cfg config.ReplicationModeConfig) erro return nil } -// IsPrimaryOrLeader returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. -func (s *Server) IsPrimaryOrLeader() bool { +// IsServing returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. +func (s *Server) IsServing() bool { return s.member.IsLeader() } -// AddPrimaryOrLeaderCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. -func (s *Server) AddPrimaryOrLeaderCallback(callbacks ...func(context.Context)) { +// AddServiceReadyCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. +func (s *Server) AddServiceReadyCallback(callbacks ...func(context.Context)) { s.leaderCallbacks = append(s.leaderCallbacks, callbacks...) } From 136fc947489bd5963fd4ef6a87fe3d41be3718b2 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Feb 2023 11:31:42 +0800 Subject: [PATCH 08/24] rename function Signed-off-by: lhy1024 --- pkg/basicserver/basic_server.go | 2 +- .../resource_manager/server/grpc_service.go | 20 +++++++++---------- pkg/mcs/resource_manager/server/manager.go | 2 +- pkg/mcs/resource_manager/server/server.go | 2 +- pkg/mcs/tso/server/server.go | 2 +- server/server.go | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pkg/basicserver/basic_server.go b/pkg/basicserver/basic_server.go index 8b0add6df06..4fcaad96741 100644 --- a/pkg/basicserver/basic_server.go +++ b/pkg/basicserver/basic_server.go @@ -39,6 +39,6 @@ type Server interface { AddStartCallback(callbacks ...func()) // IsServing returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. IsServing() bool - // AddServiceReadyCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. + // AddServiceReadyCallback adds the callback function when the server becomes the leader, if there is embedded etcd, or the primary otherwise. AddServiceReadyCallback(callbacks ...func(context.Context)) } diff --git a/pkg/mcs/resource_manager/server/grpc_service.go b/pkg/mcs/resource_manager/server/grpc_service.go index f4395597974..3e3f7232031 100644 --- a/pkg/mcs/resource_manager/server/grpc_service.go +++ b/pkg/mcs/resource_manager/server/grpc_service.go @@ -33,8 +33,8 @@ import ( ) var ( - // errNotPrimary is returned when current server is not primary. - errNotPrimary = status.Errorf(codes.Unavailable, "not primary") + // errNotServing is returned when current server is not serving. + errNotServing = status.Errorf(codes.Unavailable, "server is not serving") ) var _ rmpb.ResourceManagerServer = (*Service)(nil) @@ -84,16 +84,16 @@ func (s *Service) GetManager() *Manager { return s.manager } -func (s *Service) checkPrimary() error { +func (s *Service) checkServing() error { if !s.manager.srv.IsServing() { - return errNotPrimary + return errNotServing } return nil } // GetResourceGroup implements ResourceManagerServer.GetResourceGroup. func (s *Service) GetResourceGroup(ctx context.Context, req *rmpb.GetResourceGroupRequest) (*rmpb.GetResourceGroupResponse, error) { - if err := s.checkPrimary(); err != nil { + if err := s.checkServing(); err != nil { return nil, err } rg := s.manager.GetResourceGroup(req.ResourceGroupName) @@ -107,7 +107,7 @@ func (s *Service) GetResourceGroup(ctx context.Context, req *rmpb.GetResourceGro // ListResourceGroups implements ResourceManagerServer.ListResourceGroups. func (s *Service) ListResourceGroups(ctx context.Context, req *rmpb.ListResourceGroupsRequest) (*rmpb.ListResourceGroupsResponse, error) { - if err := s.checkPrimary(); err != nil { + if err := s.checkServing(); err != nil { return nil, err } groups := s.manager.GetResourceGroupList() @@ -122,7 +122,7 @@ func (s *Service) ListResourceGroups(ctx context.Context, req *rmpb.ListResource // AddResourceGroup implements ResourceManagerServer.AddResourceGroup. func (s *Service) AddResourceGroup(ctx context.Context, req *rmpb.PutResourceGroupRequest) (*rmpb.PutResourceGroupResponse, error) { - if err := s.checkPrimary(); err != nil { + if err := s.checkServing(); err != nil { return nil, err } rg := FromProtoResourceGroup(req.GetGroup()) @@ -135,7 +135,7 @@ func (s *Service) AddResourceGroup(ctx context.Context, req *rmpb.PutResourceGro // DeleteResourceGroup implements ResourceManagerServer.DeleteResourceGroup. func (s *Service) DeleteResourceGroup(ctx context.Context, req *rmpb.DeleteResourceGroupRequest) (*rmpb.DeleteResourceGroupResponse, error) { - if err := s.checkPrimary(); err != nil { + if err := s.checkServing(); err != nil { return nil, err } err := s.manager.DeleteResourceGroup(req.ResourceGroupName) @@ -147,7 +147,7 @@ func (s *Service) DeleteResourceGroup(ctx context.Context, req *rmpb.DeleteResou // ModifyResourceGroup implements ResourceManagerServer.ModifyResourceGroup. func (s *Service) ModifyResourceGroup(ctx context.Context, req *rmpb.PutResourceGroupRequest) (*rmpb.PutResourceGroupResponse, error) { - if err := s.checkPrimary(); err != nil { + if err := s.checkServing(); err != nil { return nil, err } err := s.manager.ModifyResourceGroup(req.GetGroup()) @@ -172,7 +172,7 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu if err != nil { return errors.WithStack(err) } - if err := s.checkPrimary(); err != nil { + if err := s.checkServing(); err != nil { return err } targetPeriodMs := request.GetTargetRequestPeriodMs() diff --git a/pkg/mcs/resource_manager/server/manager.go b/pkg/mcs/resource_manager/server/manager.go index 84245ba34ff..c926ae898ff 100644 --- a/pkg/mcs/resource_manager/server/manager.go +++ b/pkg/mcs/resource_manager/server/manager.go @@ -66,7 +66,7 @@ func NewManager(srv bs.Server) *Manager { ) m.srv = srv }) - // The second initialization after becoming primary. + // The second initialization after becoming serving. srv.AddServiceReadyCallback(m.Init) return m } diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index a759916241f..74d00a82a39 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -133,7 +133,7 @@ func (s *Server) IsServing() bool { return true } -// AddServiceReadyCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. +// AddServiceReadyCallback adds the callback function when the server becomes the leader, if there is embedded etcd, or the primary otherwise. func (s *Server) AddServiceReadyCallback(callbacks ...func(context.Context)) { s.primaryCallbacks = append(s.primaryCallbacks, callbacks...) } diff --git a/pkg/mcs/tso/server/server.go b/pkg/mcs/tso/server/server.go index 6cb407bf424..8f08f43d766 100644 --- a/pkg/mcs/tso/server/server.go +++ b/pkg/mcs/tso/server/server.go @@ -127,7 +127,7 @@ func (s *Server) IsServing() bool { return true } -// AddServiceReadyCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. +// AddServiceReadyCallback adds the callback function when the server becomes the leader, if there is embedded etcd, or the primary otherwise. // the global TSO allocator after the flag 'enable-local-tso' is set to true. func (s *Server) AddServiceReadyCallback(callbacks ...func(context.Context)) { // Leave it empty diff --git a/server/server.go b/server/server.go index 2fc6f682077..39f40f88aee 100644 --- a/server/server.go +++ b/server/server.go @@ -1323,7 +1323,7 @@ func (s *Server) IsServing() bool { return s.member.IsLeader() } -// AddServiceReadyCallback adds a callback when the server becomes the leader, if there is embedded etcd, or the primary otherwise. +// AddServiceReadyCallback adds the callback function when the server becomes the leader, if there is embedded etcd, or the primary otherwise. func (s *Server) AddServiceReadyCallback(callbacks ...func(context.Context)) { s.leaderCallbacks = append(s.leaderCallbacks, callbacks...) } From fc02ece90a51ba530fddaa5771431208758fb969 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Feb 2023 12:15:13 +0800 Subject: [PATCH 09/24] use version info Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/server.go | 28 +++++++---------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index 74d00a82a39..f1912b216a7 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -17,7 +17,6 @@ package server import ( "context" "crypto/tls" - "flag" "net" "net/http" "net/url" @@ -29,7 +28,6 @@ import ( "time" grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/soheilhy/cmux" "github.com/spf13/cobra" @@ -37,6 +35,7 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/metricutil" + "github.com/tikv/pd/pkg/versioninfo" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" "google.golang.org/grpc" @@ -224,30 +223,19 @@ func CreateServerWrapper(cmd *cobra.Command, args []string) { cfg := NewConfig() flagSet := cmd.Flags() err := cfg.Parse(flagSet) + defer logutil.LogPanic() + if err != nil { cmd.Println(err) return } - printVersion, err := flagSet.GetBool("version") - if err != nil { + if printVersion, err := flagSet.GetBool("version"); err != nil { cmd.Println(err) return - } - if printVersion { - // TODO: support printing resource manager server info - // server.PrintPDInfo() - exit(0) - } - - defer logutil.LogPanic() - - switch errors.Cause(err) { - case nil: - case flag.ErrHelp: + } else if printVersion { + versioninfo.Print() exit(0) - default: - log.Fatal("parse cmd flags error", errs.ZapError(err)) } // New zap logger @@ -260,8 +248,8 @@ func CreateServerWrapper(cmd *cobra.Command, args []string) { // Flushing any buffered log entries defer log.Sync() - // TODO: support printing resource manager server info - // server.LogPDInfo() + versioninfo.Log("TSO") + log.Info("TSO Config", zap.Reflect("config", cfg)) grpcprometheus.EnableHandlingTimeHistogram() From d834b1139766d48efa053a7efc687d988f0bb07e Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Feb 2023 14:07:24 +0800 Subject: [PATCH 10/24] address comments Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/server.go | 10 ++++++---- tests/mcs/resource_manager/server_test.go | 6 ++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index f1912b216a7..2647c5a4256 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -41,6 +41,8 @@ import ( "google.golang.org/grpc" ) +const tcp = "tcp" + // Server is the resource manager server, and it implements bs.Server. // nolint type Server struct { @@ -128,8 +130,8 @@ func (s *Server) AddStartCallback(callbacks ...func()) { // IsServing returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. func (s *Server) IsServing() bool { - // TODO: implement this - return true + // TODO: implement this function with primary. + return s.isServing == 1 } // AddServiceReadyCallback adds the callback function when the server becomes the leader, if there is embedded etcd, or the primary otherwise. @@ -156,13 +158,13 @@ func (s *Server) startServer() error { return err } if tlsConfig != nil { - l, err := tls.Listen("tcp", s.cfg.ListenAddr, tlsConfig) + l, err := tls.Listen(tcp, s.cfg.ListenAddr, tlsConfig) if err != nil { return err } s.mux = cmux.New(l) } else { - l, err := net.Listen("tcp", s.cfg.ListenAddr) + l, err := net.Listen(tcp, s.cfg.ListenAddr) if err != nil { return err } diff --git a/tests/mcs/resource_manager/server_test.go b/tests/mcs/resource_manager/server_test.go index 5da769e0cfc..da1b7392bfe 100644 --- a/tests/mcs/resource_manager/server_test.go +++ b/tests/mcs/resource_manager/server_test.go @@ -21,11 +21,11 @@ import ( "net/http" "strings" "testing" - "time" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/stretchr/testify/require" rm "github.com/tikv/pd/pkg/mcs/resource_manager/server" + "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/tests" "google.golang.org/grpc" "google.golang.org/grpc/test/grpc_testing" @@ -52,7 +52,9 @@ func TestResourceManagerServer(t *testing.T) { svr := rm.NewServer(ctx, cfg, "ResourceManager") go svr.Run() - time.Sleep(2 * time.Second) // wait for server start + testutil.Eventually(re, func() bool { + return svr.IsServing() + }) defer svr.Close() // Test registered GRPC Service From f07206e0d11292277231fb75b0c2ae9d918c93da Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Feb 2023 14:18:57 +0800 Subject: [PATCH 11/24] add multiple endpoints parse Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/server.go | 25 ++++++++++++++++------- tests/mcs/resource_manager/server_test.go | 3 ++- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index 2647c5a4256..fb275343c0c 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -22,12 +22,14 @@ import ( "net/url" "os" "os/signal" + "strings" "sync" "sync/atomic" "syscall" "time" grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/soheilhy/cmux" "github.com/spf13/cobra" @@ -49,9 +51,10 @@ type Server struct { cfg *Config mux cmux.CMux // Server state. 0 is not serving, 1 is serving. - isServing int64 - ctx context.Context - name string + isServing int64 + ctx context.Context + name string + backendUrls []*url.URL etcdClient *clientv3.Client httpClient *http.Client @@ -144,11 +147,19 @@ func (s *Server) initClient() error { if err != nil { return err } - endpoint, err := url.Parse(s.cfg.BackendEndpoints) - if err != nil { - return err + endpoints := strings.Split(s.cfg.BackendEndpoints, ",") + for _, endpoint := range endpoints { + e, err := url.Parse(endpoint) + if err != nil { + return err + } + s.backendUrls = append(s.backendUrls, e) + } + if len(s.backendUrls) == 0 { + return errs.ErrURLParse.Wrap(errors.New("no backend url found")) } - s.etcdClient, s.httpClient, err = etcdutil.CreateClients(tlsConfig, []url.URL{*endpoint}) + s.etcdClient, s.httpClient, err = etcdutil.CreateClients(tlsConfig, []url.URL{*s.backendUrls[0]}) + // TODO: support multi-endpoints. return err } diff --git a/tests/mcs/resource_manager/server_test.go b/tests/mcs/resource_manager/server_test.go index da1b7392bfe..02ce2e01be6 100644 --- a/tests/mcs/resource_manager/server_test.go +++ b/tests/mcs/resource_manager/server_test.go @@ -21,6 +21,7 @@ import ( "net/http" "strings" "testing" + "time" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/stretchr/testify/require" @@ -54,7 +55,7 @@ func TestResourceManagerServer(t *testing.T) { go svr.Run() testutil.Eventually(re, func() bool { return svr.IsServing() - }) + }, testutil.WithWaitFor(5*time.Second)) defer svr.Close() // Test registered GRPC Service From 3f8328c47ec3beabfdbdbc2686689fbce9c8f721 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Feb 2023 14:38:42 +0800 Subject: [PATCH 12/24] fix test Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/server.go | 2 +- tests/mcs/resource_manager/server_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index fb275343c0c..1898b98caa4 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -134,7 +134,7 @@ func (s *Server) AddStartCallback(callbacks ...func()) { // IsServing returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. func (s *Server) IsServing() bool { // TODO: implement this function with primary. - return s.isServing == 1 + return atomic.LoadInt64(&s.isServing) == 1 } // AddServiceReadyCallback adds the callback function when the server becomes the leader, if there is embedded etcd, or the primary otherwise. diff --git a/tests/mcs/resource_manager/server_test.go b/tests/mcs/resource_manager/server_test.go index 02ce2e01be6..53f83693e46 100644 --- a/tests/mcs/resource_manager/server_test.go +++ b/tests/mcs/resource_manager/server_test.go @@ -55,7 +55,7 @@ func TestResourceManagerServer(t *testing.T) { go svr.Run() testutil.Eventually(re, func() bool { return svr.IsServing() - }, testutil.WithWaitFor(5*time.Second)) + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) defer svr.Close() // Test registered GRPC Service From c7a2e576900d948105823fea5fafa4d99100bbad Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Feb 2023 14:47:55 +0800 Subject: [PATCH 13/24] address comments Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/grpc_service.go | 6 +++--- pkg/mcs/resource_manager/server/server.go | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/mcs/resource_manager/server/grpc_service.go b/pkg/mcs/resource_manager/server/grpc_service.go index 3e3f7232031..f57c2d3b41a 100644 --- a/pkg/mcs/resource_manager/server/grpc_service.go +++ b/pkg/mcs/resource_manager/server/grpc_service.go @@ -33,8 +33,8 @@ import ( ) var ( - // errNotServing is returned when current server is not serving. - errNotServing = status.Errorf(codes.Unavailable, "server is not serving") + // errNotLeader is returned when current server is not the leader. + errNotLeader = status.Errorf(codes.Unavailable, "not leader") ) var _ rmpb.ResourceManagerServer = (*Service)(nil) @@ -86,7 +86,7 @@ func (s *Service) GetManager() *Manager { func (s *Service) checkServing() error { if !s.manager.srv.IsServing() { - return errNotServing + return errNotLeader } return nil } diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index 1898b98caa4..b9066cf8583 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -49,7 +49,6 @@ const tcp = "tcp" // nolint type Server struct { cfg *Config - mux cmux.CMux // Server state. 0 is not serving, 1 is serving. isServing int64 ctx context.Context @@ -89,7 +88,7 @@ func (s *Server) Run() error { if err := s.startServer(); err != nil { return err } - return s.mux.Serve() + return nil } // Close closes the server. @@ -164,6 +163,7 @@ func (s *Server) initClient() error { } func (s *Server) startServer() error { + var mux cmux.CMux tlsConfig, err := s.cfg.Security.ToTLSConfig() if err != nil { return err @@ -173,17 +173,17 @@ func (s *Server) startServer() error { if err != nil { return err } - s.mux = cmux.New(l) + mux = cmux.New(l) } else { l, err := net.Listen(tcp, s.cfg.ListenAddr) if err != nil { return err } - s.mux = cmux.New(l) + mux = cmux.New(l) } - grpcL := s.mux.MatchWithWriters(cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc")) - httpL := s.mux.Match(cmux.Any()) + grpcL := mux.MatchWithWriters(cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc")) + httpL := mux.Match(cmux.Any()) manager := NewManager(s) s.service = &Service{ @@ -218,7 +218,7 @@ func (s *Server) startServer() error { } // Server has started. atomic.StoreInt64(&s.isServing, 1) - return nil + return mux.Serve() } // NewServer creates a new resource manager server. From f144e7cbadc3f00fae1bce142cff35e88ad32042 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Feb 2023 15:01:26 +0800 Subject: [PATCH 14/24] address comments Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/server.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index b9066cf8583..eb0be9e65ee 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -85,10 +85,7 @@ func (s *Server) Run() error { if err := s.initClient(); err != nil { return err } - if err := s.startServer(); err != nil { - return err - } - return nil + return s.startServer() } // Close closes the server. @@ -142,6 +139,7 @@ func (s *Server) AddServiceReadyCallback(callbacks ...func(context.Context)) { } func (s *Server) initClient() error { + // TODO: We need to keep all backend endpoints and keep updating them to the latest. Once one of them failed, need to try another one. tlsConfig, err := s.cfg.Security.ToTLSConfig() if err != nil { return err @@ -158,7 +156,6 @@ func (s *Server) initClient() error { return errs.ErrURLParse.Wrap(errors.New("no backend url found")) } s.etcdClient, s.httpClient, err = etcdutil.CreateClients(tlsConfig, []url.URL{*s.backendUrls[0]}) - // TODO: support multi-endpoints. return err } From 07ad1a761ebfe35c8c0e38b4d27dee56aa7e7d9b Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Feb 2023 15:36:12 +0800 Subject: [PATCH 15/24] address comments Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/server.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index eb0be9e65ee..f33af83f421 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -29,7 +29,6 @@ import ( "time" grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/soheilhy/cmux" "github.com/spf13/cobra" @@ -39,6 +38,7 @@ import ( "github.com/tikv/pd/pkg/utils/metricutil" "github.com/tikv/pd/pkg/versioninfo" "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/pkg/types" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -53,7 +53,7 @@ type Server struct { isServing int64 ctx context.Context name string - backendUrls []*url.URL + backendUrls []url.URL etcdClient *clientv3.Client httpClient *http.Client @@ -144,18 +144,12 @@ func (s *Server) initClient() error { if err != nil { return err } - endpoints := strings.Split(s.cfg.BackendEndpoints, ",") - for _, endpoint := range endpoints { - e, err := url.Parse(endpoint) - if err != nil { - return err - } - s.backendUrls = append(s.backendUrls, e) - } - if len(s.backendUrls) == 0 { - return errs.ErrURLParse.Wrap(errors.New("no backend url found")) + u, err := types.NewURLs(strings.Split(s.cfg.BackendEndpoints, ",")) + if err != nil { + return err } - s.etcdClient, s.httpClient, err = etcdutil.CreateClients(tlsConfig, []url.URL{*s.backendUrls[0]}) + s.backendUrls = []url.URL(u) + s.etcdClient, s.httpClient, err = etcdutil.CreateClients(tlsConfig, s.backendUrls) return err } From ea4c5bbd36ef18b387fb25ebebce155cfb2f459c Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Feb 2023 18:49:30 +0800 Subject: [PATCH 16/24] address comments Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/config.go | 3 +-- pkg/mcs/resource_manager/server/server.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/mcs/resource_manager/server/config.go b/pkg/mcs/resource_manager/server/config.go index dcacbce877d..095a987047c 100644 --- a/pkg/mcs/resource_manager/server/config.go +++ b/pkg/mcs/resource_manager/server/config.go @@ -1,4 +1,4 @@ -// Copyright 2022 TiKV Project Authors. +// Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ import ( "github.com/tikv/pd/pkg/encryption" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/metricutil" - "go.uber.org/zap" ) diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index f33af83f421..4618f38e51d 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -252,8 +252,8 @@ func CreateServerWrapper(cmd *cobra.Command, args []string) { // Flushing any buffered log entries defer log.Sync() - versioninfo.Log("TSO") - log.Info("TSO Config", zap.Reflect("config", cfg)) + versioninfo.Log("resource manager") + log.Info("resource manager config", zap.Reflect("config", cfg)) grpcprometheus.EnableHandlingTimeHistogram() From f6cefeb57d4313d7eec38fd4290f8c881db1ee23 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Feb 2023 18:52:21 +0800 Subject: [PATCH 17/24] rename Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mcs/resource_manager/server/config.go b/pkg/mcs/resource_manager/server/config.go index 095a987047c..3057a7589b4 100644 --- a/pkg/mcs/resource_manager/server/config.go +++ b/pkg/mcs/resource_manager/server/config.go @@ -25,7 +25,7 @@ import ( "go.uber.org/zap" ) -// Config is the configuration for the TSO. +// Config is the configuration for the resource manager. type Config struct { BackendEndpoints string `toml:"backend-endpoints" json:"backend-endpoints"` ListenAddr string `toml:"listen-addr" json:"listen-addr"` From 98a6fdb2ba3b209b25510fb34803e077043585ae Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Mon, 20 Feb 2023 15:35:23 +0800 Subject: [PATCH 18/24] update related to #6008 Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/config.go | 140 +++++++++++++++----- pkg/mcs/resource_manager/server/server.go | 152 +++++++++++++++------- pkg/mcs/tso/server/server.go | 1 - 3 files changed, 215 insertions(+), 78 deletions(-) diff --git a/pkg/mcs/resource_manager/server/config.go b/pkg/mcs/resource_manager/server/config.go index 3057a7589b4..8f7b58b2cdb 100644 --- a/pkg/mcs/resource_manager/server/config.go +++ b/pkg/mcs/resource_manager/server/config.go @@ -15,30 +15,50 @@ package server import ( + "fmt" + "os" + "path/filepath" + "strings" + "github.com/BurntSushi/toml" "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/spf13/pflag" - "github.com/tikv/pd/pkg/encryption" + "github.com/tikv/pd/pkg/utils/configutil" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/metricutil" "go.uber.org/zap" ) +const ( + defaultName = "Resource Manager" + defaultBackendEndpoints = "http://127.0.0.1:2379" + defaultListenAddr = "http://127.0.0.1:2382" + defaultEnableGRPCGateway = true + + defaultLogFormat = "text" + defaultDisableErrorVerbose = true +) + // Config is the configuration for the resource manager. type Config struct { - BackendEndpoints string `toml:"backend-endpoints" json:"backend-endpoints"` - ListenAddr string `toml:"listen-addr" json:"listen-addr"` + BackendEndpoints string `toml:"backend-endpoints" json:"backend-endpoints"` + ListenAddr string `toml:"listen-addr" json:"listen-addr"` + Name string `toml:"name" json:"name"` + DataDir string `toml:"data-dir" json:"data-dir"` + EnableGRPCGateway bool `json:"enable-grpc-gateway"` Metric metricutil.MetricConfig `toml:"metric" json:"metric"` + // WarningMsgs contains all warnings during parsing. + WarningMsgs []string + // Log related config. Log log.Config `toml:"log" json:"log"` Logger *zap.Logger LogProps *log.ZapProperties - - Security SecurityConfig `toml:"security" json:"security"` + Security configutil.SecurityConfig `toml:"security" json:"security"` } // NewConfig creates a new config. @@ -49,43 +69,103 @@ func NewConfig() *Config { // Parse parses flag definitions from the argument list. func (c *Config) Parse(flagSet *pflag.FlagSet) error { // Load config file if specified. + var ( + meta *toml.MetaData + err error + ) if configFile, _ := flagSet.GetString("config"); configFile != "" { - _, err := c.configFromFile(configFile) + meta, err = configutil.ConfigFromFile(c, configFile) if err != nil { return err } } - // ignore the error check here - adjustCommandlineString(flagSet, &c.Log.Level, "log-level") - adjustCommandlineString(flagSet, &c.Log.File.Filename, "log-file") - adjustCommandlineString(flagSet, &c.Metric.PushAddress, "metrics-addr") - adjustCommandlineString(flagSet, &c.Security.CAPath, "cacert") - adjustCommandlineString(flagSet, &c.Security.CertPath, "cert") - adjustCommandlineString(flagSet, &c.Security.KeyPath, "key") - adjustCommandlineString(flagSet, &c.BackendEndpoints, "backend-endpoints") - adjustCommandlineString(flagSet, &c.ListenAddr, "listen-addr") - - // TODO: Implement the main function body + // Ignore the error check here + configutil.AdjustCommandlineString(flagSet, &c.Log.Level, "log-level") + configutil.AdjustCommandlineString(flagSet, &c.Log.File.Filename, "log-file") + configutil.AdjustCommandlineString(flagSet, &c.Metric.PushAddress, "metrics-addr") + configutil.AdjustCommandlineString(flagSet, &c.Security.CAPath, "cacert") + configutil.AdjustCommandlineString(flagSet, &c.Security.CertPath, "cert") + configutil.AdjustCommandlineString(flagSet, &c.Security.KeyPath, "key") + configutil.AdjustCommandlineString(flagSet, &c.BackendEndpoints, "backend-endpoints") + configutil.AdjustCommandlineString(flagSet, &c.ListenAddr, "listen-addr") + + return c.Adjust(meta, false) +} + +// Adjust is used to adjust the PD configurations. +func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { + configMetaData := configutil.NewConfigMetadata(meta) + if err := configMetaData.CheckUndecoded(); err != nil { + c.WarningMsgs = append(c.WarningMsgs, err.Error()) + } + + if c.Name == "" { + hostname, err := os.Hostname() + if err != nil { + return err + } + configutil.AdjustString(&c.Name, fmt.Sprintf("%s-%s", defaultName, hostname)) + } + configutil.AdjustString(&c.DataDir, fmt.Sprintf("default.%s", c.Name)) + adjustPath(&c.DataDir) + + if err := c.Validate(); err != nil { + return err + } + + configutil.AdjustString(&c.BackendEndpoints, defaultBackendEndpoints) + configutil.AdjustString(&c.ListenAddr, defaultListenAddr) + + if !configMetaData.IsDefined("enable-grpc-gateway") { + c.EnableGRPCGateway = defaultEnableGRPCGateway + } + + c.adjustLog(configMetaData.Child("log")) + c.Security.Encryption.Adjust() + + if len(c.Log.Format) == 0 { + c.Log.Format = defaultLogFormat + } + return nil } -// configFromFile loads config from file. -func (c *Config) configFromFile(path string) (*toml.MetaData, error) { - meta, err := toml.DecodeFile(path, c) - return &meta, errors.WithStack(err) +func adjustPath(p *string) { + absPath, err := filepath.Abs(*p) + if err == nil { + *p = absPath + } +} + +func (c *Config) adjustLog(meta *configutil.ConfigMetaData) { + if !meta.IsDefined("disable-error-verbose") { + c.Log.DisableErrorVerbose = defaultDisableErrorVerbose + } } -// SecurityConfig indicates the security configuration for pd server -type SecurityConfig struct { - grpcutil.TLSConfig - // RedactInfoLog indicates that whether enabling redact log - RedactInfoLog bool `toml:"redact-info-log" json:"redact-info-log"` - Encryption encryption.Config `toml:"encryption" json:"encryption"` +// GetTLSConfig returns the TLS config. +func (c *Config) GetTLSConfig() *grpcutil.TLSConfig { + return &c.Security.TLSConfig } -func adjustCommandlineString(flagSet *pflag.FlagSet, v *string, name string) { - if value, _ := flagSet.GetString(name); value != "" { - *v = value +// Validate is used to validate if some configurations are right. +func (c *Config) Validate() error { + dataDir, err := filepath.Abs(c.DataDir) + if err != nil { + return errors.WithStack(err) + } + logFile, err := filepath.Abs(c.Log.File.Filename) + if err != nil { + return errors.WithStack(err) } + rel, err := filepath.Rel(dataDir, filepath.Dir(logFile)) + if err != nil { + return errors.WithStack(err) + } + if !strings.HasPrefix(rel, "..") { + return errors.New("log directory shouldn't be the subdirectory of data directory") + } + + return nil } diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index 4618f38e51d..1dd8c2f1a1e 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -43,26 +43,35 @@ import ( "google.golang.org/grpc" ) -const tcp = "tcp" +const ( + tcp = "tcp" + // defaultGRPCGracefulStopTimeout is the default timeout to wait for grpc server to gracefully stop + defaultGRPCGracefulStopTimeout = 5 * time.Second + // defaultHTTPGracefulShutdownTimeout is the default timeout to wait for http server to gracefully shutdown + defaultHTTPGracefulShutdownTimeout = 300 * time.Second +) // Server is the resource manager server, and it implements bs.Server. // nolint type Server struct { - cfg *Config // Server state. 0 is not serving, 1 is serving. - isServing int64 - ctx context.Context + isServing int64 + + ctx context.Context + serverLoopCtx context.Context + serverLoopCancel func() + serverLoopWg sync.WaitGroup + + cfg *Config name string backendUrls []url.URL etcdClient *clientv3.Client httpClient *http.Client - grpcServer *grpc.Server - httpServer *http.Server - service *Service + muxListener net.Listener + service *Service - serverLoopWg sync.WaitGroup // Callback functions for different stages // startCallbacks will be called after the server is started. startCallbacks []func() @@ -85,6 +94,7 @@ func (s *Server) Run() error { if err := s.initClient(); err != nil { return err } + s.serverLoopCtx, s.serverLoopCancel = context.WithCancel(s.ctx) return s.startServer() } @@ -95,10 +105,10 @@ func (s *Server) Close() { return } - log.Info("closing server") - - s.grpcServer.Stop() - s.httpServer.Shutdown(s.ctx) + log.Info("closing resource manager server ...") + // TODO: double check when muxListener is closed, grpc.Server.serve() and http.Server.serve() + // will also close with error cmux.ErrListenerClosed. + s.muxListener.Close() s.serverLoopWg.Wait() if s.etcdClient != nil { @@ -110,6 +120,8 @@ func (s *Server) Close() { if s.httpClient != nil { s.httpClient.CloseIdleConnections() } + + log.Info("resource manager server is closed") } // GetClient returns builtin etcd client. @@ -133,6 +145,11 @@ func (s *Server) IsServing() bool { return atomic.LoadInt64(&s.isServing) == 1 } +// IsClosed checks if the server loop is closed +func (s *Server) IsClosed() bool { + return atomic.LoadInt64(&s.isServing) == 0 +} + // AddServiceReadyCallback adds the callback function when the server becomes the leader, if there is embedded etcd, or the primary otherwise. func (s *Server) AddServiceReadyCallback(callbacks ...func(context.Context)) { s.primaryCallbacks = append(s.primaryCallbacks, callbacks...) @@ -153,63 +170,104 @@ func (s *Server) initClient() error { return err } -func (s *Server) startServer() error { - var mux cmux.CMux - tlsConfig, err := s.cfg.Security.ToTLSConfig() - if err != nil { - return err +func (s *Server) startGRPCServer(l net.Listener) { + defer s.serverLoopWg.Done() + + gs := grpc.NewServer() + s.service.RegisterGRPCService(gs) + err := gs.Serve(l) + if err != cmux.ErrListenerClosed { + log.Fatal("grpc server stops serving unexpectedly.", errs.ZapError(err)) } - if tlsConfig != nil { - l, err := tls.Listen(tcp, s.cfg.ListenAddr, tlsConfig) - if err != nil { - return err - } - mux = cmux.New(l) - } else { - l, err := net.Listen(tcp, s.cfg.ListenAddr) - if err != nil { - return err - } - mux = cmux.New(l) + log.Info("gRPC server stop serving.", errs.ZapError(err)) + + // Attempt graceful stop (waits for pending RPCs), but force a stop if + // it doesn't happen in a reasonable amount of time. + done := make(chan struct{}) + go func() { + gs.GracefulStop() + close(done) + }() + select { + case <-done: + case <-time.After(defaultGRPCGracefulStopTimeout): + log.Info("stopping grpc gracefully is taking longer than expected and force stopping now.", zap.Duration("default", defaultGRPCGracefulStopTimeout)) + gs.Stop() } +} + +func (s *Server) startHTTPServer(l net.Listener) { + defer s.serverLoopWg.Done() + handler, _ := SetUpRestHandler(s.service) + hs := &http.Server{ + Handler: handler, + ReadTimeout: 5 * time.Minute, + ReadHeaderTimeout: 5 * time.Second, + } + err := hs.Serve(l) + if err != cmux.ErrListenerClosed { + log.Fatal("http server stops serving unexpectedly.", errs.ZapError(err)) + } + log.Info("http server stop serving", errs.ZapError(err)) + + ctx, cancel := context.WithTimeout(context.Background(), defaultHTTPGracefulShutdownTimeout) + defer cancel() + err = hs.Shutdown(ctx) + log.Info("all http(s) requests finished.") + if err != nil { + log.Error("http server shutdown encountered problem.", errs.ZapError(err)) + } +} + +func (s *Server) startGRPCAndHTTPServers(l net.Listener) { + defer s.serverLoopWg.Done() + + mux := cmux.New(l) grpcL := mux.MatchWithWriters(cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc")) httpL := mux.Match(cmux.Any()) + s.serverLoopWg.Add(2) + go s.startGRPCServer(grpcL) + go s.startHTTPServer(httpL) + + if err := mux.Serve(); !strings.Contains(err.Error(), "use of closed network connection") { + log.Fatal("mux stop serving unexpectedly.", errs.ZapError(err)) + } +} + +func (s *Server) startServer() error { manager := NewManager(s) s.service = &Service{ ctx: s.ctx, manager: manager, } - s.grpcServer = grpc.NewServer() - s.service.RegisterGRPCService(s.grpcServer) - - handler, _ := SetUpRestHandler(s.service) - s.httpServer = &http.Server{ - Handler: handler, - ReadTimeout: 5 * time.Minute, - ReadHeaderTimeout: 5 * time.Second, + tlsConfig, err := s.cfg.Security.ToTLSConfig() + if err != nil { + return err + } + if tlsConfig != nil { + s.muxListener, err = tls.Listen(tcp, s.cfg.ListenAddr, tlsConfig) + } else { + s.muxListener, err = net.Listen(tcp, s.cfg.ListenAddr) + } + if err != nil { + return err } - s.serverLoopWg.Add(2) - go func() { - defer s.serverLoopWg.Done() - s.grpcServer.Serve(grpcL) - }() - go func() { - defer s.serverLoopWg.Done() - s.httpServer.Serve(httpL) - }() + s.serverLoopWg.Add(1) + go s.startGRPCAndHTTPServers(s.muxListener) // Run callbacks log.Info("triggering the start callback functions") for _, cb := range s.startCallbacks { cb() } + // Server has started. atomic.StoreInt64(&s.isServing, 1) - return mux.Serve() + return nil } // NewServer creates a new resource manager server. diff --git a/pkg/mcs/tso/server/server.go b/pkg/mcs/tso/server/server.go index 2c3ca81ee45..bd811f93726 100644 --- a/pkg/mcs/tso/server/server.go +++ b/pkg/mcs/tso/server/server.go @@ -296,7 +296,6 @@ func CreateServerWrapper(cmd *cobra.Command, args []string) { // TODO: Create the server ctx, cancel := context.WithCancel(context.Background()) svr := &Server{} - // TODO: tso server needs timeJumpBackCounter. sc := make(chan os.Signal, 1) signal.Notify(sc, syscall.SIGHUP, From 5ead2d32b50762752e5dfb52b98eb323871b783e Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Mon, 20 Feb 2023 16:13:19 +0800 Subject: [PATCH 19/24] add cmd Signed-off-by: lhy1024 --- cmd/pd-server/main.go | 25 ++++++++++++++++++++--- pkg/mcs/resource_manager/server/config.go | 4 ++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/cmd/pd-server/main.go b/cmd/pd-server/main.go index c3603d4308a..a00d2028bb2 100644 --- a/cmd/pd-server/main.go +++ b/cmd/pd-server/main.go @@ -26,6 +26,7 @@ import ( "github.com/tikv/pd/pkg/autoscaling" "github.com/tikv/pd/pkg/dashboard" "github.com/tikv/pd/pkg/errs" + resource_manager "github.com/tikv/pd/pkg/mcs/resource_manager/server" tso "github.com/tikv/pd/pkg/mcs/tso/server" "github.com/tikv/pd/pkg/swaggerserver" "github.com/tikv/pd/pkg/utils/configutil" @@ -78,14 +79,15 @@ func main() { // NewServiceCommand returns the service command. func NewServiceCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "service ", - Short: "Run a service", + Use: "service ", + Short: "Run a service, for example, tso, resource_manager", } cmd.AddCommand(NewTSOServiceCommand()) + cmd.AddCommand(NewResourceManagerServiceCommand()) return cmd } -// NewTSOServiceCommand returns the unsafe remove failed stores command. +// NewTSOServiceCommand returns the tso service command. func NewTSOServiceCommand() *cobra.Command { cmd := &cobra.Command{ Use: "tso", @@ -102,6 +104,23 @@ func NewTSOServiceCommand() *cobra.Command { return cmd } +// NewResourceManagerServiceCommand returns the resource manager service command. +func NewResourceManagerServiceCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "resource_manager", + Short: "Run the resource manager service", + Run: resource_manager.CreateServerWrapper, + } + cmd.Flags().BoolP("version", "V", false, "print version information and exit") + cmd.Flags().StringP("config", "", "", "config file") + cmd.Flags().StringP("backend-endpoints", "", "", "url for etcd client") + cmd.Flags().StringP("listen-addr", "", "", "listen address for tso service") + cmd.Flags().StringP("cacert", "", "", "path of file that contains list of trusted TLS CAs") + cmd.Flags().StringP("cert", "", "", "path of file that contains X509 certificate in PEM format") + cmd.Flags().StringP("key", "", "", "path of file that contains X509 key in PEM format") + return cmd +} + func createServerWrapper(cmd *cobra.Command, args []string) { schedulers.Register() cfg := config.NewConfig() diff --git a/pkg/mcs/resource_manager/server/config.go b/pkg/mcs/resource_manager/server/config.go index 8f7b58b2cdb..012fa5feacb 100644 --- a/pkg/mcs/resource_manager/server/config.go +++ b/pkg/mcs/resource_manager/server/config.go @@ -32,8 +32,8 @@ import ( const ( defaultName = "Resource Manager" - defaultBackendEndpoints = "http://127.0.0.1:2379" - defaultListenAddr = "http://127.0.0.1:2382" + defaultBackendEndpoints = "127.0.0.1:2379" + defaultListenAddr = "127.0.0.1:2382" defaultEnableGRPCGateway = true defaultLogFormat = "text" From 4c1911c0428aa613c850aefeadfc63995c8f327d Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Mon, 20 Feb 2023 17:44:13 +0800 Subject: [PATCH 20/24] address comments Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/config.go | 7 +++---- pkg/mcs/resource_manager/server/server.go | 14 +++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pkg/mcs/resource_manager/server/config.go b/pkg/mcs/resource_manager/server/config.go index 012fa5feacb..6050682099e 100644 --- a/pkg/mcs/resource_manager/server/config.go +++ b/pkg/mcs/resource_manager/server/config.go @@ -50,9 +50,6 @@ type Config struct { Metric metricutil.MetricConfig `toml:"metric" json:"metric"` - // WarningMsgs contains all warnings during parsing. - WarningMsgs []string - // Log related config. Log log.Config `toml:"log" json:"log"` @@ -96,9 +93,11 @@ func (c *Config) Parse(flagSet *pflag.FlagSet) error { // Adjust is used to adjust the PD configurations. func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { configMetaData := configutil.NewConfigMetadata(meta) + warningMsgs := make([]string, 0) if err := configMetaData.CheckUndecoded(); err != nil { - c.WarningMsgs = append(c.WarningMsgs, err.Error()) + warningMsgs = append(warningMsgs, err.Error()) } + configutil.PrintConfigCheckMsg(os.Stdout, warningMsgs) if c.Name == "" { hostname, err := os.Hostname() diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index 1dd8c2f1a1e..44941625c84 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -177,9 +177,9 @@ func (s *Server) startGRPCServer(l net.Listener) { s.service.RegisterGRPCService(gs) err := gs.Serve(l) if err != cmux.ErrListenerClosed { - log.Fatal("grpc server stops serving unexpectedly.", errs.ZapError(err)) + log.Fatal("grpc server stops serving unexpectedly", errs.ZapError(err)) } - log.Info("gRPC server stop serving.", errs.ZapError(err)) + log.Info("gRPC server stop serving", errs.ZapError(err)) // Attempt graceful stop (waits for pending RPCs), but force a stop if // it doesn't happen in a reasonable amount of time. @@ -191,7 +191,7 @@ func (s *Server) startGRPCServer(l net.Listener) { select { case <-done: case <-time.After(defaultGRPCGracefulStopTimeout): - log.Info("stopping grpc gracefully is taking longer than expected and force stopping now.", zap.Duration("default", defaultGRPCGracefulStopTimeout)) + log.Info("stopping grpc gracefully is taking longer than expected and force stopping now", zap.Duration("default", defaultGRPCGracefulStopTimeout)) gs.Stop() } } @@ -207,16 +207,16 @@ func (s *Server) startHTTPServer(l net.Listener) { } err := hs.Serve(l) if err != cmux.ErrListenerClosed { - log.Fatal("http server stops serving unexpectedly.", errs.ZapError(err)) + log.Fatal("http server stops serving unexpectedly", errs.ZapError(err)) } log.Info("http server stop serving", errs.ZapError(err)) ctx, cancel := context.WithTimeout(context.Background(), defaultHTTPGracefulShutdownTimeout) defer cancel() err = hs.Shutdown(ctx) - log.Info("all http(s) requests finished.") + log.Info("all http(s) requests finished") if err != nil { - log.Error("http server shutdown encountered problem.", errs.ZapError(err)) + log.Error("http server shutdown encountered problem", errs.ZapError(err)) } } @@ -232,7 +232,7 @@ func (s *Server) startGRPCAndHTTPServers(l net.Listener) { go s.startHTTPServer(httpL) if err := mux.Serve(); !strings.Contains(err.Error(), "use of closed network connection") { - log.Fatal("mux stop serving unexpectedly.", errs.ZapError(err)) + log.Fatal("mux stop serving unexpectedly", errs.ZapError(err)) } } From b0d7c5f51c9252a4400ba4517c95564b0d4f95fc Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Mon, 20 Feb 2023 20:43:28 +0800 Subject: [PATCH 21/24] address comments Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/config.go | 2 +- pkg/mcs/resource_manager/server/server.go | 6 +++--- tests/mcs/resource_manager/server_test.go | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/mcs/resource_manager/server/config.go b/pkg/mcs/resource_manager/server/config.go index 6050682099e..ef80680fe4a 100644 --- a/pkg/mcs/resource_manager/server/config.go +++ b/pkg/mcs/resource_manager/server/config.go @@ -33,7 +33,7 @@ import ( const ( defaultName = "Resource Manager" defaultBackendEndpoints = "127.0.0.1:2379" - defaultListenAddr = "127.0.0.1:2382" + defaultListenAddr = "127.0.0.1:3380" defaultEnableGRPCGateway = true defaultLogFormat = "text" diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index 44941625c84..41814de3cbc 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -271,9 +271,9 @@ func (s *Server) startServer() error { } // NewServer creates a new resource manager server. -func NewServer(ctx context.Context, cfg *Config, name string) *Server { +func NewServer(ctx context.Context, cfg *Config) *Server { return &Server{ - name: name, + name: cfg.Name, ctx: ctx, cfg: cfg, } @@ -318,7 +318,7 @@ func CreateServerWrapper(cmd *cobra.Command, args []string) { metricutil.Push(&cfg.Metric) ctx, cancel := context.WithCancel(context.Background()) - svr := NewServer(ctx, cfg, "ResourceManager") + svr := NewServer(ctx, cfg) sc := make(chan os.Signal, 1) signal.Notify(sc, diff --git a/tests/mcs/resource_manager/server_test.go b/tests/mcs/resource_manager/server_test.go index 53f83693e46..d212028f5f2 100644 --- a/tests/mcs/resource_manager/server_test.go +++ b/tests/mcs/resource_manager/server_test.go @@ -29,7 +29,6 @@ import ( "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/tests" "google.golang.org/grpc" - "google.golang.org/grpc/test/grpc_testing" ) func TestResourceManagerServer(t *testing.T) { @@ -51,7 +50,7 @@ func TestResourceManagerServer(t *testing.T) { cfg.BackendEndpoints = leader.GetAddr() cfg.ListenAddr = "127.0.0.1:8086" - svr := rm.NewServer(ctx, cfg, "ResourceManager") + svr := rm.NewServer(ctx, cfg) go svr.Run() testutil.Eventually(re, func() bool { return svr.IsServing() @@ -62,10 +61,11 @@ func TestResourceManagerServer(t *testing.T) { cc, err := grpc.DialContext(ctx, cfg.ListenAddr, grpc.WithInsecure()) re.NoError(err) defer cc.Close() - grpcClient := grpc_testing.NewTestServiceClient(cc) - resp, err := grpcClient.EmptyCall(context.Background(), &grpc_testing.Empty{}) - re.ErrorContains(err, "Unimplemented") - re.Nil(resp) + c := rmpb.NewResourceManagerClient(cc) + _, err = c.GetResourceGroup(context.Background(), &rmpb.GetResourceGroupRequest{ + ResourceGroupName: "pingcap", + }) + re.ErrorContains(err, "resource group not found") // Test registered REST HTTP Handler url := "http://" + cfg.ListenAddr + "/resource-manager/api/v1/config" From 27adb0d398c940d074de6543f4400b38c47d6d1b Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Tue, 21 Feb 2023 15:03:29 +0800 Subject: [PATCH 22/24] address comments Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/config.go | 4 +- pkg/mcs/resource_manager/server/server.go | 48 ++++++++++++++--------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/pkg/mcs/resource_manager/server/config.go b/pkg/mcs/resource_manager/server/config.go index ef80680fe4a..04cf89481d4 100644 --- a/pkg/mcs/resource_manager/server/config.go +++ b/pkg/mcs/resource_manager/server/config.go @@ -45,8 +45,8 @@ type Config struct { BackendEndpoints string `toml:"backend-endpoints" json:"backend-endpoints"` ListenAddr string `toml:"listen-addr" json:"listen-addr"` Name string `toml:"name" json:"name"` - DataDir string `toml:"data-dir" json:"data-dir"` - EnableGRPCGateway bool `json:"enable-grpc-gateway"` + DataDir string `toml:"data-dir" json:"data-dir"` // TODO: remove this after refactoring + EnableGRPCGateway bool `json:"enable-grpc-gateway"` // TODO: use it Metric metricutil.MetricConfig `toml:"metric" json:"metric"` diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index 41814de3cbc..ae768983709 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -48,7 +48,7 @@ const ( // defaultGRPCGracefulStopTimeout is the default timeout to wait for grpc server to gracefully stop defaultGRPCGracefulStopTimeout = 5 * time.Second // defaultHTTPGracefulShutdownTimeout is the default timeout to wait for http server to gracefully shutdown - defaultHTTPGracefulShutdownTimeout = 300 * time.Second + defaultHTTPGracefulShutdownTimeout = 5 * time.Second ) // Server is the resource manager server, and it implements bs.Server. @@ -57,10 +57,8 @@ type Server struct { // Server state. 0 is not serving, 1 is serving. isServing int64 - ctx context.Context - serverLoopCtx context.Context - serverLoopCancel func() - serverLoopWg sync.WaitGroup + ctx context.Context + serverLoopWg sync.WaitGroup cfg *Config name string @@ -94,7 +92,6 @@ func (s *Server) Run() error { if err := s.initClient(); err != nil { return err } - s.serverLoopCtx, s.serverLoopCancel = context.WithCancel(s.ctx) return s.startServer() } @@ -176,15 +173,13 @@ func (s *Server) startGRPCServer(l net.Listener) { gs := grpc.NewServer() s.service.RegisterGRPCService(gs) err := gs.Serve(l) - if err != cmux.ErrListenerClosed { - log.Fatal("grpc server stops serving unexpectedly", errs.ZapError(err)) - } - log.Info("gRPC server stop serving", errs.ZapError(err)) + log.Info("gRPC server stop serving") // Attempt graceful stop (waits for pending RPCs), but force a stop if // it doesn't happen in a reasonable amount of time. done := make(chan struct{}) go func() { + log.Info("try to gracefully stop the server now") gs.GracefulStop() close(done) }() @@ -194,6 +189,11 @@ func (s *Server) startGRPCServer(l net.Listener) { log.Info("stopping grpc gracefully is taking longer than expected and force stopping now", zap.Duration("default", defaultGRPCGracefulStopTimeout)) gs.Stop() } + if s.IsClosed() { + log.Info("grpc server stopped.") + } else { + log.Fatal("grpc server stopped unexpectedly", errs.ZapError(err)) + } } func (s *Server) startHTTPServer(l net.Listener) { @@ -206,17 +206,19 @@ func (s *Server) startHTTPServer(l net.Listener) { ReadHeaderTimeout: 5 * time.Second, } err := hs.Serve(l) - if err != cmux.ErrListenerClosed { - log.Fatal("http server stops serving unexpectedly", errs.ZapError(err)) - } - log.Info("http server stop serving", errs.ZapError(err)) + log.Info("http server stop serving") ctx, cancel := context.WithTimeout(context.Background(), defaultHTTPGracefulShutdownTimeout) defer cancel() - err = hs.Shutdown(ctx) - log.Info("all http(s) requests finished") - if err != nil { + if err := hs.Shutdown(ctx); err != nil { log.Error("http server shutdown encountered problem", errs.ZapError(err)) + } else { + log.Info("all http(s) requests finished") + } + if s.IsClosed() { + log.Info("http server stopped.") + } else { + log.Fatal("http server stopped unexpectedly", errs.ZapError(err)) } } @@ -231,8 +233,12 @@ func (s *Server) startGRPCAndHTTPServers(l net.Listener) { go s.startGRPCServer(grpcL) go s.startHTTPServer(httpL) - if err := mux.Serve(); !strings.Contains(err.Error(), "use of closed network connection") { - log.Fatal("mux stop serving unexpectedly", errs.ZapError(err)) + if err := mux.Serve(); err != nil { + if s.IsClosed() { + log.Info("mux stop serving", errs.ZapError(err)) + } else { + log.Fatal("mux stop serving unexpectedly", errs.ZapError(err)) + } } } @@ -264,6 +270,10 @@ func (s *Server) startServer() error { for _, cb := range s.startCallbacks { cb() } + // TODO: resolve callback for the primary + for _, cb := range s.primaryCallbacks { + cb(s.ctx) + } // Server has started. atomic.StoreInt64(&s.isServing, 1) From a9a016ae66749bfaa2fe323997b240942b32b03b Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Tue, 21 Feb 2023 17:12:59 +0800 Subject: [PATCH 23/24] address comments Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index ae768983709..8da95c20593 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -190,7 +190,7 @@ func (s *Server) startGRPCServer(l net.Listener) { gs.Stop() } if s.IsClosed() { - log.Info("grpc server stopped.") + log.Info("grpc server stopped") } else { log.Fatal("grpc server stopped unexpectedly", errs.ZapError(err)) } @@ -216,7 +216,7 @@ func (s *Server) startHTTPServer(l net.Listener) { log.Info("all http(s) requests finished") } if s.IsClosed() { - log.Info("http server stopped.") + log.Info("http server stopped") } else { log.Fatal("http server stopped unexpectedly", errs.ZapError(err)) } From a94eaabd1f2e6d5147283cbd3ac190209db158ee Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Tue, 21 Feb 2023 17:16:57 +0800 Subject: [PATCH 24/24] remove todo Signed-off-by: lhy1024 --- pkg/mcs/resource_manager/server/server.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resource_manager/server/server.go index 8da95c20593..c3cd6c2965c 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resource_manager/server/server.go @@ -103,8 +103,6 @@ func (s *Server) Close() { } log.Info("closing resource manager server ...") - // TODO: double check when muxListener is closed, grpc.Server.serve() and http.Server.serve() - // will also close with error cmux.ErrListenerClosed. s.muxListener.Close() s.serverLoopWg.Wait()