From ca83b1dc15983b3907445a5f2d50ac351aa48414 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Thu, 29 Dec 2022 20:12:54 +0800 Subject: [PATCH 01/15] impl token request Signed-off-by: Cabinfever_B --- .../resource_manager/server/grpc_service.go | 51 ++++++++++- .../resource_manager/server/token_bukets.go | 85 +++++++++++++++++-- pkg/mcs/resource_manager/server/types.go | 61 +++++++++---- 3 files changed, 174 insertions(+), 23 deletions(-) diff --git a/pkg/mcs/resource_manager/server/grpc_service.go b/pkg/mcs/resource_manager/server/grpc_service.go index a0f9c1c5c38..51a9ff405db 100644 --- a/pkg/mcs/resource_manager/server/grpc_service.go +++ b/pkg/mcs/resource_manager/server/grpc_service.go @@ -16,12 +16,16 @@ package server import ( "context" + "io" "net/http" + "time" "github.com/pingcap/errors" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/pingcap/log" "github.com/tikv/pd/pkg/mcs/registry" "github.com/tikv/pd/server" + "go.uber.org/zap" "google.golang.org/grpc" ) @@ -125,5 +129,50 @@ func (s *Service) ModifyResourceGroup(ctx context.Context, req *rmpb.PutResource // AcquireTokenBuckets implements ResourceManagerServer.AcquireTokenBuckets. func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBucketsServer) error { - return errors.New("Not implemented") + for { + select { + case <-s.ctx.Done(): + return errors.New("server closed") + default: + } + request, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return errors.WithStack(err) + } + targetPeriodMs := request.GetTargetRequestPeriodMs() + resps := &rmpb.TokenBucketsResponse{} + for _, req := range request.Requests { + rg := s.manager.GetResourceGroup(req.ResourceGroupName) + if rg == nil { + return errors.New("resource group not found") + } + now := time.Now() + resp := &rmpb.TokenBucketResponse{ + ResourceGroupName: rg.Name, + } + switch rg.Mode { + case rmpb.GroupMode_RUMode: + for _, re := range req.GetRuItems().GetRequestRU() { + switch re.Type { + case rmpb.RequestUnitType_RRU: + rg.UpdateRRU(now) + tokens := rg.RequestRRU(float64(re.Value), targetPeriodMs) + resp.GrantedRUTokens = append(resp.GrantedRUTokens, tokens) + case rmpb.RequestUnitType_WRU: + rg.UpdateWRU(now) + tokens := rg.RequestWRU(float64(re.Value), targetPeriodMs) + resp.GrantedRUTokens = append(resp.GrantedRUTokens, tokens) + } + } + case rmpb.GroupMode_NativeMode: + return errors.New("not supports the resource type") + } + log.Debug("finish token request from", zap.String("resource group", req.ResourceGroupName)) + resps.Responses = append(resps.Responses, resp) + } + stream.Send(resps) + } } diff --git a/pkg/mcs/resource_manager/server/token_bukets.go b/pkg/mcs/resource_manager/server/token_bukets.go index 00d4a3fe04f..d8b85c52dce 100644 --- a/pkg/mcs/resource_manager/server/token_bukets.go +++ b/pkg/mcs/resource_manager/server/token_bukets.go @@ -25,12 +25,30 @@ const defaultRefillRate = 10000 const defaultInitialTokens = 10 * 10000 +const defaultMaxTokens = 1e7 + +const defaultLoanMaxPeriod = 10 * time.Second + +var loanReserveRatio float64 = 0.05 + // GroupTokenBucket is a token bucket for a resource group. +// TODO: statistics Consumption type GroupTokenBucket struct { *rmpb.TokenBucket `json:"token_bucket,omitempty"` Consumption *rmpb.TokenBucketsRequest `json:"consumption,omitempty"` LastUpdate *time.Time `json:"last_update,omitempty"` Initialized bool `json:"initialized"` + LoanExpireTime *time.Time `json:"loan_time,omitempty"` + LoanMaxPeriod time.Duration `json:"loan_max_perio,omitempty"` + MaxTokens float64 `json:"max_tokens,omitempty"` +} + +func NewGroupTokenBucket(tokenBucket *rmpb.TokenBucket) GroupTokenBucket { + return GroupTokenBucket{ + TokenBucket: tokenBucket, + MaxTokens: defaultMaxTokens, + LoanMaxPeriod: defaultLoanMaxPeriod, + } } // patch patches the token bucket settings. @@ -51,8 +69,8 @@ func (t *GroupTokenBucket) patch(settings *rmpb.TokenBucket) { t.TokenBucket = tb } -// Update updates the token bucket. -func (t *GroupTokenBucket) Update(now time.Time) { +// update updates the token bucket. +func (t *GroupTokenBucket) update(now time.Time) { if !t.Initialized { t.Settings.Fillrate = defaultRefillRate t.Tokens = defaultInitialTokens @@ -66,12 +84,65 @@ func (t *GroupTokenBucket) Update(now time.Time) { t.Tokens += float64(t.Settings.Fillrate) * delta.Seconds() t.LastUpdate = &now } + if t.Tokens >= 0 { + t.LoanExpireTime = nil + } + if t.Tokens > defaultMaxTokens { + t.Tokens = defaultMaxTokens + } + } -// Request requests tokens from the token bucket. -func (t *GroupTokenBucket) Request( +// request requests tokens from the token bucket. +func (t *GroupTokenBucket) request( neededTokens float64, targetPeriodMs uint64, -) *rmpb.TokenBucket { - // TODO: Implement the token bucket algorithm. - return nil +) (*rmpb.TokenBucket, int64) { + var res rmpb.TokenBucket + res.Settings = &rmpb.TokenLimitSettings{} + // TODO: consider the shares for dispatch the fill rate + res.Settings.Fillrate = 0 + + if neededTokens <= 0 { + return &res, 0 + } + + if t.Tokens >= neededTokens { + t.Tokens -= neededTokens + // granted the total request tokens + res.Tokens = neededTokens + return &res, 0 + } + + var grantedTokens float64 + if t.Tokens > 0 { + grantedTokens = t.Tokens + t.Tokens = 0 + neededTokens -= grantedTokens + } + + now := time.Now() + var periodFilled float64 + var trickleTime int64 = int64(targetPeriodMs) + if t.LoanExpireTime != nil && t.LoanExpireTime.After(now) { + duration := t.LoanExpireTime.Sub(now) + periodFilled = float64(t.Settings.Fillrate) * (1 - loanReserveRatio) * duration.Seconds() + trickleTime = duration.Milliseconds() + } else { + now.Add(t.LoanMaxPeriod) + t.LoanExpireTime = &now + periodFilled = float64(t.Settings.Fillrate) * (1 - loanReserveRatio) * t.LoanMaxPeriod.Seconds() + } + periodFilled += t.Tokens + if periodFilled <= float64(t.Settings.Fillrate)*loanReserveRatio { + periodFilled = float64(t.Settings.Fillrate) * loanReserveRatio + } + if periodFilled >= neededTokens { + grantedTokens += neededTokens + t.Tokens -= neededTokens + } else { + grantedTokens += periodFilled + t.Tokens -= periodFilled + } + res.Tokens = grantedTokens + return &res, trickleTime } diff --git a/pkg/mcs/resource_manager/server/types.go b/pkg/mcs/resource_manager/server/types.go index 72686ad4c94..2f6968fe196 100644 --- a/pkg/mcs/resource_manager/server/types.go +++ b/pkg/mcs/resource_manager/server/types.go @@ -18,6 +18,7 @@ package server import ( "encoding/json" "sync" + "time" "github.com/pingcap/errors" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" @@ -146,27 +147,17 @@ func FromProtoResourceGroup(group *rmpb.ResourceGroup) *ResourceGroup { case rmpb.GroupMode_RUMode: if settings := group.GetSettings().GetRUSettings(); settings != nil { ruSettings = &RequestUnitSettings{ - RRU: GroupTokenBucket{ - TokenBucket: settings.GetRRU(), - }, - WRU: GroupTokenBucket{ - TokenBucket: settings.GetWRU(), - }, + RRU: NewGroupTokenBucket(settings.GetRRU()), + WRU: NewGroupTokenBucket(settings.GetWRU()), } rg.RUSettings = ruSettings } case rmpb.GroupMode_NativeMode: if settings := group.GetSettings().GetResourceSettings(); settings != nil { resourceSettings = &NativeResourceSettings{ - CPU: GroupTokenBucket{ - TokenBucket: settings.GetCpu(), - }, - IOReadBandwidth: GroupTokenBucket{ - TokenBucket: settings.GetIoRead(), - }, - IOWriteBandwidth: GroupTokenBucket{ - TokenBucket: settings.GetIoWrite(), - }, + CPU: NewGroupTokenBucket(settings.GetCpu()), + IOReadBandwidth: NewGroupTokenBucket(settings.GetIoRead()), + IOWriteBandwidth: NewGroupTokenBucket(settings.GetIoWrite()), } rg.ResourceSettings = resourceSettings } @@ -174,6 +165,46 @@ func FromProtoResourceGroup(group *rmpb.ResourceGroup) *ResourceGroup { return rg } +// UpdateRRU updates the RRU of the resource group. +func (rg *ResourceGroup) UpdateRRU(now time.Time) { + rg.Lock() + defer rg.Unlock() + if rg.RUSettings != nil { + rg.RUSettings.RRU.update(now) + } +} + +// UpdateWRU updates the WRU of the resource group. +func (rg *ResourceGroup) UpdateWRU(now time.Time) { + rg.Lock() + defer rg.Unlock() + if rg.RUSettings != nil { + rg.RUSettings.WRU.update(now) + } +} + +// RequestRRU requests the RRU of the resource group. +func (rg *ResourceGroup) RequestRRU(neededTokens float64, targetPeriodMs uint64) *rmpb.GrantedRUTokenBucket { + rg.Lock() + defer rg.Unlock() + if rg.RUSettings == nil { + return nil + } + tb, trickleTimeMs := rg.RUSettings.RRU.request(neededTokens, targetPeriodMs) + return &rmpb.GrantedRUTokenBucket{Type: rmpb.RequestUnitType_RRU, GrantedTokens: tb, TrickleTimeMs: trickleTimeMs} +} + +// RequestWRU requests the WRU of the resource group. +func (rg *ResourceGroup) RequestWRU(neededTokens float64, targetPeriodMs uint64) *rmpb.GrantedRUTokenBucket { + rg.Lock() + defer rg.Unlock() + if rg.RUSettings == nil { + return nil + } + tb, trickleTimeMs := rg.RUSettings.WRU.request(neededTokens, targetPeriodMs) + return &rmpb.GrantedRUTokenBucket{Type: rmpb.RequestUnitType_WRU, GrantedTokens: tb, TrickleTimeMs: trickleTimeMs} +} + // IntoProtoResourceGroup converts a ResourceGroup to a rmpb.ResourceGroup. func (rg *ResourceGroup) IntoProtoResourceGroup() *rmpb.ResourceGroup { rg.RLock() From 497c67d0a6e6a21844292e04978ec7f0bc4c085b Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Thu, 29 Dec 2022 20:24:02 +0800 Subject: [PATCH 02/15] impl resouce manager client Signed-off-by: Cabinfever_B --- client/client.go | 5 + client/go.mod | 2 +- client/go.sum | 4 +- client/resoucemanager_client.go | 265 ++++++++++++++++++++++++++++++++ 4 files changed, 273 insertions(+), 3 deletions(-) create mode 100644 client/resoucemanager_client.go diff --git a/client/client.go b/client/client.go index b7e15fe6eb2..8bd242a4b7a 100644 --- a/client/client.go +++ b/client/client.go @@ -137,6 +137,8 @@ type Client interface { // KeyspaceClient manages keyspace metadata. KeyspaceClient + // ResourceManagerClient + ResourceManagerClient // Close closes the client. Close() } @@ -390,6 +392,8 @@ type client struct { // dc-location -> *lastTSO lastTSMap sync.Map // Same as map[string]*lastTSO + tokenDispatcher *tokenDispatcher + // For internal usage. checkTSDeadlineCh chan struct{} leaderNetworkFailure int32 @@ -417,6 +421,7 @@ func NewClientWithContext(ctx context.Context, pdAddrs []string, security Securi } // Start the daemons. c.updateTSODispatcher() + c.createTokenispatcher() c.wg.Add(3) go c.tsLoop() go c.tsCancelLoop() diff --git a/client/go.mod b/client/go.mod index 09277d77a4d..df2e176f79e 100644 --- a/client/go.mod +++ b/client/go.mod @@ -6,7 +6,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 + github.com/pingcap/kvproto v0.0.0-20221221093947-0a9b14f1fc26 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.11.0 github.com/stretchr/testify v1.7.0 diff --git a/client/go.sum b/client/go.sum index 32758cfe55e..09ac827c37b 100644 --- a/client/go.sum +++ b/client/go.sum @@ -104,8 +104,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 h1:FYgKV9znRQmzVrrJDZ0gUfMIvKLAMU1tu1UKJib8bEQ= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20221221093947-0a9b14f1fc26 h1:Tw4afZ2Tyr8iT8Oln6/szMjh5IDs+GtlnLsDo/Y2HEE= +github.com/pingcap/kvproto v0.0.0-20221221093947-0a9b14f1fc26/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/client/resoucemanager_client.go b/client/resoucemanager_client.go new file mode 100644 index 00000000000..738e75c789d --- /dev/null +++ b/client/resoucemanager_client.go @@ -0,0 +1,265 @@ +package pd + +import ( + "context" + "time" + + "github.com/pingcap/errors" + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/pingcap/log" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +// KeyspaceClient manages keyspace metadata. +type ResourceManagerClient interface { + ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) + GetResourceGroup(ctx context.Context, resourceGroupName string) (*rmpb.ResourceGroup, error) + AddResourceGroup(ctx context.Context, resourceGroupName string, settings *rmpb.GroupSettings) (string, error) + ModifyResourceGroup(ctx context.Context, resourceGroupName string, settings *rmpb.GroupSettings) (string, error) + DeleteResourceGroup(ctx context.Context, resourceGroupName string) (string, error) + AcquireTokenBuckets(ctx context.Context, request *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) +} + +// leaderClient gets the client of current PD leader. +func (c *client) resouceManagerClient() rmpb.ResourceManagerClient { + if cc, ok := c.clientConns.Load(c.GetLeaderAddr()); ok { + return rmpb.NewResourceManagerClient(cc.(*grpc.ClientConn)) + } + return nil +} + +// ListResourceGroups loads and returns target keyspace's metadata. +func (c *client) ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) { + req := &rmpb.ListResourceGroupsRequest{} + resp, err := c.resouceManagerClient().ListResourceGroups(ctx, req) + if err != nil { + return nil, err + } + resErr := resp.GetError() + if resErr != nil { + return nil, errors.Errorf("[pd]" + resErr.Message) + } + return resp.GetGroups(), nil +} + +func (c *client) GetResourceGroup(ctx context.Context, resourceGroupName string) (*rmpb.ResourceGroup, error) { + req := &rmpb.GetResourceGroupRequest{ + ResourceGroupName: resourceGroupName, + } + resp, err := c.resouceManagerClient().GetResourceGroup(ctx, req) + if err != nil { + return nil, err + } + resErr := resp.GetError() + if resErr != nil { + return nil, errors.Errorf("[pd]" + resErr.Message) + } + return resp.GetGroup(), nil +} + +func (c *client) AddResourceGroup(ctx context.Context, resourceGroupName string, settings *rmpb.GroupSettings) (string, error) { + return c.putResourceGroup(ctx, resourceGroupName, settings, 0 /* type add resource group */) +} + +func (c *client) ModifyResourceGroup(ctx context.Context, resourceGroupName string, settings *rmpb.GroupSettings) (string, error) { + return c.putResourceGroup(ctx, resourceGroupName, settings, 1 /* type modify resource group */) +} + +func (c *client) putResourceGroup(ctx context.Context, resourceGroupName string, settings *rmpb.GroupSettings, typ int) (str string, err error) { + group := &rmpb.ResourceGroup{ + Name: resourceGroupName, + Settings: settings, + } + req := &rmpb.PutResourceGroupRequest{ + Group: group, + } + var resp *rmpb.PutResourceGroupResponse + if typ == 0 { + resp, err = c.resouceManagerClient().AddResourceGroup(ctx, req) + } else { + resp, err = c.resouceManagerClient().ModifyResourceGroup(ctx, req) + } + if err != nil { + return str, err + } + resErr := resp.GetError() + if resErr != nil { + return str, errors.Errorf("[pd]" + resErr.Message) + } + str = resp.GetBody() + return +} + +func (c *client) DeleteResourceGroup(ctx context.Context, resourceGroupName string) (string, error) { + req := &rmpb.DeleteResourceGroupRequest{ + ResourceGroupName: resourceGroupName, + } + resp, err := c.resouceManagerClient().DeleteResourceGroup(ctx, req) + if err != nil { + return "", err + } + resErr := resp.GetError() + if resErr != nil { + return "", errors.Errorf("[pd]" + resErr.Message) + } + return resp.GetBody(), nil +} + +func (c *client) AcquireTokenBuckets(ctx context.Context, request *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) { + req := &tokenRequest{ + done: make(chan error, 1), + requestCtx: ctx, + clientCtx: c.ctx, + } + req.Requeset = request + c.tokenDispatcher.tokenBatchController.tokenRequestCh <- req + grantedTokens, err := req.Wait() + if err != nil { + return nil, err + } + return grantedTokens, err +} + +type tokenRequest struct { + clientCtx context.Context + requestCtx context.Context + done chan error + Requeset *rmpb.TokenBucketsRequest + TokenBuckets []*rmpb.TokenBucketResponse +} + +func (req *tokenRequest) Wait() (tokenBuckets []*rmpb.TokenBucketResponse, err error) { + select { + case err = <-req.done: + err = errors.WithStack(err) + if err != nil { + return nil, err + } + tokenBuckets = req.TokenBuckets + return + case <-req.requestCtx.Done(): + return nil, errors.WithStack(req.requestCtx.Err()) + case <-req.clientCtx.Done(): + return nil, errors.WithStack(req.clientCtx.Err()) + } +} + +type tokenBatchController struct { + tokenRequestCh chan *tokenRequest +} + +func newTokenBatchController(tokenRequestCh chan *tokenRequest) *tokenBatchController { + return &tokenBatchController{ + tokenRequestCh: tokenRequestCh, + } +} + +type tokenDispatcher struct { + dispatcherCancel context.CancelFunc + tokenBatchController *tokenBatchController +} + +type resourceManagerConnectionContext struct { + stream rmpb.ResourceManager_AcquireTokenBucketsClient + ctx context.Context + cancel context.CancelFunc +} + +func (c *client) createTokenispatcher() { + dispatcherCtx, dispatcherCancel := context.WithCancel(c.ctx) + dispatcher := &tokenDispatcher{ + dispatcherCancel: dispatcherCancel, + tokenBatchController: newTokenBatchController( + make(chan *tokenRequest, 1)), + } + go c.handleResouceTokenDispatcher(dispatcherCtx, dispatcher.tokenBatchController) + c.tokenDispatcher = dispatcher +} + +func (c *client) handleResouceTokenDispatcher(dispatcherCtx context.Context, tbc *tokenBatchController) { + var connection resourceManagerConnectionContext + if err := c.tryResourceManagerConnect(dispatcherCtx, &connection); err != nil { + log.Warn("get stream error", zap.Error(err)) + } + + for { + var firstTSORequest *tokenRequest + select { + case <-dispatcherCtx.Done(): + return + case firstTSORequest = <-tbc.tokenRequestCh: + } + stream, streamCtx, cancel := connection.stream, connection.ctx, connection.cancel + if stream == nil { + c.tryResourceManagerConnect(dispatcherCtx, &connection) + c.finishTokenRequest(firstTSORequest, nil, errors.Errorf("no stream")) + continue + } + select { + case <-streamCtx.Done(): + log.Info("[pd] resource manager stream is canceled") + cancel() + stream = nil + continue + default: + } + err := c.processTokenRequests(stream, firstTSORequest) + if err != nil { + log.Info("processTokenRequests error", zap.Error(err)) + cancel() + connection.stream = nil + } + } + +} + +func (c *client) processTokenRequests(stream rmpb.ResourceManager_AcquireTokenBucketsClient, t *tokenRequest) error { + req := t.Requeset + if err := stream.Send(req); err != nil { + err = errors.WithStack(err) + c.finishTokenRequest(t, nil, err) + return err + } + resp, err := stream.Recv() + if err != nil { + err = errors.WithStack(err) + c.finishTokenRequest(t, nil, err) + return err + } + if resp.GetError() != nil { + return errors.Errorf("[pd]" + resp.GetError().Message) + } + tokenBuckets := resp.GetResponses() + c.finishTokenRequest(t, tokenBuckets, nil) + return nil +} + +func (c *client) finishTokenRequest(t *tokenRequest, tokenBuckets []*rmpb.TokenBucketResponse, err error) { + t.TokenBuckets = tokenBuckets + t.done <- err +} + +func (c *client) tryResourceManagerConnect(ctx context.Context, connection *resourceManagerConnectionContext) error { + var ( + err error + stream rmpb.ResourceManager_AcquireTokenBucketsClient + ) + for i := 0; i < maxRetryTimes; i++ { + cctx, cancel := context.WithCancel(ctx) + stream, err = c.resouceManagerClient().AcquireTokenBuckets(cctx) + if err == nil && stream != nil { + connection.cancel = cancel + connection.ctx = cctx + connection.stream = stream + return nil + } + cancel() + select { + case <-ctx.Done(): + return err + case <-time.After(retryInterval): + } + } + return err +} From 321c2ad17abdce0fff9698bb8deeb67bfc6043c8 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 3 Jan 2023 17:29:35 +0800 Subject: [PATCH 03/15] impl token request and fix update bug Signed-off-by: Cabinfever_B --- .../resource_manager/server/grpc_service.go | 2 +- .../resource_manager/server/token_bukets.go | 20 ++++++++----------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/pkg/mcs/resource_manager/server/grpc_service.go b/pkg/mcs/resource_manager/server/grpc_service.go index 51a9ff405db..9b5d4e395d3 100644 --- a/pkg/mcs/resource_manager/server/grpc_service.go +++ b/pkg/mcs/resource_manager/server/grpc_service.go @@ -159,7 +159,7 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu switch re.Type { case rmpb.RequestUnitType_RRU: rg.UpdateRRU(now) - tokens := rg.RequestRRU(float64(re.Value), targetPeriodMs) + tokens := rg.RequestRRU(re.Value, targetPeriodMs) resp.GrantedRUTokens = append(resp.GrantedRUTokens, tokens) case rmpb.RequestUnitType_WRU: rg.UpdateWRU(now) diff --git a/pkg/mcs/resource_manager/server/token_bukets.go b/pkg/mcs/resource_manager/server/token_bukets.go index d8b85c52dce..1cf3a84781a 100644 --- a/pkg/mcs/resource_manager/server/token_bukets.go +++ b/pkg/mcs/resource_manager/server/token_bukets.go @@ -21,8 +21,6 @@ import ( rmpb "github.com/pingcap/kvproto/pkg/resource_manager" ) -const defaultRefillRate = 10000 - const defaultInitialTokens = 10 * 10000 const defaultMaxTokens = 1e7 @@ -72,8 +70,9 @@ func (t *GroupTokenBucket) patch(settings *rmpb.TokenBucket) { // update updates the token bucket. func (t *GroupTokenBucket) update(now time.Time) { if !t.Initialized { - t.Settings.Fillrate = defaultRefillRate - t.Tokens = defaultInitialTokens + if t.Tokens < defaultInitialTokens { + t.Tokens = defaultInitialTokens + } t.LastUpdate = &now t.Initialized = true return @@ -90,7 +89,6 @@ func (t *GroupTokenBucket) update(now time.Time) { if t.Tokens > defaultMaxTokens { t.Tokens = defaultMaxTokens } - } // request requests tokens from the token bucket. @@ -101,7 +99,6 @@ func (t *GroupTokenBucket) request( res.Settings = &rmpb.TokenLimitSettings{} // TODO: consider the shares for dispatch the fill rate res.Settings.Fillrate = 0 - if neededTokens <= 0 { return &res, 0 } @@ -120,16 +117,15 @@ func (t *GroupTokenBucket) request( neededTokens -= grantedTokens } - now := time.Now() var periodFilled float64 - var trickleTime int64 = int64(targetPeriodMs) - if t.LoanExpireTime != nil && t.LoanExpireTime.After(now) { - duration := t.LoanExpireTime.Sub(now) + var trickleTime = int64(targetPeriodMs) + if t.LoanExpireTime != nil && t.LoanExpireTime.After(*t.LastUpdate) { + duration := t.LoanExpireTime.Sub(*t.LastUpdate) periodFilled = float64(t.Settings.Fillrate) * (1 - loanReserveRatio) * duration.Seconds() trickleTime = duration.Milliseconds() } else { - now.Add(t.LoanMaxPeriod) - t.LoanExpireTime = &now + et := t.LastUpdate.Add(t.LoanMaxPeriod) + t.LoanExpireTime = &et periodFilled = float64(t.Settings.Fillrate) * (1 - loanReserveRatio) * t.LoanMaxPeriod.Seconds() } periodFilled += t.Tokens From 0fc2ca5ae7fc41841c3b63036b9d0a60b61b1c6c Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 3 Jan 2023 17:30:22 +0800 Subject: [PATCH 04/15] add test Signed-off-by: Cabinfever_B impl token request and fix update bug Signed-off-by: Cabinfever_B --- .../resource_manager/server/grpc_service.go | 2 +- .../server/token_buckets_test.go | 85 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 pkg/mcs/resource_manager/server/token_buckets_test.go diff --git a/pkg/mcs/resource_manager/server/grpc_service.go b/pkg/mcs/resource_manager/server/grpc_service.go index 9b5d4e395d3..4ad156ecb40 100644 --- a/pkg/mcs/resource_manager/server/grpc_service.go +++ b/pkg/mcs/resource_manager/server/grpc_service.go @@ -163,7 +163,7 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu resp.GrantedRUTokens = append(resp.GrantedRUTokens, tokens) case rmpb.RequestUnitType_WRU: rg.UpdateWRU(now) - tokens := rg.RequestWRU(float64(re.Value), targetPeriodMs) + tokens := rg.RequestWRU(re.Value, targetPeriodMs) resp.GrantedRUTokens = append(resp.GrantedRUTokens, tokens) } } diff --git a/pkg/mcs/resource_manager/server/token_buckets_test.go b/pkg/mcs/resource_manager/server/token_buckets_test.go new file mode 100644 index 00000000000..d4cb97561a0 --- /dev/null +++ b/pkg/mcs/resource_manager/server/token_buckets_test.go @@ -0,0 +1,85 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS,g +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "math" + "testing" + "time" + + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/stretchr/testify/require" +) + +func TestGroupTokenBucketUpdateAndPatch(t *testing.T) { + re := require.New(t) + tbSetting := &rmpb.TokenBucket{ + Tokens: 200000, + Settings: &rmpb.TokenLimitSettings{ + Fillrate: 2000, + BurstLimit: 20000000, + }, + } + + tb := NewGroupTokenBucket(tbSetting) + time1 := time.Now() + tb.update(time1) + re.LessOrEqual(math.Abs(tbSetting.Tokens-tb.Tokens), 1e-7) + re.Equal(tbSetting.Settings.Fillrate, tb.Settings.Fillrate) + + tbSetting = &rmpb.TokenBucket{ + Tokens: -100000, + Settings: &rmpb.TokenLimitSettings{ + Fillrate: 1000, + BurstLimit: 10000000, + }, + } + tb.patch(tbSetting) + + time2 := time.Now() + tb.update(time2) + re.LessOrEqual(math.Abs(100000-tb.Tokens), time2.Sub(time1).Seconds()*float64(tbSetting.Settings.Fillrate)+1e7) + re.Equal(tbSetting.Settings.Fillrate, tb.Settings.Fillrate) +} + +func TestGroupTokenBucketRequest(t *testing.T) { + re := require.New(t) + tbSetting := &rmpb.TokenBucket{ + Tokens: 200000, + Settings: &rmpb.TokenLimitSettings{ + Fillrate: 2000, + BurstLimit: 20000000, + }, + } + + gtb := NewGroupTokenBucket(tbSetting) + time1 := time.Now() + gtb.update(time1) + tb, trickle := gtb.request(100000, uint64(time.Second)*10/uint64(time.Millisecond)) + re.LessOrEqual(math.Abs(tb.Tokens-100000), 1e-7) + re.Equal(trickle, int64(0)) + tb, trickle = gtb.request(101000, uint64(time.Second)*10/uint64(time.Millisecond)) + re.LessOrEqual(math.Abs(tb.Tokens-101000), 1e-7) + re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) + re.Equal(*gtb.LoanExpireTime, time1.Add(gtb.LoanMaxPeriod)) + time2 := time.Now() + gtb.update(time2) + tb, trickle = gtb.request(100000, uint64(time.Second)*10/uint64(time.Millisecond)) + re.LessOrEqual(math.Abs(tb.Tokens-19000*(1-loanReserveRatio)), time1.Add(gtb.LoanMaxPeriod).Sub(time2).Seconds()*(1-loanReserveRatio)*float64(tb.Settings.Fillrate)+1e7) + re.Equal(trickle, time1.Add(gtb.LoanMaxPeriod).Sub(time2).Milliseconds()) + tb, trickle = gtb.request(2000, uint64(time.Second)*10/uint64(time.Millisecond)) + re.LessOrEqual(tb.Tokens, time1.Add(gtb.LoanMaxPeriod).Sub(time2).Seconds()*loanReserveRatio*float64(tb.Settings.Fillrate)+1e7) + re.Equal(trickle, time1.Add(gtb.LoanMaxPeriod).Sub(time2).Milliseconds()) +} From ef257250c688ab28303aa8b5b85c4dc1850826ff Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Thu, 5 Jan 2023 16:48:51 +0800 Subject: [PATCH 05/15] address comment Signed-off-by: Cabinfever_B --- go.mod | 2 +- go.sum | 2 + .../resource_manager/server/grpc_service.go | 8 ++- pkg/mcs/resource_manager/server/manager.go | 2 +- .../server/token_buckets_test.go | 16 ++--- .../resource_manager/server/token_bukets.go | 60 ++++++++++++------- pkg/mcs/resource_manager/server/types.go | 42 ++++++------- pkg/mcs/resource_manager/server/types_test.go | 39 ++++++------ 8 files changed, 94 insertions(+), 77 deletions(-) diff --git a/go.mod b/go.mod index 23cf2ecbf39..c03b04729ce 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce - github.com/pingcap/kvproto v0.0.0-20221221093947-0a9b14f1fc26 + github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d github.com/pingcap/tidb-dashboard v0.0.0-20221201151320-ea3ee6971f2e diff --git a/go.sum b/go.sum index 19c5daac87f..cbc37e05fc2 100644 --- a/go.sum +++ b/go.sum @@ -367,6 +367,8 @@ github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5 github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20221221093947-0a9b14f1fc26 h1:Tw4afZ2Tyr8iT8Oln6/szMjh5IDs+GtlnLsDo/Y2HEE= github.com/pingcap/kvproto v0.0.0-20221221093947-0a9b14f1fc26/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1 h1:jw4NjEiCleRJPPpHM7K6l8OKzOjnZAj62eKteCAY6ro= +github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= diff --git a/pkg/mcs/resource_manager/server/grpc_service.go b/pkg/mcs/resource_manager/server/grpc_service.go index 4ad156ecb40..0de75b0ab0a 100644 --- a/pkg/mcs/resource_manager/server/grpc_service.go +++ b/pkg/mcs/resource_manager/server/grpc_service.go @@ -147,7 +147,8 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu for _, req := range request.Requests { rg := s.manager.GetResourceGroup(req.ResourceGroupName) if rg == nil { - return errors.New("resource group not found") + log.Warn("resource group not found", zap.String("resource-group", req.ResourceGroupName)) + continue } now := time.Now() resp := &rmpb.TokenBucketResponse{ @@ -167,8 +168,9 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu resp.GrantedRUTokens = append(resp.GrantedRUTokens, tokens) } } - case rmpb.GroupMode_NativeMode: - return errors.New("not supports the resource type") + case rmpb.GroupMode_RawMode: + log.Warn("not supports the resource type", zap.String("resource-group", req.ResourceGroupName), zap.String("mode", rmpb.GroupMode_name[int32(rmpb.GroupMode_RawMode)])) + continue } log.Debug("finish token request from", zap.String("resource group", req.ResourceGroupName)) resps.Responses = append(resps.Responses, resp) diff --git a/pkg/mcs/resource_manager/server/manager.go b/pkg/mcs/resource_manager/server/manager.go index 5faad4f5f07..ef6fc958649 100644 --- a/pkg/mcs/resource_manager/server/manager.go +++ b/pkg/mcs/resource_manager/server/manager.go @@ -87,7 +87,7 @@ func (m *Manager) ModifyResourceGroup(group *rmpb.ResourceGroup) error { return errors.New("not exists the group") } newGroup := curGroup.Copy() - err := newGroup.PatchSettings(group.GetSettings()) + err := newGroup.PatchSettings(group) if err != nil { return err } diff --git a/pkg/mcs/resource_manager/server/token_buckets_test.go b/pkg/mcs/resource_manager/server/token_buckets_test.go index d4cb97561a0..47e7166d7c7 100644 --- a/pkg/mcs/resource_manager/server/token_buckets_test.go +++ b/pkg/mcs/resource_manager/server/token_buckets_test.go @@ -28,7 +28,7 @@ func TestGroupTokenBucketUpdateAndPatch(t *testing.T) { tbSetting := &rmpb.TokenBucket{ Tokens: 200000, Settings: &rmpb.TokenLimitSettings{ - Fillrate: 2000, + FillRate: 2000, BurstLimit: 20000000, }, } @@ -37,12 +37,12 @@ func TestGroupTokenBucketUpdateAndPatch(t *testing.T) { time1 := time.Now() tb.update(time1) re.LessOrEqual(math.Abs(tbSetting.Tokens-tb.Tokens), 1e-7) - re.Equal(tbSetting.Settings.Fillrate, tb.Settings.Fillrate) + re.Equal(tbSetting.Settings.FillRate, tb.Settings.FillRate) tbSetting = &rmpb.TokenBucket{ Tokens: -100000, Settings: &rmpb.TokenLimitSettings{ - Fillrate: 1000, + FillRate: 1000, BurstLimit: 10000000, }, } @@ -50,8 +50,8 @@ func TestGroupTokenBucketUpdateAndPatch(t *testing.T) { time2 := time.Now() tb.update(time2) - re.LessOrEqual(math.Abs(100000-tb.Tokens), time2.Sub(time1).Seconds()*float64(tbSetting.Settings.Fillrate)+1e7) - re.Equal(tbSetting.Settings.Fillrate, tb.Settings.Fillrate) + re.LessOrEqual(math.Abs(100000-tb.Tokens), time2.Sub(time1).Seconds()*float64(tbSetting.Settings.FillRate)+1e7) + re.Equal(tbSetting.Settings.FillRate, tb.Settings.FillRate) } func TestGroupTokenBucketRequest(t *testing.T) { @@ -59,7 +59,7 @@ func TestGroupTokenBucketRequest(t *testing.T) { tbSetting := &rmpb.TokenBucket{ Tokens: 200000, Settings: &rmpb.TokenLimitSettings{ - Fillrate: 2000, + FillRate: 2000, BurstLimit: 20000000, }, } @@ -77,9 +77,9 @@ func TestGroupTokenBucketRequest(t *testing.T) { time2 := time.Now() gtb.update(time2) tb, trickle = gtb.request(100000, uint64(time.Second)*10/uint64(time.Millisecond)) - re.LessOrEqual(math.Abs(tb.Tokens-19000*(1-loanReserveRatio)), time1.Add(gtb.LoanMaxPeriod).Sub(time2).Seconds()*(1-loanReserveRatio)*float64(tb.Settings.Fillrate)+1e7) + re.LessOrEqual(math.Abs(tb.Tokens-19000*(1-loanReserveRatio)), time1.Add(gtb.LoanMaxPeriod).Sub(time2).Seconds()*(1-loanReserveRatio)*float64(tb.Settings.FillRate)+1e7) re.Equal(trickle, time1.Add(gtb.LoanMaxPeriod).Sub(time2).Milliseconds()) tb, trickle = gtb.request(2000, uint64(time.Second)*10/uint64(time.Millisecond)) - re.LessOrEqual(tb.Tokens, time1.Add(gtb.LoanMaxPeriod).Sub(time2).Seconds()*loanReserveRatio*float64(tb.Settings.Fillrate)+1e7) + re.LessOrEqual(tb.Tokens, time1.Add(gtb.LoanMaxPeriod).Sub(time2).Seconds()*loanReserveRatio*float64(tb.Settings.FillRate)+1e7) re.Equal(trickle, time1.Add(gtb.LoanMaxPeriod).Sub(time2).Milliseconds()) } diff --git a/pkg/mcs/resource_manager/server/token_bukets.go b/pkg/mcs/resource_manager/server/token_bukets.go index 1cf3a84781a..caba0633d5c 100644 --- a/pkg/mcs/resource_manager/server/token_bukets.go +++ b/pkg/mcs/resource_manager/server/token_bukets.go @@ -21,6 +21,8 @@ import ( rmpb "github.com/pingcap/kvproto/pkg/resource_manager" ) +const defaultRefillRate = 10000 + const defaultInitialTokens = 10 * 10000 const defaultMaxTokens = 1e7 @@ -33,14 +35,18 @@ var loanReserveRatio float64 = 0.05 // TODO: statistics Consumption type GroupTokenBucket struct { *rmpb.TokenBucket `json:"token_bucket,omitempty"` - Consumption *rmpb.TokenBucketsRequest `json:"consumption,omitempty"` - LastUpdate *time.Time `json:"last_update,omitempty"` - Initialized bool `json:"initialized"` - LoanExpireTime *time.Time `json:"loan_time,omitempty"` - LoanMaxPeriod time.Duration `json:"loan_max_perio,omitempty"` - MaxTokens float64 `json:"max_tokens,omitempty"` + // LoanMaxPeriod represents the maximum loan period, which together with the fill rate determines the loan amount + LoanMaxPeriod time.Duration `json:"loan_max_perio,omitempty"` + // MaxTokens limits the number of tokens that can be accumulated + MaxTokens float64 `json:"max_tokens,omitempty"` + + Consumption *rmpb.TokenBucketsRequest `json:"consumption,omitempty"` + LastUpdate *time.Time `json:"last_update,omitempty"` + Initialized bool `json:"initialized"` + LoanExpireTime *time.Time `json:"loan_time,omitempty"` } +// NewGroupTokenBucket returns a new GroupTokenBucket func NewGroupTokenBucket(tokenBucket *rmpb.TokenBucket) GroupTokenBucket { return GroupTokenBucket{ TokenBucket: tokenBucket, @@ -70,6 +76,9 @@ func (t *GroupTokenBucket) patch(settings *rmpb.TokenBucket) { // update updates the token bucket. func (t *GroupTokenBucket) update(now time.Time) { if !t.Initialized { + if t.Settings.FillRate == 0 { + t.Settings.FillRate = defaultRefillRate + } if t.Tokens < defaultInitialTokens { t.Tokens = defaultInitialTokens } @@ -80,14 +89,14 @@ func (t *GroupTokenBucket) update(now time.Time) { delta := now.Sub(*t.LastUpdate) if delta > 0 { - t.Tokens += float64(t.Settings.Fillrate) * delta.Seconds() + t.Tokens += float64(t.Settings.FillRate) * delta.Seconds() t.LastUpdate = &now } if t.Tokens >= 0 { t.LoanExpireTime = nil } - if t.Tokens > defaultMaxTokens { - t.Tokens = defaultMaxTokens + if t.Tokens > t.MaxTokens { + t.Tokens = t.MaxTokens } } @@ -97,12 +106,13 @@ func (t *GroupTokenBucket) request( ) (*rmpb.TokenBucket, int64) { var res rmpb.TokenBucket res.Settings = &rmpb.TokenLimitSettings{} - // TODO: consider the shares for dispatch the fill rate - res.Settings.Fillrate = 0 + // FillRate is used for the token server unavailable in abnormal situation. + res.Settings.FillRate = 0 if neededTokens <= 0 { return &res, 0 } + // If the current tokens can directly meet the requirement, returns the need token if t.Tokens >= neededTokens { t.Tokens -= neededTokens // granted the total request tokens @@ -110,6 +120,7 @@ func (t *GroupTokenBucket) request( return &res, 0 } + // Firstly allocate the remaining tokens var grantedTokens float64 if t.Tokens > 0 { grantedTokens = t.Tokens @@ -117,20 +128,27 @@ func (t *GroupTokenBucket) request( neededTokens -= grantedTokens } + // Consider using a loan to get tokens var periodFilled float64 - var trickleTime = int64(targetPeriodMs) - if t.LoanExpireTime != nil && t.LoanExpireTime.After(*t.LastUpdate) { - duration := t.LoanExpireTime.Sub(*t.LastUpdate) - periodFilled = float64(t.Settings.Fillrate) * (1 - loanReserveRatio) * duration.Seconds() - trickleTime = duration.Milliseconds() - } else { + var trickleTime = time.Duration(targetPeriodMs) * time.Millisecond + // If the loan has been used, and within the expiration date of the loan, + // We calculate `periodFilled` which is the number of tokens that can be allocated + // according to fillrate, remaining loan time and retention ratio + if t.LoanExpireTime != nil { + if t.LoanExpireTime.After(*t.LastUpdate) { + duration := t.LoanExpireTime.Sub(*t.LastUpdate) + periodFilled = float64(t.Settings.FillRate) * (1 - loanReserveRatio) * duration.Seconds() + trickleTime = duration + } + } else { // Apply for a loan et := t.LastUpdate.Add(t.LoanMaxPeriod) t.LoanExpireTime = &et - periodFilled = float64(t.Settings.Fillrate) * (1 - loanReserveRatio) * t.LoanMaxPeriod.Seconds() + periodFilled = float64(t.Settings.FillRate) * (1 - loanReserveRatio) * t.LoanMaxPeriod.Seconds() } + // have to deduct what resource group already owe periodFilled += t.Tokens - if periodFilled <= float64(t.Settings.Fillrate)*loanReserveRatio { - periodFilled = float64(t.Settings.Fillrate) * loanReserveRatio + if periodFilled <= float64(t.Settings.FillRate)*loanReserveRatio*trickleTime.Seconds() { + periodFilled = float64(t.Settings.FillRate) * loanReserveRatio * trickleTime.Seconds() } if periodFilled >= neededTokens { grantedTokens += neededTokens @@ -140,5 +158,5 @@ func (t *GroupTokenBucket) request( t.Tokens -= periodFilled } res.Tokens = grantedTokens - return &res, trickleTime + return &res, trickleTime.Milliseconds() } diff --git a/pkg/mcs/resource_manager/server/types.go b/pkg/mcs/resource_manager/server/types.go index 2f6968fe196..515b555bff5 100644 --- a/pkg/mcs/resource_manager/server/types.go +++ b/pkg/mcs/resource_manager/server/types.go @@ -82,7 +82,7 @@ func (rg *ResourceGroup) CheckAndInit() error { if len(rg.Name) == 0 || len(rg.Name) > 32 { return errors.New("invalid resource group name, the length should be in [1,32]") } - if rg.Mode != rmpb.GroupMode_RUMode && rg.Mode != rmpb.GroupMode_NativeMode { + if rg.Mode != rmpb.GroupMode_RUMode && rg.Mode != rmpb.GroupMode_RawMode { return errors.New("invalid resource group mode") } if rg.Mode == rmpb.GroupMode_RUMode { @@ -93,7 +93,7 @@ func (rg *ResourceGroup) CheckAndInit() error { return errors.New("invalid resource group settings, RU mode should not set resource settings") } } - if rg.Mode == rmpb.GroupMode_NativeMode { + if rg.Mode == rmpb.GroupMode_RawMode { if rg.ResourceSettings == nil { rg.ResourceSettings = &NativeResourceSettings{} } @@ -107,7 +107,7 @@ func (rg *ResourceGroup) CheckAndInit() error { // PatchSettings patches the resource group settings. // Only used to patch the resource group when updating. // Note: the tokens is the delta value to patch. -func (rg *ResourceGroup) PatchSettings(groupSettings *rmpb.GroupSettings) error { +func (rg *ResourceGroup) PatchSettings(groupSettings *rmpb.ResourceGroup) error { rg.Lock() defer rg.Unlock() if groupSettings.GetMode() != rg.Mode { @@ -120,7 +120,7 @@ func (rg *ResourceGroup) PatchSettings(groupSettings *rmpb.GroupSettings) error } rg.RUSettings.RRU.patch(groupSettings.GetRUSettings().GetRRU()) rg.RUSettings.WRU.patch(groupSettings.GetRUSettings().GetWRU()) - case rmpb.GroupMode_NativeMode: + case rmpb.GroupMode_RawMode: if groupSettings.GetResourceSettings() == nil { return errors.New("invalid resource group settings, native mode should set resource settings") } @@ -141,19 +141,19 @@ func FromProtoResourceGroup(group *rmpb.ResourceGroup) *ResourceGroup { rg := &ResourceGroup{ Name: group.Name, - Mode: group.Settings.Mode, + Mode: group.Mode, } - switch group.GetSettings().GetMode() { + switch group.GetMode() { case rmpb.GroupMode_RUMode: - if settings := group.GetSettings().GetRUSettings(); settings != nil { + if settings := group.GetRUSettings(); settings != nil { ruSettings = &RequestUnitSettings{ RRU: NewGroupTokenBucket(settings.GetRRU()), WRU: NewGroupTokenBucket(settings.GetWRU()), } rg.RUSettings = ruSettings } - case rmpb.GroupMode_NativeMode: - if settings := group.GetSettings().GetResourceSettings(); settings != nil { + case rmpb.GroupMode_RawMode: + if settings := group.GetResourceSettings(); settings != nil { resourceSettings = &NativeResourceSettings{ CPU: NewGroupTokenBucket(settings.GetCpu()), IOReadBandwidth: NewGroupTokenBucket(settings.GetIoRead()), @@ -213,25 +213,21 @@ func (rg *ResourceGroup) IntoProtoResourceGroup() *rmpb.ResourceGroup { case rmpb.GroupMode_RUMode: // RU mode group := &rmpb.ResourceGroup{ Name: rg.Name, - Settings: &rmpb.GroupSettings{ - Mode: rmpb.GroupMode_RUMode, - RUSettings: &rmpb.GroupRequestUnitSettings{ - RRU: rg.RUSettings.RRU.TokenBucket, - WRU: rg.RUSettings.WRU.TokenBucket, - }, + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RRU: rg.RUSettings.RRU.TokenBucket, + WRU: rg.RUSettings.WRU.TokenBucket, }, } return group - case rmpb.GroupMode_NativeMode: // Native mode + case rmpb.GroupMode_RawMode: // Raw mode group := &rmpb.ResourceGroup{ Name: rg.Name, - Settings: &rmpb.GroupSettings{ - Mode: rmpb.GroupMode_NativeMode, - ResourceSettings: &rmpb.GroupResourceSettings{ - Cpu: rg.ResourceSettings.CPU.TokenBucket, - IoRead: rg.ResourceSettings.IOReadBandwidth.TokenBucket, - IoWrite: rg.ResourceSettings.IOWriteBandwidth.TokenBucket, - }, + Mode: rmpb.GroupMode_RawMode, + ResourceSettings: &rmpb.GroupResourceSettings{ + Cpu: rg.ResourceSettings.CPU.TokenBucket, + IoRead: rg.ResourceSettings.IOReadBandwidth.TokenBucket, + IoWrite: rg.ResourceSettings.IOWriteBandwidth.TokenBucket, }, } return group diff --git a/pkg/mcs/resource_manager/server/types_test.go b/pkg/mcs/resource_manager/server/types_test.go index 1e22559f591..ed7750217a2 100644 --- a/pkg/mcs/resource_manager/server/types_test.go +++ b/pkg/mcs/resource_manager/server/types_test.go @@ -17,21 +17,21 @@ func TestPatchResourceGroup(t *testing.T) { patchJSONString string expectJSONString string }{ - {`{"mode":0, "r_u_settings": {"r_r_u":{"settings":{"fillrate": 200000}}}}`, - `{"name":"test","mode":0,"r_u_settings":{"rru":{"token_bucket":{"settings":{"fillrate":200000}},"initialized":false},"wru":{"initialized":false}}}`}, - {`{"mode":0, "r_u_settings": {"w_r_u":{"settings":{"fillrate": 200000}}}}`, - `{"name":"test","mode":0,"r_u_settings":{"rru":{"initialized":false},"wru":{"token_bucket":{"settings":{"fillrate":200000}},"initialized":false}}}`}, - {`{"mode":0, "r_u_settings": {"w_r_u":{"settings":{"fillrate": 200000, "burst": 100000}}}}`, - `{"name":"test","mode":0,"r_u_settings":{"rru":{"initialized":false},"wru":{"token_bucket":{"settings":{"fillrate":200000}},"initialized":false}}}`}, - {`{"mode":0, "r_u_settings": {"r_r_u":{"settings":{"fillrate": 200000, "burst": 100000}}}}`, - `{"name":"test","mode":0,"r_u_settings":{"rru":{"token_bucket":{"settings":{"fillrate":200000}},"initialized":false},"wru":{"initialized":false}}}`}, - {`{"mode":0, "r_u_settings": {"r_r_u":{"settings":{"fillrate": 200000, "burst": 100000}}, "w_r_u":{"settings":{"fillrate": 200000}}}}`, - `{"name":"test","mode":0,"r_u_settings":{"rru":{"token_bucket":{"settings":{"fillrate":200000}},"initialized":false},"wru":{"token_bucket":{"settings":{"fillrate":200000}},"initialized":false}}}`}, + {`{"name":"test", "mode":0, "r_u_settings": {"r_r_u":{"settings":{"fill_rate": 200000}}}}`, + `{"name":"test","mode":0,"r_u_settings":{"rru":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false},"wru":{"initialized":false}}}`}, + {`{"name":"test", "mode":0, "r_u_settings": {"w_r_u":{"settings":{"fill_rate": 200000}}}}`, + `{"name":"test","mode":0,"r_u_settings":{"rru":{"initialized":false},"wru":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false}}}`}, + {`{"name":"test", "mode":0, "r_u_settings": {"w_r_u":{"settings":{"fill_rate": 200000, "burst": 100000}}}}`, + `{"name":"test","mode":0,"r_u_settings":{"rru":{"initialized":false},"wru":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false}}}`}, + {`{"name":"test", "mode":0, "r_u_settings": {"r_r_u":{"settings":{"fill_rate": 200000, "burst": 100000}}}}`, + `{"name":"test","mode":0,"r_u_settings":{"rru":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false},"wru":{"initialized":false}}}`}, + {`{"name":"test", "mode":0, "r_u_settings": {"r_r_u":{"settings":{"fill_rate": 200000, "burst": 100000}}, "w_r_u":{"settings":{"fill_rate": 200000}}}}`, + `{"name":"test","mode":0,"r_u_settings":{"rru":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false},"wru":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false}}}`}, } for _, ca := range testCaseRU { rg := rg1.Copy() - patch := &rmpb.GroupSettings{} + patch := &rmpb.ResourceGroup{} err := json.Unmarshal([]byte(ca.patchJSONString), patch) re.NoError(err) err = rg.PatchSettings(patch) @@ -41,24 +41,23 @@ func TestPatchResourceGroup(t *testing.T) { re.Equal(ca.expectJSONString, string(res)) } - rg2 := &ResourceGroup{Name: "test", Mode: rmpb.GroupMode_NativeMode} + rg2 := &ResourceGroup{Name: "test", Mode: rmpb.GroupMode_RawMode} err = rg2.CheckAndInit() re.NoError(err) testCaseResource := []struct { patchJSONString string expectJSONString string }{ - {`{"mode":1, "resource_settings": {"cpu":{"settings":{"fillrate": 200000}}}}`, - `{"name":"test","mode":1,"resource_settings":{"cpu":{"token_bucket":{"settings":{"fillrate":200000}},"initialized":false},"io_read_bandwidth":{"initialized":false},"io_write_bandwidth":{"initialized":false}}}`}, - {`{"mode":1, "resource_settings": {"io_read":{"settings":{"fillrate": 200000}}}}`, - `{"name":"test","mode":1,"resource_settings":{"cpu":{"initialized":false},"io_read_bandwidth":{"token_bucket":{"settings":{"fillrate":200000}},"initialized":false},"io_write_bandwidth":{"initialized":false}}}`}, - {`{"mode":1, "resource_settings": {"io_write":{"settings":{"fillrate": 200000}}}}`, - `{"name":"test","mode":1,"resource_settings":{"cpu":{"initialized":false},"io_read_bandwidth":{"initialized":false},"io_write_bandwidth":{"token_bucket":{"settings":{"fillrate":200000}},"initialized":false}}}`}, + {`{"name":"test", "mode":1, "resource_settings": {"cpu":{"settings":{"fill_rate": 200000}}}}`, + `{"name":"test","mode":1,"resource_settings":{"cpu":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false},"io_read_bandwidth":{"initialized":false},"io_write_bandwidth":{"initialized":false}}}`}, + {`{"name":"test", "mode":1, "resource_settings": {"io_read":{"settings":{"fill_rate": 200000}}}}`, + `{"name":"test","mode":1,"resource_settings":{"cpu":{"initialized":false},"io_read_bandwidth":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false},"io_write_bandwidth":{"initialized":false}}}`}, + {`{"name":"test", "mode":1, "resource_settings": {"io_write":{"settings":{"fill_rate": 200000}}}}`, + `{"name":"test","mode":1,"resource_settings":{"cpu":{"initialized":false},"io_read_bandwidth":{"initialized":false},"io_write_bandwidth":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false}}}`}, } - for _, ca := range testCaseResource { rg := rg2.Copy() - patch := &rmpb.GroupSettings{} + patch := &rmpb.ResourceGroup{} err := json.Unmarshal([]byte(ca.patchJSONString), patch) re.NoError(err) err = rg.PatchSettings(patch) From 6a13caba65fb089f4d41b6038fdfa8020de52013 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Thu, 5 Jan 2023 18:48:55 +0800 Subject: [PATCH 06/15] address comment Signed-off-by: Cabinfever_B --- .../server/token_buckets_test.go | 21 ++++--- .../resource_manager/server/token_bukets.go | 60 +++++++------------ 2 files changed, 34 insertions(+), 47 deletions(-) diff --git a/pkg/mcs/resource_manager/server/token_buckets_test.go b/pkg/mcs/resource_manager/server/token_buckets_test.go index 47e7166d7c7..f897c589d7c 100644 --- a/pkg/mcs/resource_manager/server/token_buckets_test.go +++ b/pkg/mcs/resource_manager/server/token_buckets_test.go @@ -73,13 +73,18 @@ func TestGroupTokenBucketRequest(t *testing.T) { tb, trickle = gtb.request(101000, uint64(time.Second)*10/uint64(time.Millisecond)) re.LessOrEqual(math.Abs(tb.Tokens-101000), 1e-7) re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) - re.Equal(*gtb.LoanExpireTime, time1.Add(gtb.LoanMaxPeriod)) - time2 := time.Now() + tb, trickle = gtb.request(17000, uint64(time.Second)*10/uint64(time.Millisecond)) + re.LessOrEqual(math.Abs(tb.Tokens-17000), 1e-7) + re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) + tb, trickle = gtb.request(4000, uint64(time.Second)*10/uint64(time.Millisecond)) + re.LessOrEqual(math.Abs(tb.Tokens-4000), 1e-7) + re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) + tb, trickle = gtb.request(19000, uint64(time.Second)*10/uint64(time.Millisecond)) + re.LessOrEqual(math.Abs(tb.Tokens-18000), 1e-7) + re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) + time2 := time1.Add(10 * time.Second) gtb.update(time2) - tb, trickle = gtb.request(100000, uint64(time.Second)*10/uint64(time.Millisecond)) - re.LessOrEqual(math.Abs(tb.Tokens-19000*(1-loanReserveRatio)), time1.Add(gtb.LoanMaxPeriod).Sub(time2).Seconds()*(1-loanReserveRatio)*float64(tb.Settings.FillRate)+1e7) - re.Equal(trickle, time1.Add(gtb.LoanMaxPeriod).Sub(time2).Milliseconds()) - tb, trickle = gtb.request(2000, uint64(time.Second)*10/uint64(time.Millisecond)) - re.LessOrEqual(tb.Tokens, time1.Add(gtb.LoanMaxPeriod).Sub(time2).Seconds()*loanReserveRatio*float64(tb.Settings.FillRate)+1e7) - re.Equal(trickle, time1.Add(gtb.LoanMaxPeriod).Sub(time2).Milliseconds()) + tb, trickle = gtb.request(20000, uint64(time.Second)*10/uint64(time.Millisecond)) + re.LessOrEqual(math.Abs(tb.Tokens-20000), 1e-7) + re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) } diff --git a/pkg/mcs/resource_manager/server/token_bukets.go b/pkg/mcs/resource_manager/server/token_bukets.go index caba0633d5c..bcf4356e83c 100644 --- a/pkg/mcs/resource_manager/server/token_bukets.go +++ b/pkg/mcs/resource_manager/server/token_bukets.go @@ -15,6 +15,8 @@ package server import ( + "fmt" + "math" "time" "github.com/gogo/protobuf/proto" @@ -27,31 +29,25 @@ const defaultInitialTokens = 10 * 10000 const defaultMaxTokens = 1e7 -const defaultLoanMaxPeriod = 10 * time.Second - -var loanReserveRatio float64 = 0.05 +var reserveRatio float64 = 0.05 // GroupTokenBucket is a token bucket for a resource group. // TODO: statistics Consumption type GroupTokenBucket struct { *rmpb.TokenBucket `json:"token_bucket,omitempty"` - // LoanMaxPeriod represents the maximum loan period, which together with the fill rate determines the loan amount - LoanMaxPeriod time.Duration `json:"loan_max_perio,omitempty"` // MaxTokens limits the number of tokens that can be accumulated MaxTokens float64 `json:"max_tokens,omitempty"` - Consumption *rmpb.TokenBucketsRequest `json:"consumption,omitempty"` - LastUpdate *time.Time `json:"last_update,omitempty"` - Initialized bool `json:"initialized"` - LoanExpireTime *time.Time `json:"loan_time,omitempty"` + Consumption *rmpb.TokenBucketsRequest `json:"consumption,omitempty"` + LastUpdate *time.Time `json:"last_update,omitempty"` + Initialized bool `json:"initialized"` } // NewGroupTokenBucket returns a new GroupTokenBucket func NewGroupTokenBucket(tokenBucket *rmpb.TokenBucket) GroupTokenBucket { return GroupTokenBucket{ - TokenBucket: tokenBucket, - MaxTokens: defaultMaxTokens, - LoanMaxPeriod: defaultLoanMaxPeriod, + TokenBucket: tokenBucket, + MaxTokens: defaultMaxTokens, } } @@ -92,9 +88,6 @@ func (t *GroupTokenBucket) update(now time.Time) { t.Tokens += float64(t.Settings.FillRate) * delta.Seconds() t.LastUpdate = &now } - if t.Tokens >= 0 { - t.LoanExpireTime = nil - } if t.Tokens > t.MaxTokens { t.Tokens = t.MaxTokens } @@ -124,39 +117,28 @@ func (t *GroupTokenBucket) request( var grantedTokens float64 if t.Tokens > 0 { grantedTokens = t.Tokens - t.Tokens = 0 neededTokens -= grantedTokens } - // Consider using a loan to get tokens - var periodFilled float64 var trickleTime = time.Duration(targetPeriodMs) * time.Millisecond - // If the loan has been used, and within the expiration date of the loan, - // We calculate `periodFilled` which is the number of tokens that can be allocated - // according to fillrate, remaining loan time and retention ratio - if t.LoanExpireTime != nil { - if t.LoanExpireTime.After(*t.LastUpdate) { - duration := t.LoanExpireTime.Sub(*t.LastUpdate) - periodFilled = float64(t.Settings.FillRate) * (1 - loanReserveRatio) * duration.Seconds() - trickleTime = duration + availableRate := float64(t.Settings.FillRate) + if debt := -t.Tokens; debt > 0 { + fmt.Println(debt) + debt -= float64(t.Settings.FillRate) * trickleTime.Seconds() + if debt > 0 { + debtRate := debt / float64(targetPeriodMs/1000) + availableRate -= debtRate + availableRate = math.Max(availableRate, reserveRatio*float64(t.Settings.FillRate)) } - } else { // Apply for a loan - et := t.LastUpdate.Add(t.LoanMaxPeriod) - t.LoanExpireTime = &et - periodFilled = float64(t.Settings.FillRate) * (1 - loanReserveRatio) * t.LoanMaxPeriod.Seconds() - } - // have to deduct what resource group already owe - periodFilled += t.Tokens - if periodFilled <= float64(t.Settings.FillRate)*loanReserveRatio*trickleTime.Seconds() { - periodFilled = float64(t.Settings.FillRate) * loanReserveRatio * trickleTime.Seconds() } - if periodFilled >= neededTokens { + + consumptionDuration := time.Duration(float64(time.Second) * (neededTokens / availableRate)) + if consumptionDuration <= trickleTime { grantedTokens += neededTokens - t.Tokens -= neededTokens } else { - grantedTokens += periodFilled - t.Tokens -= periodFilled + grantedTokens += availableRate * trickleTime.Seconds() } + t.Tokens -= grantedTokens res.Tokens = grantedTokens return &res, trickleTime.Milliseconds() } From 6f18863419530ee223d5f5ab5c17180f53405ca7 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Fri, 6 Jan 2023 14:17:05 +0800 Subject: [PATCH 07/15] address comment Signed-off-by: Cabinfever_B --- client/resoucemanager_client.go | 265 -------------------------------- 1 file changed, 265 deletions(-) delete mode 100644 client/resoucemanager_client.go diff --git a/client/resoucemanager_client.go b/client/resoucemanager_client.go deleted file mode 100644 index 738e75c789d..00000000000 --- a/client/resoucemanager_client.go +++ /dev/null @@ -1,265 +0,0 @@ -package pd - -import ( - "context" - "time" - - "github.com/pingcap/errors" - rmpb "github.com/pingcap/kvproto/pkg/resource_manager" - "github.com/pingcap/log" - "go.uber.org/zap" - "google.golang.org/grpc" -) - -// KeyspaceClient manages keyspace metadata. -type ResourceManagerClient interface { - ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) - GetResourceGroup(ctx context.Context, resourceGroupName string) (*rmpb.ResourceGroup, error) - AddResourceGroup(ctx context.Context, resourceGroupName string, settings *rmpb.GroupSettings) (string, error) - ModifyResourceGroup(ctx context.Context, resourceGroupName string, settings *rmpb.GroupSettings) (string, error) - DeleteResourceGroup(ctx context.Context, resourceGroupName string) (string, error) - AcquireTokenBuckets(ctx context.Context, request *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) -} - -// leaderClient gets the client of current PD leader. -func (c *client) resouceManagerClient() rmpb.ResourceManagerClient { - if cc, ok := c.clientConns.Load(c.GetLeaderAddr()); ok { - return rmpb.NewResourceManagerClient(cc.(*grpc.ClientConn)) - } - return nil -} - -// ListResourceGroups loads and returns target keyspace's metadata. -func (c *client) ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) { - req := &rmpb.ListResourceGroupsRequest{} - resp, err := c.resouceManagerClient().ListResourceGroups(ctx, req) - if err != nil { - return nil, err - } - resErr := resp.GetError() - if resErr != nil { - return nil, errors.Errorf("[pd]" + resErr.Message) - } - return resp.GetGroups(), nil -} - -func (c *client) GetResourceGroup(ctx context.Context, resourceGroupName string) (*rmpb.ResourceGroup, error) { - req := &rmpb.GetResourceGroupRequest{ - ResourceGroupName: resourceGroupName, - } - resp, err := c.resouceManagerClient().GetResourceGroup(ctx, req) - if err != nil { - return nil, err - } - resErr := resp.GetError() - if resErr != nil { - return nil, errors.Errorf("[pd]" + resErr.Message) - } - return resp.GetGroup(), nil -} - -func (c *client) AddResourceGroup(ctx context.Context, resourceGroupName string, settings *rmpb.GroupSettings) (string, error) { - return c.putResourceGroup(ctx, resourceGroupName, settings, 0 /* type add resource group */) -} - -func (c *client) ModifyResourceGroup(ctx context.Context, resourceGroupName string, settings *rmpb.GroupSettings) (string, error) { - return c.putResourceGroup(ctx, resourceGroupName, settings, 1 /* type modify resource group */) -} - -func (c *client) putResourceGroup(ctx context.Context, resourceGroupName string, settings *rmpb.GroupSettings, typ int) (str string, err error) { - group := &rmpb.ResourceGroup{ - Name: resourceGroupName, - Settings: settings, - } - req := &rmpb.PutResourceGroupRequest{ - Group: group, - } - var resp *rmpb.PutResourceGroupResponse - if typ == 0 { - resp, err = c.resouceManagerClient().AddResourceGroup(ctx, req) - } else { - resp, err = c.resouceManagerClient().ModifyResourceGroup(ctx, req) - } - if err != nil { - return str, err - } - resErr := resp.GetError() - if resErr != nil { - return str, errors.Errorf("[pd]" + resErr.Message) - } - str = resp.GetBody() - return -} - -func (c *client) DeleteResourceGroup(ctx context.Context, resourceGroupName string) (string, error) { - req := &rmpb.DeleteResourceGroupRequest{ - ResourceGroupName: resourceGroupName, - } - resp, err := c.resouceManagerClient().DeleteResourceGroup(ctx, req) - if err != nil { - return "", err - } - resErr := resp.GetError() - if resErr != nil { - return "", errors.Errorf("[pd]" + resErr.Message) - } - return resp.GetBody(), nil -} - -func (c *client) AcquireTokenBuckets(ctx context.Context, request *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) { - req := &tokenRequest{ - done: make(chan error, 1), - requestCtx: ctx, - clientCtx: c.ctx, - } - req.Requeset = request - c.tokenDispatcher.tokenBatchController.tokenRequestCh <- req - grantedTokens, err := req.Wait() - if err != nil { - return nil, err - } - return grantedTokens, err -} - -type tokenRequest struct { - clientCtx context.Context - requestCtx context.Context - done chan error - Requeset *rmpb.TokenBucketsRequest - TokenBuckets []*rmpb.TokenBucketResponse -} - -func (req *tokenRequest) Wait() (tokenBuckets []*rmpb.TokenBucketResponse, err error) { - select { - case err = <-req.done: - err = errors.WithStack(err) - if err != nil { - return nil, err - } - tokenBuckets = req.TokenBuckets - return - case <-req.requestCtx.Done(): - return nil, errors.WithStack(req.requestCtx.Err()) - case <-req.clientCtx.Done(): - return nil, errors.WithStack(req.clientCtx.Err()) - } -} - -type tokenBatchController struct { - tokenRequestCh chan *tokenRequest -} - -func newTokenBatchController(tokenRequestCh chan *tokenRequest) *tokenBatchController { - return &tokenBatchController{ - tokenRequestCh: tokenRequestCh, - } -} - -type tokenDispatcher struct { - dispatcherCancel context.CancelFunc - tokenBatchController *tokenBatchController -} - -type resourceManagerConnectionContext struct { - stream rmpb.ResourceManager_AcquireTokenBucketsClient - ctx context.Context - cancel context.CancelFunc -} - -func (c *client) createTokenispatcher() { - dispatcherCtx, dispatcherCancel := context.WithCancel(c.ctx) - dispatcher := &tokenDispatcher{ - dispatcherCancel: dispatcherCancel, - tokenBatchController: newTokenBatchController( - make(chan *tokenRequest, 1)), - } - go c.handleResouceTokenDispatcher(dispatcherCtx, dispatcher.tokenBatchController) - c.tokenDispatcher = dispatcher -} - -func (c *client) handleResouceTokenDispatcher(dispatcherCtx context.Context, tbc *tokenBatchController) { - var connection resourceManagerConnectionContext - if err := c.tryResourceManagerConnect(dispatcherCtx, &connection); err != nil { - log.Warn("get stream error", zap.Error(err)) - } - - for { - var firstTSORequest *tokenRequest - select { - case <-dispatcherCtx.Done(): - return - case firstTSORequest = <-tbc.tokenRequestCh: - } - stream, streamCtx, cancel := connection.stream, connection.ctx, connection.cancel - if stream == nil { - c.tryResourceManagerConnect(dispatcherCtx, &connection) - c.finishTokenRequest(firstTSORequest, nil, errors.Errorf("no stream")) - continue - } - select { - case <-streamCtx.Done(): - log.Info("[pd] resource manager stream is canceled") - cancel() - stream = nil - continue - default: - } - err := c.processTokenRequests(stream, firstTSORequest) - if err != nil { - log.Info("processTokenRequests error", zap.Error(err)) - cancel() - connection.stream = nil - } - } - -} - -func (c *client) processTokenRequests(stream rmpb.ResourceManager_AcquireTokenBucketsClient, t *tokenRequest) error { - req := t.Requeset - if err := stream.Send(req); err != nil { - err = errors.WithStack(err) - c.finishTokenRequest(t, nil, err) - return err - } - resp, err := stream.Recv() - if err != nil { - err = errors.WithStack(err) - c.finishTokenRequest(t, nil, err) - return err - } - if resp.GetError() != nil { - return errors.Errorf("[pd]" + resp.GetError().Message) - } - tokenBuckets := resp.GetResponses() - c.finishTokenRequest(t, tokenBuckets, nil) - return nil -} - -func (c *client) finishTokenRequest(t *tokenRequest, tokenBuckets []*rmpb.TokenBucketResponse, err error) { - t.TokenBuckets = tokenBuckets - t.done <- err -} - -func (c *client) tryResourceManagerConnect(ctx context.Context, connection *resourceManagerConnectionContext) error { - var ( - err error - stream rmpb.ResourceManager_AcquireTokenBucketsClient - ) - for i := 0; i < maxRetryTimes; i++ { - cctx, cancel := context.WithCancel(ctx) - stream, err = c.resouceManagerClient().AcquireTokenBuckets(cctx) - if err == nil && stream != nil { - connection.cancel = cancel - connection.ctx = cctx - connection.stream = stream - return nil - } - cancel() - select { - case <-ctx.Done(): - return err - case <-time.After(retryInterval): - } - } - return err -} From 887d22157d29a5e39fe6684659877a003a77b174 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Fri, 6 Jan 2023 17:19:50 +0800 Subject: [PATCH 08/15] merge master Signed-off-by: Cabinfever_B --- pkg/mcs/resource_manager/server/types_test.go | 69 ------------------- 1 file changed, 69 deletions(-) delete mode 100644 pkg/mcs/resource_manager/server/types_test.go diff --git a/pkg/mcs/resource_manager/server/types_test.go b/pkg/mcs/resource_manager/server/types_test.go deleted file mode 100644 index ed7750217a2..00000000000 --- a/pkg/mcs/resource_manager/server/types_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package server - -import ( - "encoding/json" - "testing" - - rmpb "github.com/pingcap/kvproto/pkg/resource_manager" - "github.com/stretchr/testify/require" -) - -func TestPatchResourceGroup(t *testing.T) { - re := require.New(t) - rg1 := &ResourceGroup{Name: "test", Mode: rmpb.GroupMode_RUMode} - err := rg1.CheckAndInit() - re.NoError(err) - testCaseRU := []struct { - patchJSONString string - expectJSONString string - }{ - {`{"name":"test", "mode":0, "r_u_settings": {"r_r_u":{"settings":{"fill_rate": 200000}}}}`, - `{"name":"test","mode":0,"r_u_settings":{"rru":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false},"wru":{"initialized":false}}}`}, - {`{"name":"test", "mode":0, "r_u_settings": {"w_r_u":{"settings":{"fill_rate": 200000}}}}`, - `{"name":"test","mode":0,"r_u_settings":{"rru":{"initialized":false},"wru":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false}}}`}, - {`{"name":"test", "mode":0, "r_u_settings": {"w_r_u":{"settings":{"fill_rate": 200000, "burst": 100000}}}}`, - `{"name":"test","mode":0,"r_u_settings":{"rru":{"initialized":false},"wru":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false}}}`}, - {`{"name":"test", "mode":0, "r_u_settings": {"r_r_u":{"settings":{"fill_rate": 200000, "burst": 100000}}}}`, - `{"name":"test","mode":0,"r_u_settings":{"rru":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false},"wru":{"initialized":false}}}`}, - {`{"name":"test", "mode":0, "r_u_settings": {"r_r_u":{"settings":{"fill_rate": 200000, "burst": 100000}}, "w_r_u":{"settings":{"fill_rate": 200000}}}}`, - `{"name":"test","mode":0,"r_u_settings":{"rru":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false},"wru":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false}}}`}, - } - - for _, ca := range testCaseRU { - rg := rg1.Copy() - patch := &rmpb.ResourceGroup{} - err := json.Unmarshal([]byte(ca.patchJSONString), patch) - re.NoError(err) - err = rg.PatchSettings(patch) - re.NoError(err) - res, err := json.Marshal(rg) - re.NoError(err) - re.Equal(ca.expectJSONString, string(res)) - } - - rg2 := &ResourceGroup{Name: "test", Mode: rmpb.GroupMode_RawMode} - err = rg2.CheckAndInit() - re.NoError(err) - testCaseResource := []struct { - patchJSONString string - expectJSONString string - }{ - {`{"name":"test", "mode":1, "resource_settings": {"cpu":{"settings":{"fill_rate": 200000}}}}`, - `{"name":"test","mode":1,"resource_settings":{"cpu":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false},"io_read_bandwidth":{"initialized":false},"io_write_bandwidth":{"initialized":false}}}`}, - {`{"name":"test", "mode":1, "resource_settings": {"io_read":{"settings":{"fill_rate": 200000}}}}`, - `{"name":"test","mode":1,"resource_settings":{"cpu":{"initialized":false},"io_read_bandwidth":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false},"io_write_bandwidth":{"initialized":false}}}`}, - {`{"name":"test", "mode":1, "resource_settings": {"io_write":{"settings":{"fill_rate": 200000}}}}`, - `{"name":"test","mode":1,"resource_settings":{"cpu":{"initialized":false},"io_read_bandwidth":{"initialized":false},"io_write_bandwidth":{"token_bucket":{"settings":{"fill_rate":200000}},"initialized":false}}}`}, - } - for _, ca := range testCaseResource { - rg := rg2.Copy() - patch := &rmpb.ResourceGroup{} - err := json.Unmarshal([]byte(ca.patchJSONString), patch) - re.NoError(err) - err = rg.PatchSettings(patch) - re.NoError(err) - res, err := json.Marshal(rg) - re.NoError(err) - re.Equal(ca.expectJSONString, string(res)) - } -} From 7bc3bc9cc5c1612ba6fa0ae669290fba779cef28 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Fri, 6 Jan 2023 18:32:17 +0800 Subject: [PATCH 09/15] address comment Signed-off-by: Cabinfever_B --- client/go.mod | 2 +- client/go.sum | 4 +- client/resourcemanager_client.go | 282 +++++++++++++++++++++++++++++++ 3 files changed, 285 insertions(+), 3 deletions(-) create mode 100644 client/resourcemanager_client.go diff --git a/client/go.mod b/client/go.mod index df2e176f79e..2257442da1a 100644 --- a/client/go.mod +++ b/client/go.mod @@ -6,7 +6,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20221221093947-0a9b14f1fc26 + github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.11.0 github.com/stretchr/testify v1.7.0 diff --git a/client/go.sum b/client/go.sum index 09ac827c37b..a07369118ea 100644 --- a/client/go.sum +++ b/client/go.sum @@ -104,8 +104,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20221221093947-0a9b14f1fc26 h1:Tw4afZ2Tyr8iT8Oln6/szMjh5IDs+GtlnLsDo/Y2HEE= -github.com/pingcap/kvproto v0.0.0-20221221093947-0a9b14f1fc26/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1 h1:jw4NjEiCleRJPPpHM7K6l8OKzOjnZAj62eKteCAY6ro= +github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/client/resourcemanager_client.go b/client/resourcemanager_client.go new file mode 100644 index 00000000000..caaa5a8c987 --- /dev/null +++ b/client/resourcemanager_client.go @@ -0,0 +1,282 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pd + +import ( + "context" + "time" + + "github.com/pingcap/errors" + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/pingcap/log" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +type actionType int + +const ( + add actionType = 0 + modify actionType = 1 +) + +// ResourceManagerClient manages resource group info and token request. +type ResourceManagerClient interface { + ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) + GetResourceGroup(ctx context.Context, resourceGroupName string) (*rmpb.ResourceGroup, error) + AddResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) + ModifyResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) + DeleteResourceGroup(ctx context.Context, resourceGroupName string) (string, error) + AcquireTokenBuckets(ctx context.Context, request *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) +} + +// resouceManagerClient gets the ResourceManager client of current PD leader. +func (c *client) resouceManagerClient() rmpb.ResourceManagerClient { + if cc, ok := c.clientConns.Load(c.GetLeaderAddr()); ok { + return rmpb.NewResourceManagerClient(cc.(*grpc.ClientConn)) + } + return nil +} + +// ListResourceGroups loads and returns target keyspace's metadata. +func (c *client) ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) { + req := &rmpb.ListResourceGroupsRequest{} + resp, err := c.resouceManagerClient().ListResourceGroups(ctx, req) + if err != nil { + return nil, err + } + resErr := resp.GetError() + if resErr != nil { + return nil, errors.Errorf("[resource_manager]" + resErr.Message) + } + return resp.GetGroups(), nil +} + +func (c *client) GetResourceGroup(ctx context.Context, resourceGroupName string) (*rmpb.ResourceGroup, error) { + req := &rmpb.GetResourceGroupRequest{ + ResourceGroupName: resourceGroupName, + } + resp, err := c.resouceManagerClient().GetResourceGroup(ctx, req) + if err != nil { + return nil, err + } + resErr := resp.GetError() + if resErr != nil { + return nil, errors.Errorf("[resource_manager]" + resErr.Message) + } + return resp.GetGroup(), nil +} + +func (c *client) AddResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) { + return c.putResourceGroup(ctx, metaGroup, add) +} + +func (c *client) ModifyResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) { + return c.putResourceGroup(ctx, metaGroup, modify) +} + +func (c *client) putResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup, typ actionType) (str string, err error) { + req := &rmpb.PutResourceGroupRequest{ + Group: metaGroup, + } + var resp *rmpb.PutResourceGroupResponse + switch typ { + case add: + resp, err = c.resouceManagerClient().AddResourceGroup(ctx, req) + case modify: + resp, err = c.resouceManagerClient().ModifyResourceGroup(ctx, req) + } + if err != nil { + return str, err + } + resErr := resp.GetError() + if resErr != nil { + return str, errors.Errorf("[resource_manager]" + resErr.Message) + } + str = resp.GetBody() + return +} + +func (c *client) DeleteResourceGroup(ctx context.Context, resourceGroupName string) (string, error) { + req := &rmpb.DeleteResourceGroupRequest{ + ResourceGroupName: resourceGroupName, + } + resp, err := c.resouceManagerClient().DeleteResourceGroup(ctx, req) + if err != nil { + return "", err + } + resErr := resp.GetError() + if resErr != nil { + return "", errors.Errorf("[resource_manager]" + resErr.Message) + } + return resp.GetBody(), nil +} + +func (c *client) AcquireTokenBuckets(ctx context.Context, request *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) { + req := &tokenRequest{ + done: make(chan error, 1), + requestCtx: ctx, + clientCtx: c.ctx, + Requeset: request, + } + c.tokenDispatcher.tokenBatchController.tokenRequestCh <- req + grantedTokens, err := req.Wait() + if err != nil { + return nil, err + } + return grantedTokens, err +} + +type tokenRequest struct { + clientCtx context.Context + requestCtx context.Context + done chan error + Requeset *rmpb.TokenBucketsRequest + TokenBuckets []*rmpb.TokenBucketResponse +} + +func (req *tokenRequest) Wait() (tokenBuckets []*rmpb.TokenBucketResponse, err error) { + select { + case err = <-req.done: + err = errors.WithStack(err) + if err != nil { + return nil, err + } + tokenBuckets = req.TokenBuckets + return + case <-req.requestCtx.Done(): + return nil, errors.WithStack(req.requestCtx.Err()) + case <-req.clientCtx.Done(): + return nil, errors.WithStack(req.clientCtx.Err()) + } +} + +type tokenBatchController struct { + tokenRequestCh chan *tokenRequest +} + +func newTokenBatchController(tokenRequestCh chan *tokenRequest) *tokenBatchController { + return &tokenBatchController{ + tokenRequestCh: tokenRequestCh, + } +} + +type tokenDispatcher struct { + dispatcherCancel context.CancelFunc + tokenBatchController *tokenBatchController +} + +type resourceManagerConnectionContext struct { + stream rmpb.ResourceManager_AcquireTokenBucketsClient + ctx context.Context + cancel context.CancelFunc +} + +func (c *client) createTokenispatcher() { + dispatcherCtx, dispatcherCancel := context.WithCancel(c.ctx) + dispatcher := &tokenDispatcher{ + dispatcherCancel: dispatcherCancel, + tokenBatchController: newTokenBatchController( + make(chan *tokenRequest, 1)), + } + go c.handleResouceTokenDispatcher(dispatcherCtx, dispatcher.tokenBatchController) + c.tokenDispatcher = dispatcher +} + +func (c *client) handleResouceTokenDispatcher(dispatcherCtx context.Context, tbc *tokenBatchController) { + var connection resourceManagerConnectionContext + if err := c.tryResourceManagerConnect(dispatcherCtx, &connection); err != nil { + log.Warn("get stream error", zap.Error(err)) + } + + for { + var firstRequest *tokenRequest + select { + case <-dispatcherCtx.Done(): + return + case firstRequest = <-tbc.tokenRequestCh: + } + stream, streamCtx, cancel := connection.stream, connection.ctx, connection.cancel + if stream == nil { + c.tryResourceManagerConnect(dispatcherCtx, &connection) + c.finishTokenRequest(firstRequest, nil, errors.Errorf("no stream")) + continue + } + select { + case <-streamCtx.Done(): + log.Info("[resource_manager] resource manager stream is canceled") + cancel() + stream = nil + continue + default: + } + err := c.processTokenRequests(stream, firstRequest) + if err != nil { + log.Info("processTokenRequests error", zap.Error(err)) + cancel() + connection.stream = nil + } + } +} + +func (c *client) processTokenRequests(stream rmpb.ResourceManager_AcquireTokenBucketsClient, t *tokenRequest) error { + req := t.Requeset + if err := stream.Send(req); err != nil { + err = errors.WithStack(err) + c.finishTokenRequest(t, nil, err) + return err + } + resp, err := stream.Recv() + if err != nil { + err = errors.WithStack(err) + c.finishTokenRequest(t, nil, err) + return err + } + if resp.GetError() != nil { + return errors.Errorf("[resource_manager]" + resp.GetError().Message) + } + tokenBuckets := resp.GetResponses() + c.finishTokenRequest(t, tokenBuckets, nil) + return nil +} + +func (c *client) finishTokenRequest(t *tokenRequest, tokenBuckets []*rmpb.TokenBucketResponse, err error) { + t.TokenBuckets = tokenBuckets + t.done <- err +} + +func (c *client) tryResourceManagerConnect(ctx context.Context, connection *resourceManagerConnectionContext) error { + var ( + err error + stream rmpb.ResourceManager_AcquireTokenBucketsClient + ) + for i := 0; i < maxRetryTimes; i++ { + cctx, cancel := context.WithCancel(ctx) + stream, err = c.resouceManagerClient().AcquireTokenBuckets(cctx) + if err == nil && stream != nil { + connection.cancel = cancel + connection.ctx = cctx + connection.stream = stream + return nil + } + cancel() + select { + case <-ctx.Done(): + return err + case <-time.After(retryInterval): + } + } + return err +} From f73f7823f23bb8d8cb4f0abdc66f9db90a31454f Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 9 Jan 2023 14:28:48 +0800 Subject: [PATCH 10/15] address comment Signed-off-by: Cabinfever_B --- pkg/mcs/resource_manager/server/token_bukets.go | 10 ++++------ tools/pd-tso-bench/go.sum | 6 ++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/pkg/mcs/resource_manager/server/token_bukets.go b/pkg/mcs/resource_manager/server/token_bukets.go index bcf4356e83c..879a9e2e97c 100644 --- a/pkg/mcs/resource_manager/server/token_bukets.go +++ b/pkg/mcs/resource_manager/server/token_bukets.go @@ -15,7 +15,6 @@ package server import ( - "fmt" "math" "time" @@ -25,9 +24,10 @@ import ( const defaultRefillRate = 10000 -const defaultInitialTokens = 10 * 10000 - -const defaultMaxTokens = 1e7 +const ( + defaultInitialTokens = 10 * 10000 + defaultMaxTokens = 1e7 +) var reserveRatio float64 = 0.05 @@ -100,7 +100,6 @@ func (t *GroupTokenBucket) request( var res rmpb.TokenBucket res.Settings = &rmpb.TokenLimitSettings{} // FillRate is used for the token server unavailable in abnormal situation. - res.Settings.FillRate = 0 if neededTokens <= 0 { return &res, 0 } @@ -123,7 +122,6 @@ func (t *GroupTokenBucket) request( var trickleTime = time.Duration(targetPeriodMs) * time.Millisecond availableRate := float64(t.Settings.FillRate) if debt := -t.Tokens; debt > 0 { - fmt.Println(debt) debt -= float64(t.Settings.FillRate) * trickleTime.Seconds() if debt > 0 { debtRate := debt / float64(targetPeriodMs/1000) diff --git a/tools/pd-tso-bench/go.sum b/tools/pd-tso-bench/go.sum index a7b5f5a2981..44901c1f486 100644 --- a/tools/pd-tso-bench/go.sum +++ b/tools/pd-tso-bench/go.sum @@ -104,10 +104,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad h1:lGKxsEwdE0pVXzHYD1SQ1vfa3t/bFVU/latrQz8b/w0= -github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 h1:FYgKV9znRQmzVrrJDZ0gUfMIvKLAMU1tu1UKJib8bEQ= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1 h1:jw4NjEiCleRJPPpHM7K6l8OKzOjnZAj62eKteCAY6ro= +github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From d5aa646b6a886fbca213c85ae27964798db8c581 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 9 Jan 2023 15:50:45 +0800 Subject: [PATCH 11/15] address comment Signed-off-by: Cabinfever_B --- .../resource_manager/server/token_bukets.go | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/pkg/mcs/resource_manager/server/token_bukets.go b/pkg/mcs/resource_manager/server/token_bukets.go index bcf4356e83c..280859d412d 100644 --- a/pkg/mcs/resource_manager/server/token_bukets.go +++ b/pkg/mcs/resource_manager/server/token_bukets.go @@ -15,7 +15,6 @@ package server import ( - "fmt" "math" "time" @@ -25,9 +24,10 @@ import ( const defaultRefillRate = 10000 -const defaultInitialTokens = 10 * 10000 - -const defaultMaxTokens = 1e7 +const ( + defaultInitialTokens = 10 * 10000 + defaultMaxTokens = 1e7 +) var reserveRatio float64 = 0.05 @@ -78,6 +78,10 @@ func (t *GroupTokenBucket) update(now time.Time) { if t.Tokens < defaultInitialTokens { t.Tokens = defaultInitialTokens } + // TODO: If we support init or modify MaxTokens in the future, we can move following code. + if t.Tokens > t.MaxTokens { + t.MaxTokens = t.Tokens + } t.LastUpdate = &now t.Initialized = true return @@ -100,7 +104,6 @@ func (t *GroupTokenBucket) request( var res rmpb.TokenBucket res.Settings = &rmpb.TokenLimitSettings{} // FillRate is used for the token server unavailable in abnormal situation. - res.Settings.FillRate = 0 if neededTokens <= 0 { return &res, 0 } @@ -122,8 +125,20 @@ func (t *GroupTokenBucket) request( var trickleTime = time.Duration(targetPeriodMs) * time.Millisecond availableRate := float64(t.Settings.FillRate) + // When there are debt, the allotment will match the fill rate. + // We will have a threshold, beyond which the token allocation will be a minimum. + // the current threshold is fill rate * target period * 2. + // | + // fill rate |· · · · · · · · · + // | · + // | · + // | · + // | · + // reserve rate | · · · · + // | + // rate 0 ----------------------------------------------- + // debt period token 2*period token if debt := -t.Tokens; debt > 0 { - fmt.Println(debt) debt -= float64(t.Settings.FillRate) * trickleTime.Seconds() if debt > 0 { debtRate := debt / float64(targetPeriodMs/1000) From 29f44e1814b479b42e51ca9cee54491b5032aed1 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 10 Jan 2023 13:22:13 +0800 Subject: [PATCH 12/15] address comment Signed-off-by: Cabinfever_B --- client/resourcemanager_client.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/client/resourcemanager_client.go b/client/resourcemanager_client.go index caaa5a8c987..dba309de9a6 100644 --- a/client/resourcemanager_client.go +++ b/client/resourcemanager_client.go @@ -42,8 +42,8 @@ type ResourceManagerClient interface { AcquireTokenBuckets(ctx context.Context, request *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) } -// resouceManagerClient gets the ResourceManager client of current PD leader. -func (c *client) resouceManagerClient() rmpb.ResourceManagerClient { +// resourceManagerClient gets the ResourceManager client of current PD leader. +func (c *client) resourceManagerClient() rmpb.ResourceManagerClient { if cc, ok := c.clientConns.Load(c.GetLeaderAddr()); ok { return rmpb.NewResourceManagerClient(cc.(*grpc.ClientConn)) } @@ -53,7 +53,7 @@ func (c *client) resouceManagerClient() rmpb.ResourceManagerClient { // ListResourceGroups loads and returns target keyspace's metadata. func (c *client) ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) { req := &rmpb.ListResourceGroupsRequest{} - resp, err := c.resouceManagerClient().ListResourceGroups(ctx, req) + resp, err := c.resourceManagerClient().ListResourceGroups(ctx, req) if err != nil { return nil, err } @@ -68,7 +68,7 @@ func (c *client) GetResourceGroup(ctx context.Context, resourceGroupName string) req := &rmpb.GetResourceGroupRequest{ ResourceGroupName: resourceGroupName, } - resp, err := c.resouceManagerClient().GetResourceGroup(ctx, req) + resp, err := c.resourceManagerClient().GetResourceGroup(ctx, req) if err != nil { return nil, err } @@ -94,9 +94,9 @@ func (c *client) putResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceG var resp *rmpb.PutResourceGroupResponse switch typ { case add: - resp, err = c.resouceManagerClient().AddResourceGroup(ctx, req) + resp, err = c.resourceManagerClient().AddResourceGroup(ctx, req) case modify: - resp, err = c.resouceManagerClient().ModifyResourceGroup(ctx, req) + resp, err = c.resourceManagerClient().ModifyResourceGroup(ctx, req) } if err != nil { return str, err @@ -113,7 +113,7 @@ func (c *client) DeleteResourceGroup(ctx context.Context, resourceGroupName stri req := &rmpb.DeleteResourceGroupRequest{ ResourceGroupName: resourceGroupName, } - resp, err := c.resouceManagerClient().DeleteResourceGroup(ctx, req) + resp, err := c.resourceManagerClient().DeleteResourceGroup(ctx, req) if err != nil { return "", err } @@ -191,11 +191,11 @@ func (c *client) createTokenispatcher() { tokenBatchController: newTokenBatchController( make(chan *tokenRequest, 1)), } - go c.handleResouceTokenDispatcher(dispatcherCtx, dispatcher.tokenBatchController) + go c.handleResourceTokenDispatcher(dispatcherCtx, dispatcher.tokenBatchController) c.tokenDispatcher = dispatcher } -func (c *client) handleResouceTokenDispatcher(dispatcherCtx context.Context, tbc *tokenBatchController) { +func (c *client) handleResourceTokenDispatcher(dispatcherCtx context.Context, tbc *tokenBatchController) { var connection resourceManagerConnectionContext if err := c.tryResourceManagerConnect(dispatcherCtx, &connection); err != nil { log.Warn("get stream error", zap.Error(err)) @@ -264,7 +264,7 @@ func (c *client) tryResourceManagerConnect(ctx context.Context, connection *reso ) for i := 0; i < maxRetryTimes; i++ { cctx, cancel := context.WithCancel(ctx) - stream, err = c.resouceManagerClient().AcquireTokenBuckets(cctx) + stream, err = c.resourceManagerClient().AcquireTokenBuckets(cctx) if err == nil && stream != nil { connection.cancel = cancel connection.ctx = cctx From eefe996d02aa9b3589ccb5d216071a4e5aa69d3a Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 10 Jan 2023 18:15:11 +0800 Subject: [PATCH 13/15] address comment Signed-off-by: Cabinfever_B --- client/resourcemanager_client.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/client/resourcemanager_client.go b/client/resourcemanager_client.go index dba309de9a6..3e8c47f16c9 100644 --- a/client/resourcemanager_client.go +++ b/client/resourcemanager_client.go @@ -50,7 +50,7 @@ func (c *client) resourceManagerClient() rmpb.ResourceManagerClient { return nil } -// ListResourceGroups loads and returns target keyspace's metadata. +// ListResourceGroups loads and returns all metadata of resource groups. func (c *client) ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) { req := &rmpb.ListResourceGroupsRequest{} resp, err := c.resourceManagerClient().ListResourceGroups(ctx, req) @@ -211,7 +211,7 @@ func (c *client) handleResourceTokenDispatcher(dispatcherCtx context.Context, tb stream, streamCtx, cancel := connection.stream, connection.ctx, connection.cancel if stream == nil { c.tryResourceManagerConnect(dispatcherCtx, &connection) - c.finishTokenRequest(firstRequest, nil, errors.Errorf("no stream")) + firstRequest.done <- errors.Errorf("no stream") continue } select { @@ -235,26 +235,22 @@ func (c *client) processTokenRequests(stream rmpb.ResourceManager_AcquireTokenBu req := t.Requeset if err := stream.Send(req); err != nil { err = errors.WithStack(err) - c.finishTokenRequest(t, nil, err) + t.done <- err return err } resp, err := stream.Recv() if err != nil { err = errors.WithStack(err) - c.finishTokenRequest(t, nil, err) + t.done <- err return err } if resp.GetError() != nil { return errors.Errorf("[resource_manager]" + resp.GetError().Message) } tokenBuckets := resp.GetResponses() - c.finishTokenRequest(t, tokenBuckets, nil) - return nil -} - -func (c *client) finishTokenRequest(t *tokenRequest, tokenBuckets []*rmpb.TokenBucketResponse, err error) { t.TokenBuckets = tokenBuckets - t.done <- err + t.done <- nil + return nil } func (c *client) tryResourceManagerConnect(ctx context.Context, connection *resourceManagerConnectionContext) error { From 6115bc6e9de6d94327006a5c3347dc21a78119bd Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 11 Jan 2023 11:45:05 +0800 Subject: [PATCH 14/15] merge master Signed-off-by: Cabinfever_B merge master Signed-off-by: Cabinfever_B --- client/resourcemanager_client.go | 2 +- .../resource_manager/server/resource_group.go | 35 ------------------- 2 files changed, 1 insertion(+), 36 deletions(-) diff --git a/client/resourcemanager_client.go b/client/resourcemanager_client.go index 3e8c47f16c9..6a8f99e909a 100644 --- a/client/resourcemanager_client.go +++ b/client/resourcemanager_client.go @@ -218,7 +218,7 @@ func (c *client) handleResourceTokenDispatcher(dispatcherCtx context.Context, tb case <-streamCtx.Done(): log.Info("[resource_manager] resource manager stream is canceled") cancel() - stream = nil + connection.stream = nil continue default: } diff --git a/pkg/mcs/resource_manager/server/resource_group.go b/pkg/mcs/resource_manager/server/resource_group.go index 2addff375e8..4a711f3596a 100644 --- a/pkg/mcs/resource_manager/server/resource_group.go +++ b/pkg/mcs/resource_manager/server/resource_group.go @@ -172,60 +172,25 @@ func FromProtoResourceGroup(group *rmpb.ResourceGroup) *ResourceGroup { return rg } -<<<<<<< HEAD -// UpdateRRU updates the RRU of the resource group. -func (rg *ResourceGroup) UpdateRRU(now time.Time) { - rg.Lock() - defer rg.Unlock() - if rg.RUSettings != nil { - rg.RUSettings.RRU.update(now) - } -} - -// UpdateWRU updates the WRU of the resource group. -func (rg *ResourceGroup) UpdateWRU(now time.Time) { - rg.Lock() - defer rg.Unlock() - if rg.RUSettings != nil { - rg.RUSettings.WRU.update(now) - } -} - -// RequestRRU requests the RRU of the resource group. -func (rg *ResourceGroup) RequestRRU(neededTokens float64, targetPeriodMs uint64) *rmpb.GrantedRUTokenBucket { -======= // RequestRRU requests the RRU of the resource group. func (rg *ResourceGroup) RequestRRU(now time.Time, neededTokens float64, targetPeriodMs uint64) *rmpb.GrantedRUTokenBucket { ->>>>>>> master rg.Lock() defer rg.Unlock() if rg.RUSettings == nil { return nil } -<<<<<<< HEAD - tb, trickleTimeMs := rg.RUSettings.RRU.request(neededTokens, targetPeriodMs) -======= tb, trickleTimeMs := rg.RUSettings.RRU.request(now, neededTokens, targetPeriodMs) ->>>>>>> master return &rmpb.GrantedRUTokenBucket{Type: rmpb.RequestUnitType_RRU, GrantedTokens: tb, TrickleTimeMs: trickleTimeMs} } // RequestWRU requests the WRU of the resource group. -<<<<<<< HEAD -func (rg *ResourceGroup) RequestWRU(neededTokens float64, targetPeriodMs uint64) *rmpb.GrantedRUTokenBucket { -======= func (rg *ResourceGroup) RequestWRU(now time.Time, neededTokens float64, targetPeriodMs uint64) *rmpb.GrantedRUTokenBucket { ->>>>>>> master rg.Lock() defer rg.Unlock() if rg.RUSettings == nil { return nil } -<<<<<<< HEAD - tb, trickleTimeMs := rg.RUSettings.WRU.request(neededTokens, targetPeriodMs) -======= tb, trickleTimeMs := rg.RUSettings.WRU.request(now, neededTokens, targetPeriodMs) ->>>>>>> master return &rmpb.GrantedRUTokenBucket{Type: rmpb.RequestUnitType_WRU, GrantedTokens: tb, TrickleTimeMs: trickleTimeMs} } From d776fca0ac3d6dc47247217b904d4bd2f7cabcd8 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 11 Jan 2023 16:45:57 +0800 Subject: [PATCH 15/15] add test Signed-off-by: Cabinfever_B --- client/client.go | 2 +- tests/client/go.sum | 1 - tests/msc/go.mod | 169 +++++ tests/msc/go.sum | 699 ++++++++++++++++++ .../resource_manager/resource_manager_test.go | 146 +++- 5 files changed, 988 insertions(+), 29 deletions(-) create mode 100644 tests/msc/go.mod create mode 100644 tests/msc/go.sum diff --git a/client/client.go b/client/client.go index 8bd242a4b7a..efd686302e6 100644 --- a/client/client.go +++ b/client/client.go @@ -137,7 +137,7 @@ type Client interface { // KeyspaceClient manages keyspace metadata. KeyspaceClient - // ResourceManagerClient + // ResourceManagerClient manages resource group metadata and token assignment. ResourceManagerClient // Close closes the client. Close() diff --git a/tests/client/go.sum b/tests/client/go.sum index 753d14903e1..bfc90237bba 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -327,7 +327,6 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1 h1:jw4NjEiCleRJPPpHM7K6l8OKzOjnZAj62eKteCAY6ro= github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= diff --git a/tests/msc/go.mod b/tests/msc/go.mod new file mode 100644 index 00000000000..725c1f74a0b --- /dev/null +++ b/tests/msc/go.mod @@ -0,0 +1,169 @@ +module github.com/tikv/pd/tests/msc + +go 1.19 + +require ( + github.com/gogo/protobuf v1.3.2 // indirect + github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect + github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1 + github.com/stretchr/testify v1.8.0 + github.com/tikv/pd v0.0.0-00010101000000-000000000000 + github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 + go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 // indirect + go.uber.org/goleak v1.1.12 + google.golang.org/grpc v1.43.0 // indirect +) + +require ( + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/KyleBanks/depth v1.2.1 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/ReneKroon/ttlcache/v2 v2.3.0 // indirect + github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/VividCortex/mysqlerr v1.0.0 // indirect + github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 // indirect + github.com/aws/aws-sdk-go v1.35.3 // indirect + github.com/benbjohnson/clock v1.1.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bitly/go-simplejson v0.5.0 // indirect + github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect + github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect + github.com/cenkalti/backoff/v4 v4.0.2 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/go-units v0.4.0 // indirect + github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect + github.com/elliotchance/pie/v2 v2.1.0 // indirect + github.com/fogleman/gg v1.3.0 // indirect + github.com/gin-contrib/cors v1.4.0 // indirect + github.com/gin-contrib/gzip v0.0.1 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gin-gonic/gin v1.8.1 // indirect + github.com/go-ole/go-ole v1.2.4 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/swag v0.19.15 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/go-playground/validator/v10 v10.10.0 // indirect + github.com/go-resty/resty/v2 v2.6.0 // indirect + github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/goccy/go-graphviz v0.0.9 // indirect + github.com/goccy/go-json v0.9.7 // indirect + github.com/golang-jwt/jwt v3.2.1+incompatible // indirect + github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/btree v1.1.2 // indirect + github.com/google/pprof v0.0.0-20211122183932-1daafda22083 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/gorilla/mux v1.7.4 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.12.1 // indirect + github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 // indirect + github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.2 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/joho/godotenv v1.4.0 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/joomcode/errorx v1.0.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-sqlite3 v1.14.9 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/minio/sio v0.3.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oleiade/reflections v1.0.1 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 // indirect + github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect + github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 // indirect + github.com/pingcap/errcode v0.3.0 // indirect + github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect + github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect + github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20221201151320-ea3ee6971f2e // indirect + github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.11.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/rs/cors v1.7.0 // indirect + github.com/sasha-s/go-deadlock v0.2.0 // indirect + github.com/shirou/gopsutil v3.21.3+incompatible // indirect + github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 // indirect + github.com/sirupsen/logrus v1.6.0 // indirect + github.com/soheilhy/cmux v0.1.4 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.4.0 // indirect + github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 // indirect + github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba // indirect + github.com/swaggo/swag v1.8.3 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect + github.com/thoas/go-funk v0.8.0 // indirect + github.com/tklauser/go-sysconf v0.3.4 // indirect + github.com/tklauser/numcpus v0.2.1 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 // indirect + github.com/ugorji/go/codec v1.2.7 // indirect + github.com/unrolled/render v1.0.1 // indirect + github.com/urfave/negroni v0.3.0 // indirect + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + go.etcd.io/bbolt v1.3.6 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/dig v1.9.0 // indirect + go.uber.org/fx v1.12.0 // indirect + go.uber.org/multierr v1.7.0 // indirect + go.uber.org/zap v1.20.0 // indirect + golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect + golang.org/x/exp v0.0.0-20220321173239-a90fa8a75705 // indirect + golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect + golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect + golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect + golang.org/x/text v0.3.7 // indirect + golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect + golang.org/x/tools v0.1.10 // indirect + google.golang.org/appengine v1.4.0 // indirect + google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c // indirect + google.golang.org/protobuf v1.28.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/mysql v1.0.6 // indirect + gorm.io/driver/sqlite v1.1.4 // indirect + gorm.io/gorm v1.21.9 // indirect + moul.io/zapgorm2 v1.1.0 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect +) + +replace ( + github.com/tikv/pd => ../../ + github.com/tikv/pd/client => ../../client +) + +// reset grpc and protobuf deps in order to import client and server at the same time +replace ( + github.com/golang/protobuf v1.5.2 => github.com/golang/protobuf v1.3.4 + google.golang.org/grpc v1.43.0 => google.golang.org/grpc v1.26.0 + google.golang.org/protobuf v1.26.0 => github.com/golang/protobuf v1.3.4 +) diff --git a/tests/msc/go.sum b/tests/msc/go.sum new file mode 100644 index 00000000000..fc86d605185 --- /dev/null +++ b/tests/msc/go.sum @@ -0,0 +1,699 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/ReneKroon/ttlcache/v2 v2.3.0 h1:qZnUjRKIrbKHH6vF5T7Y9Izn5ObfTZfyYpGhvz2BKPo= +github.com/ReneKroon/ttlcache/v2 v2.3.0/go.mod h1:zbo6Pv/28e21Z8CzzqgYRArQYGYtjONRxaAKGxzQvG4= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VividCortex/mysqlerr v1.0.0 h1:5pZ2TZA+YnzPgzBfiUWGqWmKDVNBdrkf9g+DNe1Tiq8= +github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= +github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o= +github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo= +github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4= +github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= +github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54= +github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch h1:KLE/YeX+9FNaGVW5MtImRVPhjDpfpgJhvkuYWBmOYbo= +github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch/go.mod h1:KjBLriHXe7L6fGceqWzTod8HUB/TP1WWDtfuSYtYXaI= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= +github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= +github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= +github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/elliotchance/pie/v2 v2.1.0 h1:KEVAAzxYxTyFs4hvebFZVzBdEo3YeMzl2HYDWn+P3F4= +github.com/elliotchance/pie/v2 v2.1.0/go.mod h1:18t0dgGFH006g4eVdDtWfgFZPQEgl10IoEO8YWEq3Og= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= +github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= +github.com/gin-contrib/gzip v0.0.1 h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc= +github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= +github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= +github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= +github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= +github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0= +github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= +github.com/go-resty/resty/v2 v2.6.0 h1:joIR5PNLM2EFqqESUjCMGXrWmXNHEU9CEiK813oKYS4= +github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goccy/go-graphviz v0.0.9 h1:s/FMMJ1Joj6La3S5ApO3Jk2cwM4LpXECC2muFx3IPQQ= +github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= +github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= +github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20211122183932-1daafda22083 h1:c8EUapQFi+kjzedr4c6WqbwMdmB95+oDBWZ5XFHFYxY= +github.com/google/pprof v0.0.0-20211122183932-1daafda22083/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.12.1 h1:zCy2xE9ablevUOrUZc3Dl72Dt+ya2FNAvC2yLYMHzi4= +github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= +github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0= +github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d h1:uGg2frlt3IcT7kbV6LEp5ONv4vmoO2FW4qSO+my/aoM= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= +github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= +github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/joomcode/errorx v1.0.1 h1:CalpDWz14ZHd68fIqluJasJosAewpz2TFaJALrUxjrk= +github.com/joomcode/errorx v1.0.1/go.mod h1:kgco15ekB6cs+4Xjzo7SPeXzx38PbJzBwbnu9qfVNHQ= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= +github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= +github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= +github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= +github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oleiade/reflections v1.0.1 h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0HHQM= +github.com/oleiade/reflections v1.0.1/go.mod h1:rdFxbxq4QXVZWj0F+e9jqjDkc7dbp97vkRixKo2JR60= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= +github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= +github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo= +github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= +github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= +github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= +github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= +github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0= +github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= +github.com/pingcap/errcode v0.3.0 h1:IF6LC/4+b1KNwrMlr2rBTUrojFPMexXBcDWZSpNwxjg= +github.com/pingcap/errcode v0.3.0/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= +github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= +github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= +github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1 h1:jw4NjEiCleRJPPpHM7K6l8OKzOjnZAj62eKteCAY6ro= +github.com/pingcap/kvproto v0.0.0-20230105060948-64890fa4f6c1/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= +github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= +github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= +github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d h1:k3/APKZjXOyJrFy8VyYwRlZhMelpD3qBLJNsw3bPl/g= +github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d/go.mod h1:7j18ezaWTao2LHOyMlsc2Dg1vW+mDY9dEbPzVyOlaeM= +github.com/pingcap/tidb-dashboard v0.0.0-20221201151320-ea3ee6971f2e h1:FUdoQ6zWktVjIWLokNeulEcqIzGn6TnoOjdS9bQcFUo= +github.com/pingcap/tidb-dashboard v0.0.0-20221201151320-ea3ee6971f2e/go.mod h1:NNF1CfnM5TqrLNfzfSal723h2fVQlieyVBBdQBzfPTg= +github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= +github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= +github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= +github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 h1:mj/nMDAwTBiaCqMEs4cYCqF7pO6Np7vhy1D1wcQGz+E= +github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 h1:PyYN9JH5jY9j6av01SpfRMb+1DWg/i3MbGOKPxJ2wjM= +github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E= +github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI= +github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba h1:lUPlXKqgbqT2SVg2Y+eT9mu5wbqMnG+i/+Q9nK7C0Rs= +github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba/go.mod h1:O1lAbCgAAX/KZ80LM/OXwtWFI/5TvZlwxSg8Cq08PV0= +github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y= +github.com/swaggo/swag v1.6.3/go.mod h1:wcc83tB4Mb2aNiL/HP4MFeQdpHUrca+Rp/DRNgWAUio= +github.com/swaggo/swag v1.8.3 h1:3pZSSCQ//gAH88lfmxM3Cd1+JCsxV8Md6f36b9hrZ5s= +github.com/swaggo/swag v1.8.3/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg= +github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= +github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= +github.com/thoas/go-funk v0.8.0 h1:JP9tKSvnpFVclYgDM0Is7FD9M4fhPvqA0s0BsXmzSRQ= +github.com/thoas/go-funk v0.8.0/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= +github.com/tidwall/gjson v1.9.3 h1:hqzS9wAHMO+KVBBkLxYdkEeeFHuqr95GfClRLKlgK0E= +github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= +github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M= +github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= +github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc= +github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= +github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= +github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= +github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/unrolled/render v1.0.1 h1:VDDnQQVfBMsOsp3VaCJszSO0nkBIVEYoPWeRThk9spY= +github.com/unrolled/render v1.0.1/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/negroni v0.3.0 h1:PaXOb61mWeZJxc1Ji2xJjpVg9QfPo0rrB+lHyBxGNSU= +github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 h1:fqmtdYQlwZ/vKWSz5amW+a4cnjg23ojz5iL7rjf08Wg= +go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793/go.mod h1:eBhtbxXP1qpW0F6+WxoJ64DM1Mrfx46PHtVxEdkLe0I= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/dig v1.9.0 h1:pJTDXKEhRqBI8W7rU7kwT5EgyRZuSMVSFcZolOvKK9U= +go.uber.org/dig v1.9.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= +go.uber.org/fx v1.12.0 h1:+1+3Cz9M0dFMPy9SW9XUIUHye8bnPUm7q7DroNGWYG4= +go.uber.org/fx v1.12.0/go.mod h1:egT3Kyg1JFYQkvKLZ3EsykxkNrZxgXS+gKoKo7abERY= +go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= +go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20220321173239-a90fa8a75705 h1:ba9YlqfDGTTQ5aZ2fwOoQ1hf32QySyQkR6ODGDzHlnE= +golang.org/x/exp v0.0.0-20220321173239-a90fa8a75705/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 h1:M73Iuj3xbbb9Uk1DYhzydthsj6oOd6l9bpuFcNoUvTs= +golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191114200427-caa0b0f7d508/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201125231158-b5590deeca9b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.0.6 h1:mA0XRPjIKi4bkE9nv+NKs6qj6QWOchqUSdWOcpd3x1E= +gorm.io/driver/mysql v1.0.6/go.mod h1:KdrTanmfLPPyAOeYGyG+UpDys7/7eeWT1zCq+oekYnU= +gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM= +gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw= +gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.21.9 h1:INieZtn4P2Pw6xPJ8MzT0G4WUOsHq3RhfuDF1M6GW0E= +gorm.io/gorm v1.21.9/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +moul.io/zapgorm2 v1.1.0 h1:qwAlMBYf+qJkJ7PAzJl4oCe6eS6QGiKAXUPeis0+RBE= +moul.io/zapgorm2 v1.1.0/go.mod h1:emRfKjNqSzVj5lcgasBdovIXY1jSOwFz2GQZn1Rddks= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/tests/msc/resource_manager/resource_manager_test.go b/tests/msc/resource_manager/resource_manager_test.go index 0e8760656b3..1dd62102cc3 100644 --- a/tests/msc/resource_manager/resource_manager_test.go +++ b/tests/msc/resource_manager/resource_manager_test.go @@ -21,14 +21,15 @@ import ( "net/http" "strings" "testing" + "time" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" - "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + pd "github.com/tikv/pd/client" "github.com/tikv/pd/pkg/mcs/resource_manager/server" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/tests" "go.uber.org/goleak" - "google.golang.org/grpc" // Register Service _ "github.com/tikv/pd/pkg/mcs/registry" @@ -39,27 +40,114 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, testutil.LeakOptions...) } -func TestBasicReourceGroupCURD(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() +type resourceManagerClientTestSuite struct { + suite.Suite + ctx context.Context + clean context.CancelFunc + cluster *tests.TestCluster + client pd.Client +} + +func TestResourceManagerClientTestSuite(t *testing.T) { + suite.Run(t, new(resourceManagerClientTestSuite)) +} + +func (suite *resourceManagerClientTestSuite) SetupSuite() { + var err error + re := suite.Require() + + suite.ctx, suite.clean = context.WithCancel(context.Background()) + + suite.cluster, err = tests.NewTestCluster(suite.ctx, 1) + re.NoError(err) - cluster, err := tests.NewTestCluster(ctx, 1) - defer cluster.Destroy() + err = suite.cluster.RunInitialServers() re.NoError(err) - err = cluster.RunInitialServers() + leaderName := suite.cluster.WaitLeader() + leader := suite.cluster.GetServer(leaderName) + suite.client, err = pd.NewClientWithContext(suite.ctx, []string{leader.GetAddr()}, pd.SecurityOption{}) re.NoError(err) - leaderName := cluster.WaitLeader() - leader := cluster.GetServer(leaderName) +} +func (suite *resourceManagerClientTestSuite) TearDownSuite() { + suite.client.Close() + suite.clean() + suite.cluster.Destroy() +} - // Test registered GRPC Service - cc, err := grpc.DialContext(ctx, strings.TrimPrefix(leader.GetAddr(), "http://"), grpc.WithInsecure()) +func (suite *resourceManagerClientTestSuite) TestAcquireTokenBucket() { + re := suite.Require() + cli := suite.client + + groups := []*rmpb.ResourceGroup{ + { + Name: "test1", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RRU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 10000, + }, + Tokens: 100000, + }, + }, + }, + { + Name: "test2", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RRU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 40000, + }, + Tokens: 100000, + }, + }, + }, + } + for _, group := range groups { + resp, err := cli.AddResourceGroup(suite.ctx, group) + re.NoError(err) + re.Contains(resp, "Success!") + } + reqs := &rmpb.TokenBucketsRequest{ + Requests: make([]*rmpb.TokenBucketRequest, 0), + TargetRequestPeriodMs: uint64(time.Second * 10 / time.Millisecond), + } + for _, group := range groups { + requests := make([]*rmpb.RequestUnitItem, 0) + requests = append(requests, &rmpb.RequestUnitItem{ + Type: rmpb.RequestUnitType_RRU, + Value: 10000, + }) + req := &rmpb.TokenBucketRequest{ + ResourceGroupName: group.Name, + Request: &rmpb.TokenBucketRequest_RuItems{ + RuItems: &rmpb.TokenBucketRequest_RequestRU{ + RequestRU: requests, + }, + }, + } + reqs.Requests = append(reqs.Requests, req) + } + aresp, err := cli.AcquireTokenBuckets(suite.ctx, reqs) re.NoError(err) - defer cc.Close() + for _, resp := range aresp { + re.Len(resp.GrantedRUTokens, 1) + re.Equal(resp.GrantedRUTokens[0].GrantedTokens.Tokens, float64(10000.)) + } + for _, g := range groups { + // Delete Resource Group + dresp, err := cli.DeleteResourceGroup(suite.ctx, g.Name) + re.NoError(err) + re.Contains(dresp, "Success!") + } +} - grpcclient := rmpb.NewResourceManagerClient(cc) +func (suite *resourceManagerClientTestSuite) TestBasicReourceGroupCURD() { + re := suite.Require() + cli := suite.client testCasesSet1 := []struct { name string @@ -148,45 +236,49 @@ func TestBasicReourceGroupCURD(t *testing.T) { Mode: tcase.mode, } // Create Resource Group - resp, err := grpcclient.AddResourceGroup(ctx, &rmpb.PutResourceGroupRequest{Group: group}) + resp, err := cli.AddResourceGroup(suite.ctx, group) checkErr(err, tcase.addSuccess) if tcase.addSuccess { finalNum++ - re.Contains(resp.Body, "Success!") + re.Contains(resp, "Success!") } // Modify Resource Group tcase.modifySettings(group) - mresp, err := grpcclient.ModifyResourceGroup(ctx, &rmpb.PutResourceGroupRequest{Group: group}) + mresp, err := cli.ModifyResourceGroup(suite.ctx, group) checkErr(err, tcase.modifySuccess) if tcase.modifySuccess { - re.Contains(mresp.Body, "Success!") + re.Contains(mresp, "Success!") } // Get Resource Group - gresp, err := grpcclient.GetResourceGroup(ctx, &rmpb.GetResourceGroupRequest{ResourceGroupName: tcase.name}) + gresp, err := cli.GetResourceGroup(suite.ctx, tcase.name) re.NoError(err) - re.Equal(tcase.name, gresp.Group.Name) + re.Equal(tcase.name, gresp.Name) if tcase.modifySuccess { - re.Equal(group, gresp.Group) + re.Equal(group, gresp) } // Last one, Check list and delete all resource groups if i == len(testCasesSet1)-1 { // List Resource Group - lresp, err := grpcclient.ListResourceGroups(ctx, &rmpb.ListResourceGroupsRequest{}) + lresp, err := cli.ListResourceGroups(suite.ctx) re.NoError(err) - re.Equal(finalNum, len(lresp.Groups)) + re.Equal(finalNum, len(lresp)) - for _, g := range lresp.Groups { + for _, g := range lresp { // Delete Resource Group - dresp, err := grpcclient.DeleteResourceGroup(ctx, &rmpb.DeleteResourceGroupRequest{ResourceGroupName: g.Name}) + dresp, err := cli.DeleteResourceGroup(suite.ctx, g.Name) re.NoError(err) - re.Contains(dresp.Body, "Success!") + re.Contains(dresp, "Success!") + _, err = cli.GetResourceGroup(suite.ctx, g.Name) + re.EqualError(err, "rpc error: code = Unknown desc = resource group not found") } } } + leaderName := suite.cluster.WaitLeader() + leader := suite.cluster.GetServer(leaderName) // Test Resource Group CURD via HTTP finalNum = 0 for i, tcase := range testCasesSet1 {