From e8ebc3ab69abaea4cea2f3a6dbee4de79db85053 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Mon, 10 Oct 2022 21:41:49 +0800 Subject: [PATCH 1/3] This is an automated cherry-pick of #5551 close tikv/pd#5207 Signed-off-by: ti-chi-bot --- client/client.go | 70 +++++++++++++++++++++++++++++++++++-- client/errs/errno.go | 63 +++++++++++++++++++++++++++++++++ tests/client/client_test.go | 70 +++++++++++++++++++++++++++++++++++++ 3 files changed, 201 insertions(+), 2 deletions(-) create mode 100644 client/errs/errno.go diff --git a/client/client.go b/client/client.go index 46eab397412..d0b1bf60198 100644 --- a/client/client.go +++ b/client/client.go @@ -280,8 +280,13 @@ const ( updateMemberTimeout = time.Second // Use a shorter timeout to recover faster from network isolation. tsLoopDCCheckInterval = time.Minute defaultMaxTSOBatchSize = 10000 // should be higher if client is sending requests in burst +<<<<<<< HEAD retryInterval = 1 * time.Second maxRetryTimes = 5 +======= + retryInterval = 500 * time.Millisecond + maxRetryTimes = 6 +>>>>>>> d50e5fe43 (client: fix Stream timeout logic (#5551)) ) // LeaderHealthCheckInterval might be changed in the unit to shorten the testing time. @@ -653,11 +658,19 @@ func (c *client) handleDispatcher( dc string, tbc *tsoBatchController) { var ( +<<<<<<< HEAD retryTimeConsuming time.Duration err error streamAddr string stream pdpb.PD_TsoClient cancel context.CancelFunc +======= + err error + streamAddr string + stream pdpb.PD_TsoClient + streamCtx context.Context + cancel context.CancelFunc +>>>>>>> d50e5fe43 (client: fix Stream timeout logic (#5551)) // addr -> connectionContext connectionCtxs sync.Map opts []opentracing.StartSpanOption @@ -714,6 +727,11 @@ func (c *client) handleDispatcher( } // Loop through each batch of TSO requests and send them for processing. +<<<<<<< HEAD +======= + streamLoopTimer := time.NewTimer(c.option.timeout) +tsoBatchLoop: +>>>>>>> d50e5fe43 (client: fix Stream timeout logic (#5551)) for { select { case <-dispatcherCtx.Done(): @@ -755,6 +773,48 @@ func (c *client) handleDispatcher( if maxBatchWaitInterval >= 0 { tbc.adjustBestBatchSize() } +<<<<<<< HEAD +======= + streamLoopTimer.Reset(c.option.timeout) + // Choose a stream to send the TSO gRPC request. + streamChoosingLoop: + for { + connectionCtx := c.chooseStream(&connectionCtxs) + if connectionCtx != nil { + streamAddr, stream, streamCtx, cancel = connectionCtx.streamAddr, connectionCtx.stream, connectionCtx.ctx, connectionCtx.cancel + } + // Check stream and retry if necessary. + if stream == nil { + log.Info("[pd] tso stream is not ready", zap.String("dc", dc)) + if c.updateConnectionCtxs(dispatcherCtx, dc, &connectionCtxs) { + continue streamChoosingLoop + } + select { + case <-dispatcherCtx.Done(): + return + case <-streamLoopTimer.C: + err = errs.ErrClientCreateTSOStream.FastGenByArgs(errs.RetryTimeoutErr) + log.Error("[pd] create tso stream error", zap.String("dc-location", dc), errs.ZapError(err)) + c.ScheduleCheckLeader() + c.finishTSORequest(tbc.getCollectedRequests(), 0, 0, 0, errors.WithStack(err)) + continue tsoBatchLoop + case <-time.After(retryInterval): + continue streamChoosingLoop + } + } + select { + case <-streamCtx.Done(): + log.Info("[pd] tso stream is canceled", zap.String("dc", dc), zap.String("stream-addr", streamAddr)) + // Set `stream` to nil and remove this stream from the `connectionCtxs` due to being canceled. + connectionCtxs.Delete(streamAddr) + cancel() + stream = nil + continue + default: + break streamChoosingLoop + } + } +>>>>>>> d50e5fe43 (client: fix Stream timeout logic (#5551)) done := make(chan struct{}) dl := deadline{ timer: time.After(c.option.timeout), @@ -835,7 +895,7 @@ type connectionContext struct { cancel context.CancelFunc } -func (c *client) updateConnectionCtxs(updaterCtx context.Context, dc string, connectionCtxs *sync.Map) { +func (c *client) updateConnectionCtxs(updaterCtx context.Context, dc string, connectionCtxs *sync.Map) bool { // Normal connection creating, it will be affected by the `enableForwarding`. createTSOConnection := c.tryConnect if c.allowTSOFollowerProxy(dc) { @@ -843,7 +903,9 @@ func (c *client) updateConnectionCtxs(updaterCtx context.Context, dc string, con } if err := createTSOConnection(updaterCtx, dc, connectionCtxs); err != nil { log.Error("[pd] update connection contexts failed", zap.String("dc", dc), errs.ZapError(err)) + return false } + return true } // tryConnect will try to connect to the TSO allocator leader. If the connection becomes unreachable @@ -859,6 +921,8 @@ func (c *client) tryConnect( networkErrNum uint64 err error stream pdpb.PD_TsoClient + url string + cc *grpc.ClientConn ) updateAndClear := func(newAddr string, connectionCtx *connectionContext) { if cc, loaded := connectionCtxs.LoadOrStore(newAddr, connectionCtx); loaded { @@ -874,9 +938,11 @@ func (c *client) tryConnect( return true }) } - cc, url := c.getAllocatorClientConnByDCLocation(dc) // retry several times before falling back to the follower when the network problem happens + for i := 0; i < maxRetryTimes; i++ { + c.ScheduleCheckLeader() + cc, url = c.getAllocatorClientConnByDCLocation(dc) cctx, cancel := context.WithCancel(dispatcherCtx) stream, err = c.createTsoStream(cctx, cancel, pdpb.NewPDClient(cc)) failpoint.Inject("unreachableNetwork", func() { diff --git a/client/errs/errno.go b/client/errs/errno.go new file mode 100644 index 00000000000..14bcb933046 --- /dev/null +++ b/client/errs/errno.go @@ -0,0 +1,63 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package errs + +import "github.com/pingcap/errors" + +const ( + // NotLeaderErr indicates the the non-leader member received the requests which should be received by leader. + NotLeaderErr = "is not leader" + // MismatchLeaderErr indicates the the non-leader member received the requests which should be received by leader. + MismatchLeaderErr = "mismatch leader id" + RetryTimeoutErr = "retry timeout" +) + +// client errors +var ( + ErrClientCreateTSOStream = errors.Normalize("create TSO stream failed, %s", errors.RFCCodeText("PD:client:ErrClientCreateTSOStream")) + ErrClientGetTSOTimeout = errors.Normalize("get TSO timeout", errors.RFCCodeText("PD:client:ErrClientGetTSOTimeout")) + ErrClientGetTSO = errors.Normalize("get TSO failed, %v", errors.RFCCodeText("PD:client:ErrClientGetTSO")) + ErrClientGetLeader = errors.Normalize("get leader from %v error", errors.RFCCodeText("PD:client:ErrClientGetLeader")) + ErrClientGetMember = errors.Normalize("get member failed", errors.RFCCodeText("PD:client:ErrClientGetMember")) + ErrClientUpdateMember = errors.Normalize("update member failed, %v", errors.RFCCodeText("PD:client:ErrUpdateMember")) +) + +// grpcutil errors +var ( + ErrSecurityConfig = errors.Normalize("security config error: %s", errors.RFCCodeText("PD:grpcutil:ErrSecurityConfig")) +) + +// The third-party project error. +// url errors +var ( + ErrURLParse = errors.Normalize("parse url error", errors.RFCCodeText("PD:url:ErrURLParse")) +) + +// grpc errors +var ( + ErrGRPCDial = errors.Normalize("dial error", errors.RFCCodeText("PD:grpc:ErrGRPCDial")) + ErrCloseGRPCConn = errors.Normalize("close gRPC connection failed", errors.RFCCodeText("PD:grpc:ErrCloseGRPCConn")) +) + +// etcd errors +var ( + ErrEtcdTLSConfig = errors.Normalize("etcd TLS config error", errors.RFCCodeText("PD:etcd:ErrEtcdTLSConfig")) +) + +// crypto +var ( + ErrCryptoX509KeyPair = errors.Normalize("x509 keypair error", errors.RFCCodeText("PD:crypto:ErrCryptoX509KeyPair")) + ErrCryptoAppendCertsFromPEM = errors.Normalize("cert pool append certs error", errors.RFCCodeText("PD:crypto:ErrCryptoAppendCertsFromPEM")) +) diff --git a/tests/client/client_test.go b/tests/client/client_test.go index b3f5f6df15e..0d241d1a155 100644 --- a/tests/client/client_test.go +++ b/tests/client/client_test.go @@ -316,7 +316,77 @@ func (s *clientTestSuite) TestTSOFollowerProxy(c *C) { wg.Wait() } +<<<<<<< HEAD func (s *clientTestSuite) TestGlobalAndLocalTSO(c *C) { +======= +// TestUnavailableTimeAfterLeaderIsReady is used to test https://github.com/tikv/pd/issues/5207 +func TestUnavailableTimeAfterLeaderIsReady(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cluster, err := tests.NewTestCluster(ctx, 3) + re.NoError(err) + defer cluster.Destroy() + + endpoints := runServer(re, cluster) + cli := setupCli(re, ctx, endpoints) + + var wg sync.WaitGroup + var maxUnavailableTime, leaderReadyTime time.Time + getTsoFunc := func() { + defer wg.Done() + var lastTS uint64 + for i := 0; i < tsoRequestRound; i++ { + var physical, logical int64 + var ts uint64 + physical, logical, err = cli.GetTS(context.Background()) + ts = tsoutil.ComposeTS(physical, logical) + if err != nil { + maxUnavailableTime = time.Now() + continue + } + re.NoError(err) + re.Less(lastTS, ts) + lastTS = ts + } + } + + // test resign pd leader or stop pd leader + wg.Add(1 + 1) + go getTsoFunc() + go func() { + defer wg.Done() + leader := cluster.GetServer(cluster.GetLeader()) + leader.Stop() + cluster.WaitLeader() + leaderReadyTime = time.Now() + cluster.RunServers([]*tests.TestServer{leader}) + }() + wg.Wait() + re.Less(maxUnavailableTime.UnixMilli(), leaderReadyTime.Add(1*time.Second).UnixMilli()) + + // test kill pd leader pod or network of leader is unreachable + wg.Add(1 + 1) + maxUnavailableTime, leaderReadyTime = time.Time{}, time.Time{} + go getTsoFunc() + go func() { + defer wg.Done() + leader := cluster.GetServer(cluster.GetLeader()) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork", "return(true)")) + leader.Stop() + cluster.WaitLeader() + re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork")) + leaderReadyTime = time.Now() + }() + wg.Wait() + re.Less(maxUnavailableTime.UnixMilli(), leaderReadyTime.Add(1*time.Second).UnixMilli()) +} + +func TestGlobalAndLocalTSO(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() +>>>>>>> d50e5fe43 (client: fix Stream timeout logic (#5551)) dcLocationConfig := map[string]string{ "pd1": "dc-1", "pd2": "dc-2", From 2780d56b1560942beb3312f0af1e9007bf6c7f4e Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 1 Nov 2022 15:35:32 +0800 Subject: [PATCH 2/3] fix Signed-off-by: Cabinfever_B --- client/client.go | 78 ++++--------------------------------- tests/client/client_test.go | 40 +++++++++---------- 2 files changed, 26 insertions(+), 92 deletions(-) diff --git a/client/client.go b/client/client.go index d0b1bf60198..18d170e8bed 100644 --- a/client/client.go +++ b/client/client.go @@ -280,13 +280,8 @@ const ( updateMemberTimeout = time.Second // Use a shorter timeout to recover faster from network isolation. tsLoopDCCheckInterval = time.Minute defaultMaxTSOBatchSize = 10000 // should be higher if client is sending requests in burst -<<<<<<< HEAD - retryInterval = 1 * time.Second - maxRetryTimes = 5 -======= retryInterval = 500 * time.Millisecond maxRetryTimes = 6 ->>>>>>> d50e5fe43 (client: fix Stream timeout logic (#5551)) ) // LeaderHealthCheckInterval might be changed in the unit to shorten the testing time. @@ -658,19 +653,10 @@ func (c *client) handleDispatcher( dc string, tbc *tsoBatchController) { var ( -<<<<<<< HEAD - retryTimeConsuming time.Duration - err error - streamAddr string - stream pdpb.PD_TsoClient - cancel context.CancelFunc -======= err error streamAddr string stream pdpb.PD_TsoClient - streamCtx context.Context cancel context.CancelFunc ->>>>>>> d50e5fe43 (client: fix Stream timeout logic (#5551)) // addr -> connectionContext connectionCtxs sync.Map opts []opentracing.StartSpanOption @@ -727,11 +713,7 @@ func (c *client) handleDispatcher( } // Loop through each batch of TSO requests and send them for processing. -<<<<<<< HEAD -======= streamLoopTimer := time.NewTimer(c.option.timeout) -tsoBatchLoop: ->>>>>>> d50e5fe43 (client: fix Stream timeout logic (#5551)) for { select { case <-dispatcherCtx.Done(): @@ -746,24 +728,22 @@ tsoBatchLoop: // Check stream and retry if necessary. if stream == nil { log.Info("[pd] tso stream is not ready", zap.String("dc", dc)) - c.updateConnectionCtxs(dispatcherCtx, dc, &connectionCtxs) - if retryTimeConsuming >= c.option.timeout { - err = errs.ErrClientCreateTSOStream.FastGenByArgs() - log.Error("[pd] create tso stream error", zap.String("dc-location", dc), errs.ZapError(err)) - c.ScheduleCheckLeader() - c.revokeTSORequest(errors.WithStack(err), tbc.tsoRequestCh) - retryTimeConsuming = 0 + if c.updateConnectionCtxs(dispatcherCtx, dc, &connectionCtxs) { continue } select { case <-dispatcherCtx.Done(): return - case <-time.After(time.Second): - retryTimeConsuming += time.Second + case <-streamLoopTimer.C: + err = errs.ErrClientCreateTSOStream.FastGenByArgs() + log.Error("[pd] create tso stream error", zap.String("dc-location", dc), errs.ZapError(err)) + c.ScheduleCheckLeader() + c.revokeTSORequest(errors.WithStack(err), tbc.tsoRequestCh) + continue + case <-time.After(retryInterval): continue } } - retryTimeConsuming = 0 // Start to collect the TSO requests. maxBatchWaitInterval := c.option.getMaxTSOBatchWaitInterval() if err = tbc.fetchPendingRequests(dispatcherCtx, maxBatchWaitInterval); err != nil { @@ -773,48 +753,6 @@ tsoBatchLoop: if maxBatchWaitInterval >= 0 { tbc.adjustBestBatchSize() } -<<<<<<< HEAD -======= - streamLoopTimer.Reset(c.option.timeout) - // Choose a stream to send the TSO gRPC request. - streamChoosingLoop: - for { - connectionCtx := c.chooseStream(&connectionCtxs) - if connectionCtx != nil { - streamAddr, stream, streamCtx, cancel = connectionCtx.streamAddr, connectionCtx.stream, connectionCtx.ctx, connectionCtx.cancel - } - // Check stream and retry if necessary. - if stream == nil { - log.Info("[pd] tso stream is not ready", zap.String("dc", dc)) - if c.updateConnectionCtxs(dispatcherCtx, dc, &connectionCtxs) { - continue streamChoosingLoop - } - select { - case <-dispatcherCtx.Done(): - return - case <-streamLoopTimer.C: - err = errs.ErrClientCreateTSOStream.FastGenByArgs(errs.RetryTimeoutErr) - log.Error("[pd] create tso stream error", zap.String("dc-location", dc), errs.ZapError(err)) - c.ScheduleCheckLeader() - c.finishTSORequest(tbc.getCollectedRequests(), 0, 0, 0, errors.WithStack(err)) - continue tsoBatchLoop - case <-time.After(retryInterval): - continue streamChoosingLoop - } - } - select { - case <-streamCtx.Done(): - log.Info("[pd] tso stream is canceled", zap.String("dc", dc), zap.String("stream-addr", streamAddr)) - // Set `stream` to nil and remove this stream from the `connectionCtxs` due to being canceled. - connectionCtxs.Delete(streamAddr) - cancel() - stream = nil - continue - default: - break streamChoosingLoop - } - } ->>>>>>> d50e5fe43 (client: fix Stream timeout logic (#5551)) done := make(chan struct{}) dl := deadline{ timer: time.After(c.option.timeout), diff --git a/tests/client/client_test.go b/tests/client/client_test.go index 0d241d1a155..7f952a4bacb 100644 --- a/tests/client/client_test.go +++ b/tests/client/client_test.go @@ -316,20 +316,14 @@ func (s *clientTestSuite) TestTSOFollowerProxy(c *C) { wg.Wait() } -<<<<<<< HEAD -func (s *clientTestSuite) TestGlobalAndLocalTSO(c *C) { -======= // TestUnavailableTimeAfterLeaderIsReady is used to test https://github.com/tikv/pd/issues/5207 -func TestUnavailableTimeAfterLeaderIsReady(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - cluster, err := tests.NewTestCluster(ctx, 3) - re.NoError(err) +func (s *clientTestSuite) TestUnavailableTimeAfterLeaderIsReady(c *C) { + cluster, err := tests.NewTestCluster(s.ctx, 3) + c.Assert(err, IsNil) defer cluster.Destroy() - endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints) + endpoints := s.runServer(c, cluster) + cli := setupCli(c, s.ctx, endpoints) var wg sync.WaitGroup var maxUnavailableTime, leaderReadyTime time.Time @@ -345,8 +339,8 @@ func TestUnavailableTimeAfterLeaderIsReady(t *testing.T) { maxUnavailableTime = time.Now() continue } - re.NoError(err) - re.Less(lastTS, ts) + c.Assert(err, IsNil) + c.Assert(lastTS, Less, ts) lastTS = ts } } @@ -363,7 +357,10 @@ func TestUnavailableTimeAfterLeaderIsReady(t *testing.T) { cluster.RunServers([]*tests.TestServer{leader}) }() wg.Wait() - re.Less(maxUnavailableTime.UnixMilli(), leaderReadyTime.Add(1*time.Second).UnixMilli()) + c.Assert(maxUnavailableTime.Unix(), LessEqual, leaderReadyTime.Add(1*time.Second).Unix()) + if maxUnavailableTime.Unix() == leaderReadyTime.Add(1*time.Second).Unix() { + c.Assert(maxUnavailableTime.Nanosecond(), Less, leaderReadyTime.Add(1*time.Second).Nanosecond()) + } // test kill pd leader pod or network of leader is unreachable wg.Add(1 + 1) @@ -372,21 +369,20 @@ func TestUnavailableTimeAfterLeaderIsReady(t *testing.T) { go func() { defer wg.Done() leader := cluster.GetServer(cluster.GetLeader()) - re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork", "return(true)")) + c.Assert(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork", "return(true)"), IsNil) leader.Stop() cluster.WaitLeader() - re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork")) + c.Assert(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork"), IsNil) leaderReadyTime = time.Now() }() wg.Wait() - re.Less(maxUnavailableTime.UnixMilli(), leaderReadyTime.Add(1*time.Second).UnixMilli()) + c.Assert(maxUnavailableTime.Unix(), LessEqual, leaderReadyTime.Add(1*time.Second).Unix()) + if maxUnavailableTime.Unix() == leaderReadyTime.Add(1*time.Second).Unix() { + c.Assert(maxUnavailableTime.Nanosecond(), Less, leaderReadyTime.Add(1*time.Second).Nanosecond()) + } } -func TestGlobalAndLocalTSO(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() ->>>>>>> d50e5fe43 (client: fix Stream timeout logic (#5551)) +func (s *clientTestSuite) TestGlobalAndLocalTSO(c *C) { dcLocationConfig := map[string]string{ "pd1": "dc-1", "pd2": "dc-2", From 81850fefc95e78d4948fce4d137ff87fdd276104 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 1 Nov 2022 15:55:18 +0800 Subject: [PATCH 3/3] fix Signed-off-by: Cabinfever_B --- client/errs/errno.go | 63 -------------------------------------------- 1 file changed, 63 deletions(-) delete mode 100644 client/errs/errno.go diff --git a/client/errs/errno.go b/client/errs/errno.go deleted file mode 100644 index 14bcb933046..00000000000 --- a/client/errs/errno.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2022 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package errs - -import "github.com/pingcap/errors" - -const ( - // NotLeaderErr indicates the the non-leader member received the requests which should be received by leader. - NotLeaderErr = "is not leader" - // MismatchLeaderErr indicates the the non-leader member received the requests which should be received by leader. - MismatchLeaderErr = "mismatch leader id" - RetryTimeoutErr = "retry timeout" -) - -// client errors -var ( - ErrClientCreateTSOStream = errors.Normalize("create TSO stream failed, %s", errors.RFCCodeText("PD:client:ErrClientCreateTSOStream")) - ErrClientGetTSOTimeout = errors.Normalize("get TSO timeout", errors.RFCCodeText("PD:client:ErrClientGetTSOTimeout")) - ErrClientGetTSO = errors.Normalize("get TSO failed, %v", errors.RFCCodeText("PD:client:ErrClientGetTSO")) - ErrClientGetLeader = errors.Normalize("get leader from %v error", errors.RFCCodeText("PD:client:ErrClientGetLeader")) - ErrClientGetMember = errors.Normalize("get member failed", errors.RFCCodeText("PD:client:ErrClientGetMember")) - ErrClientUpdateMember = errors.Normalize("update member failed, %v", errors.RFCCodeText("PD:client:ErrUpdateMember")) -) - -// grpcutil errors -var ( - ErrSecurityConfig = errors.Normalize("security config error: %s", errors.RFCCodeText("PD:grpcutil:ErrSecurityConfig")) -) - -// The third-party project error. -// url errors -var ( - ErrURLParse = errors.Normalize("parse url error", errors.RFCCodeText("PD:url:ErrURLParse")) -) - -// grpc errors -var ( - ErrGRPCDial = errors.Normalize("dial error", errors.RFCCodeText("PD:grpc:ErrGRPCDial")) - ErrCloseGRPCConn = errors.Normalize("close gRPC connection failed", errors.RFCCodeText("PD:grpc:ErrCloseGRPCConn")) -) - -// etcd errors -var ( - ErrEtcdTLSConfig = errors.Normalize("etcd TLS config error", errors.RFCCodeText("PD:etcd:ErrEtcdTLSConfig")) -) - -// crypto -var ( - ErrCryptoX509KeyPair = errors.Normalize("x509 keypair error", errors.RFCCodeText("PD:crypto:ErrCryptoX509KeyPair")) - ErrCryptoAppendCertsFromPEM = errors.Normalize("cert pool append certs error", errors.RFCCodeText("PD:crypto:ErrCryptoAppendCertsFromPEM")) -)