From 81585443af7887515117e39a069399125f746ee5 Mon Sep 17 00:00:00 2001 From: D3Hunter Date: Thu, 2 Nov 2023 15:41:08 +0800 Subject: [PATCH 1/2] This is an automated cherry-pick of #48210 Signed-off-by: ti-chi-bot --- br/pkg/lightning/common/retry.go | 4 +++- br/pkg/lightning/common/retry_test.go | 4 ++++ br/pkg/pdutil/pd.go | 5 +++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/br/pkg/lightning/common/retry.go b/br/pkg/lightning/common/retry.go index 892ae6ab8300a..789c4f2d20a32 100644 --- a/br/pkg/lightning/common/retry.go +++ b/br/pkg/lightning/common/retry.go @@ -105,7 +105,9 @@ func isSingleRetryableError(err error) bool { if nerr.Timeout() { return true } - if syscallErr, ok := goerrors.Unwrap(err).(*os.SyscallError); ok { + // the error might be nested, such as *url.Error -> *net.OpError -> *os.SyscallError + var syscallErr *os.SyscallError + if goerrors.As(nerr, &syscallErr) { return syscallErr.Err == syscall.ECONNREFUSED || syscallErr.Err == syscall.ECONNRESET } return false diff --git a/br/pkg/lightning/common/retry_test.go b/br/pkg/lightning/common/retry_test.go index 939f4bb956942..114e500b3334c 100644 --- a/br/pkg/lightning/common/retry_test.go +++ b/br/pkg/lightning/common/retry_test.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "net" + "net/url" "testing" "github.com/go-sql-driver/mysql" @@ -66,6 +67,9 @@ func TestIsRetryableError(t *testing.T) { _, err := net.Dial("tcp", "localhost:65533") require.Error(t, err) require.True(t, IsRetryableError(err)) + // wrap net.OpErr inside url.Error + urlErr := &url.Error{Op: "post", Err: err} + require.True(t, IsRetryableError(urlErr)) // MySQL Errors require.False(t, IsRetryableError(&mysql.MySQLError{})) diff --git a/br/pkg/pdutil/pd.go b/br/pkg/pdutil/pd.go index 851009e7aeeb0..ff84d54bc5510 100644 --- a/br/pkg/pdutil/pd.go +++ b/br/pkg/pdutil/pd.go @@ -52,8 +52,12 @@ const ( pauseTimeout = 5 * time.Minute // pd request retry time when connection fail +<<<<<<< HEAD pdRequestRetryTime = 10 +======= + pdRequestRetryTime = 120 +>>>>>>> 9c92e065734 (pdutil/backend: enlarge max retry time and fix nested retriable error (#48210)) // set max-pending-peer-count to a large value to avoid scatter region failed. maxPendingPeerUnlimited uint64 = math.MaxInt32 ) @@ -169,6 +173,7 @@ func pdRequestWithCode( resp *http.Response ) count := 0 + // the total retry duration: 120*1 = 2min for { req, err = http.NewRequestWithContext(ctx, method, reqURL, body) if err != nil { From de09fe3152a32d71bd4d3997baf89a8af5770efe Mon Sep 17 00:00:00 2001 From: D3Hunter Date: Thu, 2 Nov 2023 16:04:26 +0800 Subject: [PATCH 2/2] fix conflict --- br/pkg/pdutil/pd.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/br/pkg/pdutil/pd.go b/br/pkg/pdutil/pd.go index ff84d54bc5510..c551e358c959c 100644 --- a/br/pkg/pdutil/pd.go +++ b/br/pkg/pdutil/pd.go @@ -52,12 +52,7 @@ const ( pauseTimeout = 5 * time.Minute // pd request retry time when connection fail -<<<<<<< HEAD - pdRequestRetryTime = 10 - -======= pdRequestRetryTime = 120 ->>>>>>> 9c92e065734 (pdutil/backend: enlarge max retry time and fix nested retriable error (#48210)) // set max-pending-peer-count to a large value to avoid scatter region failed. maxPendingPeerUnlimited uint64 = math.MaxInt32 )