diff --git a/br/pkg/lightning/backend/local/local.go b/br/pkg/lightning/backend/local/local.go index 0d19c6887c4f8..b05091ae00bcd 100644 --- a/br/pkg/lightning/backend/local/local.go +++ b/br/pkg/lightning/backend/local/local.go @@ -1808,6 +1808,18 @@ func (local *local) isIngestRetryable( return retryNone, nil, common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage()) case errPb.RegionNotFound != nil: return retryNone, nil, common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage()) + case errPb.ReadIndexNotReady != nil: + // this error happens when this region is splitting, the error might be: + // read index not ready, reason can not read index due to split, region 64037 + // we have paused schedule, but it's temporary, + // if next request takes a long time, there's chance schedule is enabled again + // or on key range border, another engine sharing this region tries to split this + // region may cause this error too. + newRegion, err = getRegion() + if err != nil { + return retryNone, nil, errors.Trace(err) + } + return retryWrite, newRegion, common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage()) } return retryNone, nil, errors.Errorf("non-retryable error: %s", resp.GetError().GetMessage()) } diff --git a/br/pkg/lightning/backend/local/local_test.go b/br/pkg/lightning/backend/local/local_test.go index 0711bfb1fc463..5ee3c31ebe13c 100644 --- a/br/pkg/lightning/backend/local/local_test.go +++ b/br/pkg/lightning/backend/local/local_test.go @@ -509,6 +509,15 @@ func TestIsIngestRetryable(t *testing.T) { retryType, _, err = local.isIngestRetryable(ctx, resp, region, metas) require.Equal(t, retryNone, retryType) require.EqualError(t, err, "non-retryable error: unknown error") + + resp.Error = &errorpb.Error{ + ReadIndexNotReady: &errorpb.ReadIndexNotReady{ + Reason: "test", + }, + } + retryType, _, err = local.isIngestRetryable(ctx, resp, region, metas) + require.Equal(t, retryWrite, retryType) + require.Error(t, err) } type testIngester struct{} diff --git a/br/pkg/lightning/common/errors.go b/br/pkg/lightning/common/errors.go index ccb465ec4b86a..4f1c598a84535 100644 --- a/br/pkg/lightning/common/errors.go +++ b/br/pkg/lightning/common/errors.go @@ -70,13 +70,14 @@ var ( ErrCreatePDClient = errors.Normalize("create pd client error", errors.RFCCodeText("Lightning:PD:ErrCreatePDClient")) ErrPauseGC = errors.Normalize("pause gc error", errors.RFCCodeText("Lightning:PD:ErrPauseGC")) - ErrCheckKVVersion = errors.Normalize("check tikv version error", errors.RFCCodeText("Lightning:KV:ErrCheckKVVersion")) - ErrCreateKVClient = errors.Normalize("create kv client error", errors.RFCCodeText("Lightning:KV:ErrCreateKVClient")) - ErrCheckMultiIngest = errors.Normalize("check multi-ingest support error", errors.RFCCodeText("Lightning:KV:ErrCheckMultiIngest")) - ErrKVEpochNotMatch = errors.Normalize("epoch not match", errors.RFCCodeText("Lightning:KV:EpochNotMatch")) - ErrKVNotLeader = errors.Normalize("not leader", errors.RFCCodeText("Lightning:KV:NotLeader")) - ErrKVServerIsBusy = errors.Normalize("server is busy", errors.RFCCodeText("Lightning:KV:ServerIsBusy")) - ErrKVRegionNotFound = errors.Normalize("region not found", errors.RFCCodeText("Lightning:KV:RegionNotFound")) + ErrCheckKVVersion = errors.Normalize("check tikv version error", errors.RFCCodeText("Lightning:KV:ErrCheckKVVersion")) + ErrCreateKVClient = errors.Normalize("create kv client error", errors.RFCCodeText("Lightning:KV:ErrCreateKVClient")) + ErrCheckMultiIngest = errors.Normalize("check multi-ingest support error", errors.RFCCodeText("Lightning:KV:ErrCheckMultiIngest")) + ErrKVEpochNotMatch = errors.Normalize("epoch not match", errors.RFCCodeText("Lightning:KV:EpochNotMatch")) + ErrKVNotLeader = errors.Normalize("not leader", errors.RFCCodeText("Lightning:KV:NotLeader")) + ErrKVServerIsBusy = errors.Normalize("server is busy", errors.RFCCodeText("Lightning:KV:ServerIsBusy")) + ErrKVRegionNotFound = errors.Normalize("region not found", errors.RFCCodeText("Lightning:KV:RegionNotFound")) + ErrKVReadIndexNotReady = errors.Normalize("read index not ready", errors.RFCCodeText("Lightning:KV:ReadIndexNotReady")) ErrUnknownBackend = errors.Normalize("unknown backend %s", errors.RFCCodeText("Lightning:Restore:ErrUnknownBackend")) ErrCheckLocalFile = errors.Normalize("cannot find local file for table: %s engineDir: %s", errors.RFCCodeText("Lightning:Restore:ErrCheckLocalFile")) diff --git a/errors.toml b/errors.toml index d8112d297e6fe..c9e3aa6c9da9b 100755 --- a/errors.toml +++ b/errors.toml @@ -371,6 +371,11 @@ error = ''' not leader ''' +["Lightning:KV:ReadIndexNotReady"] +error = ''' +read index not ready +''' + ["Lightning:KV:RegionNotFound"] error = ''' region not found