From d7d994c6956fb5960219c13edbc31d5e80430390 Mon Sep 17 00:00:00 2001 From: you06 Date: Tue, 2 Jul 2024 16:21:11 +0900 Subject: [PATCH 01/10] add reproduce test Signed-off-by: you06 --- internal/locate/region_cache_test.go | 50 ++++++++++++++++++++++++++ internal/mockstore/mocktikv/cluster.go | 10 ++++++ 2 files changed, 60 insertions(+) diff --git a/internal/locate/region_cache_test.go b/internal/locate/region_cache_test.go index 689654058..95f457236 100644 --- a/internal/locate/region_cache_test.go +++ b/internal/locate/region_cache_test.go @@ -2679,3 +2679,53 @@ func (s *testRegionCacheSuite) TestBatchScanRegions() { s.cluster.Split(newID1, newID2, []byte("e1"), newPeers, newPeers[0]) }, []uint64{regions[1], regions[3], regions[4], newID1, regions[5], regions[6]}) } + +func (s *testRegionCacheSuite) TestScanRegionsWithCavity() { + // Split at "a", "c", "e" + // nil --- 'a' --- 'c' --- 'e' --- nil + // <- 0 -> <- 1 -> <- 2 -> <- 3 --> + regions := s.cluster.AllocIDs(3) + regions = append([]uint64{s.region1}, regions...) + + peers := [][]uint64{{s.peer1, s.peer2}} + for i := 0; i < 3; i++ { + peers = append(peers, s.cluster.AllocIDs(2)) + } + + for i := 0; i < 3; i++ { + s.cluster.Split(regions[i], regions[i+1], []byte{'a' + 2*byte(i)}, peers[i+1], peers[i+1][0]) + } + + // the last region is not reported to PD yet + getRegionIDsWithInject := func(fn func() ([]*Region, error)) []uint64 { + s.cache.clear() + err := failpoint.Enable("tikvclient/mockSplitRegionNotReportToPD", fmt.Sprintf(`return(%d)`, regions[2])) + s.Nil(err) + resCh := make(chan []*Region) + errCh := make(chan error) + go func() { + rs, err := fn() + errCh <- err + resCh <- rs + }() + time.Sleep(time.Second) + failpoint.Disable("tikvclient/mockSplitRegionNotReportToPD") + s.Nil(<-errCh) + rs := <-resCh + regionIDs := make([]uint64, 0, len(rs)) + for _, r := range rs { + regionIDs = append(regionIDs, r.GetID()) + } + return regionIDs + } + + scanRegionRes := getRegionIDsWithInject(func() ([]*Region, error) { + return s.cache.BatchLoadRegionsWithKeyRange(s.bo, []byte(""), []byte(""), 10) + }) + s.Equal(scanRegionRes, regions) + + batchScanRegionRes := getRegionIDsWithInject(func() ([]*Region, error) { + return s.cache.BatchLoadRegionsWithKeyRanges(s.bo, []pd.KeyRange{{StartKey: []byte{}, EndKey: []byte{}}}, 10) + }) + s.Equal(batchScanRegionRes, regions) +} diff --git a/internal/mockstore/mocktikv/cluster.go b/internal/mockstore/mocktikv/cluster.go index e1c9374db..748900abd 100644 --- a/internal/mockstore/mocktikv/cluster.go +++ b/internal/mockstore/mocktikv/cluster.go @@ -47,6 +47,7 @@ import ( "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/client-go/v2/internal/mockstore/cluster" + "github.com/tikv/client-go/v2/util" pd "github.com/tikv/pd/client" ) @@ -378,6 +379,15 @@ func (c *Cluster) ScanRegions(startKey, endKey []byte, limit int, opts ...pd.Get regions = regions[:endPos] } } + if rid, err := util.EvalFailpoint("mockSplitRegionNotReportToPD"); err == nil { + notReportRegionID := uint64(rid.(int)) + for i, r := range regions { + if r.Meta.Id == notReportRegionID { + regions = append(regions[:i], regions[i+1:]...) + break + } + } + } if limit > 0 && len(regions) > limit { regions = regions[:limit] } From ac636f869ec414d0d8170b08e5bf91f9c2c863f9 Mon Sep 17 00:00:00 2001 From: you06 Date: Tue, 2 Jul 2024 17:42:02 +0900 Subject: [PATCH 02/10] check if the pd returned regions covers the ranges Signed-off-by: you06 --- internal/locate/region_cache.go | 66 ++++++++++++++++++++- internal/locate/region_cache_test.go | 87 +++++++++++++++++++++++++++- 2 files changed, 150 insertions(+), 3 deletions(-) diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go index 80565b0df..5b2fb4610 100644 --- a/internal/locate/region_cache.go +++ b/internal/locate/region_cache.go @@ -2154,7 +2154,12 @@ func (c *RegionCache) scanRegions(bo *retry.Backoffer, startKey, endKey []byte, metrics.RegionCacheCounterWithScanRegionsOK.Inc() if len(regionsInfo) == 0 { - return nil, errors.Errorf("PD returned no region, limit: %d", limit) + backoffErr = errors.Errorf("PD returned no region, limit: %d", limit) + continue + } + if !rangesAreCovered([]pd.KeyRange{{StartKey: startKey, EndKey: endKey}}, regionsInfo) { + backoffErr = errors.Errorf("PD returned regions have gaps, limit: %d", limit) + continue } return c.handleRegionInfos(bo, regionsInfo, true) } @@ -2207,15 +2212,72 @@ func (c *RegionCache) batchScanRegions(bo *retry.Backoffer, keyRanges []pd.KeyRa metrics.RegionCacheCounterWithBatchScanRegionsOK.Inc() if len(regionsInfo) == 0 { - return nil, errors.Errorf( + backoffErr = errors.Errorf( "PD returned no region, range num: %d, limit: %d", len(keyRanges), limit, ) + continue + } + if !rangesAreCovered(keyRanges, regionsInfo) { + backoffErr = errors.Errorf( + "PD returned regions have gaps, range num: %d, limit: %d", + len(keyRanges), limit, + ) + continue } return c.handleRegionInfos(bo, regionsInfo, opt.needRegionHasLeaderPeer) } } +func rangesAreCovered(ranges []pd.KeyRange, regionsInfo []*pd.Region) bool { + if len(ranges) == 0 { + return true + } + if len(regionsInfo) == 0 { + return false + } + checkIdx := 0 // checked index of ranges + checkKey := ranges[0].StartKey // checked key of ranges + for _, r := range regionsInfo { + if r.Meta == nil { + return false + } + if len(r.Meta.StartKey) == 0 { + checkKey = r.Meta.EndKey + } else if bytes.Compare(r.Meta.StartKey, checkKey) > 0 { + // there is a gap between returned region's start_key and current check key + return false + } + if len(r.Meta.EndKey) == 0 { + // the current region contains all the rest ranges. + return true + } + checkKey = r.Meta.EndKey + for len(ranges[checkIdx].EndKey) > 0 && bytes.Compare(checkKey, ranges[checkIdx].EndKey) >= 0 { + // the end_key of returned region can cover multi ranges. + checkIdx++ + if checkIdx == len(ranges) { + // all ranges are covered. + return true + } + } + if bytes.Compare(checkKey, ranges[checkIdx].StartKey) < 0 { + // if check_key < start_key, move it forward to start_key. + checkKey = ranges[checkIdx].StartKey + } + } + if checkIdx < len(ranges)-1 { + // there are still some ranges not covered. + return false + } + if len(checkKey) == 0 { + return true + } else if len(ranges[checkIdx].EndKey) == 0 { + return false + } + return bytes.Compare(checkKey, ranges[checkIdx].EndKey) >= 0 +} + func (c *RegionCache) handleRegionInfos(bo *retry.Backoffer, regionsInfo []*pd.Region, needLeader bool) ([]*Region, error) { regions := make([]*Region, 0, len(regionsInfo)) for _, r := range regionsInfo { diff --git a/internal/locate/region_cache_test.go b/internal/locate/region_cache_test.go index 95f457236..21b381e60 100644 --- a/internal/locate/region_cache_test.go +++ b/internal/locate/region_cache_test.go @@ -2680,7 +2680,92 @@ func (s *testRegionCacheSuite) TestBatchScanRegions() { }, []uint64{regions[1], regions[3], regions[4], newID1, regions[5], regions[6]}) } -func (s *testRegionCacheSuite) TestScanRegionsWithCavity() { +func (s *testRegionCacheSuite) TestRangesAreCoveredCheck() { + check := func(ranges []string, regions []string, expect bool) { + rs := make([]pd.KeyRange, 0, len(ranges)/2) + for i := 0; i < len(ranges); i += 2 { + rs = append(rs, pd.KeyRange{StartKey: []byte(ranges[i]), EndKey: []byte(ranges[i+1])}) + } + rgs := make([]*pd.Region, 0, len(regions)) + for i := 0; i < len(regions); i += 2 { + rgs = append(rgs, &pd.Region{Meta: &metapb.Region{ + StartKey: []byte(regions[i]), + EndKey: []byte(regions[i+1]), + }}) + } + s.Equal(expect, rangesAreCovered(rs, rgs)) + } + boundCases := [][]string{ + {"a", "c"}, + {"a", "b", "b", "c"}, + {"a", "a1", "a1", "b", "b", "b1", "b1", "c"}, + } + for _, boundCase := range boundCases { + // positive + check(boundCase, []string{"a", "c"}, true) + check(boundCase, []string{"a", ""}, true) + check(boundCase, []string{"", "c"}, true) + // negative + check(boundCase, []string{"a", "b"}, false) + check(boundCase, []string{"b", "c"}, false) + check(boundCase, []string{"b", ""}, false) + check(boundCase, []string{"", "b"}, false) + // positive + check(boundCase, []string{"a", "b", "b", "c"}, true) + check(boundCase, []string{"", "b", "b", "c"}, true) + check(boundCase, []string{"a", "b", "b", ""}, true) + check(boundCase, []string{"", "b", "b", ""}, true) + // negative + check(boundCase, []string{"a", "b", "b1", "c"}, false) + check(boundCase, []string{"", "b", "b1", "c"}, false) + check(boundCase, []string{"a", "b", "b1", ""}, false) + check(boundCase, []string{"", "b", "b1", ""}, false) + check(boundCase, []string{}, false) + } + + nonContinuousCases := [][]string{ + {"a", "b", "c", "d"}, + {"a", "b1", "b1", "b", "c", "d"}, + {"a", "b", "c", "c1", "c1", "d"}, + {"a", "b1", "b1", "b", "c", "c1", "c1", "d"}, + } + for _, nonContinuousCase := range nonContinuousCases { + // positive + check(nonContinuousCase, []string{"a", "d"}, true) + check(nonContinuousCase, []string{"", "d"}, true) + check(nonContinuousCase, []string{"a", ""}, true) + check(nonContinuousCase, []string{"", ""}, true) + // negative + check(nonContinuousCase, []string{"a", "b"}, false) + check(nonContinuousCase, []string{"b", "c"}, false) + check(nonContinuousCase, []string{"c", "d"}, false) + check(nonContinuousCase, []string{"", "b"}, false) + check(nonContinuousCase, []string{"c", ""}, false) + } + + unboundCases := [][]string{ + {"", ""}, + {"", "b", "b", ""}, + {"", "a1", "a1", "b", "b", "b1", "b1", ""}, + } + for _, unboundCase := range unboundCases { + // positive + check(unboundCase, []string{"", ""}, true) + // negative + check(unboundCase, []string{"a", "c"}, false) + check(unboundCase, []string{"a", ""}, false) + check(unboundCase, []string{"", "c"}, false) + // positive + check(unboundCase, []string{"", "b", "b", ""}, true) + // negative + check(unboundCase, []string{"", "b", "b1", ""}, false) + check(unboundCase, []string{"a", "b", "b", ""}, false) + check(unboundCase, []string{"", "b", "b", "c"}, false) + check(unboundCase, []string{}, false) + } +} + +func (s *testRegionCacheSuite) TestScanRegionsWithGaps() { // Split at "a", "c", "e" // nil --- 'a' --- 'c' --- 'e' --- nil // <- 0 -> <- 1 -> <- 2 -> <- 3 --> From 8f9515c3a1a710145ec3ac2f293b223e0243cb95 Mon Sep 17 00:00:00 2001 From: you06 Date: Tue, 2 Jul 2024 18:17:18 +0900 Subject: [PATCH 03/10] handle limit Signed-off-by: you06 --- internal/locate/region_cache.go | 31 +++++++----- internal/locate/region_cache_test.go | 72 ++++++++++++++-------------- 2 files changed, 54 insertions(+), 49 deletions(-) diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go index 5b2fb4610..e5cd79971 100644 --- a/internal/locate/region_cache.go +++ b/internal/locate/region_cache.go @@ -2157,7 +2157,7 @@ func (c *RegionCache) scanRegions(bo *retry.Backoffer, startKey, endKey []byte, backoffErr = errors.Errorf("PD returned no region, limit: %d", limit) continue } - if !rangesAreCovered([]pd.KeyRange{{StartKey: startKey, EndKey: endKey}}, regionsInfo) { + if regionsHaveGap([]pd.KeyRange{{StartKey: startKey, EndKey: endKey}}, regionsInfo, limit) { backoffErr = errors.Errorf("PD returned regions have gaps, limit: %d", limit) continue } @@ -2218,7 +2218,7 @@ func (c *RegionCache) batchScanRegions(bo *retry.Backoffer, keyRanges []pd.KeyRa ) continue } - if !rangesAreCovered(keyRanges, regionsInfo) { + if regionsHaveGap(keyRanges, regionsInfo, limit) { backoffErr = errors.Errorf( "PD returned regions have gaps, range num: %d, limit: %d", len(keyRanges), limit, @@ -2229,28 +2229,28 @@ func (c *RegionCache) batchScanRegions(bo *retry.Backoffer, keyRanges []pd.KeyRa } } -func rangesAreCovered(ranges []pd.KeyRange, regionsInfo []*pd.Region) bool { +func regionsHaveGap(ranges []pd.KeyRange, regionsInfo []*pd.Region, limit int) bool { if len(ranges) == 0 { - return true + return false } if len(regionsInfo) == 0 { - return false + return true } checkIdx := 0 // checked index of ranges checkKey := ranges[0].StartKey // checked key of ranges for _, r := range regionsInfo { if r.Meta == nil { - return false + return true } if len(r.Meta.StartKey) == 0 { checkKey = r.Meta.EndKey } else if bytes.Compare(r.Meta.StartKey, checkKey) > 0 { // there is a gap between returned region's start_key and current check key - return false + return true } if len(r.Meta.EndKey) == 0 { // the current region contains all the rest ranges. - return true + return false } checkKey = r.Meta.EndKey for len(ranges[checkIdx].EndKey) > 0 && bytes.Compare(checkKey, ranges[checkIdx].EndKey) >= 0 { @@ -2258,7 +2258,7 @@ func rangesAreCovered(ranges []pd.KeyRange, regionsInfo []*pd.Region) bool { checkIdx++ if checkIdx == len(ranges) { // all ranges are covered. - return true + return false } } if bytes.Compare(checkKey, ranges[checkIdx].StartKey) < 0 { @@ -2266,16 +2266,21 @@ func rangesAreCovered(ranges []pd.KeyRange, regionsInfo []*pd.Region) bool { checkKey = ranges[checkIdx].StartKey } } + if limit > 0 && len(regionsInfo) == limit { + // the regionsInfo is limited by the limit, so there may be some ranges not covered. + // But the previous regions are continuous, so we just need to check the rest ranges. + return false + } if checkIdx < len(ranges)-1 { // there are still some ranges not covered. - return false + return true } if len(checkKey) == 0 { - return true - } else if len(ranges[checkIdx].EndKey) == 0 { return false + } else if len(ranges[checkIdx].EndKey) == 0 { + return true } - return bytes.Compare(checkKey, ranges[checkIdx].EndKey) >= 0 + return bytes.Compare(checkKey, ranges[checkIdx].EndKey) < 0 } func (c *RegionCache) handleRegionInfos(bo *retry.Backoffer, regionsInfo []*pd.Region, needLeader bool) ([]*Region, error) { diff --git a/internal/locate/region_cache_test.go b/internal/locate/region_cache_test.go index 21b381e60..4ae0ee6c6 100644 --- a/internal/locate/region_cache_test.go +++ b/internal/locate/region_cache_test.go @@ -2681,7 +2681,7 @@ func (s *testRegionCacheSuite) TestBatchScanRegions() { } func (s *testRegionCacheSuite) TestRangesAreCoveredCheck() { - check := func(ranges []string, regions []string, expect bool) { + check := func(ranges []string, regions []string, limit int, expect bool) { rs := make([]pd.KeyRange, 0, len(ranges)/2) for i := 0; i < len(ranges); i += 2 { rs = append(rs, pd.KeyRange{StartKey: []byte(ranges[i]), EndKey: []byte(ranges[i+1])}) @@ -2693,7 +2693,7 @@ func (s *testRegionCacheSuite) TestRangesAreCoveredCheck() { EndKey: []byte(regions[i+1]), }}) } - s.Equal(expect, rangesAreCovered(rs, rgs)) + s.Equal(expect, regionsHaveGap(rs, rgs, limit)) } boundCases := [][]string{ {"a", "c"}, @@ -2702,25 +2702,25 @@ func (s *testRegionCacheSuite) TestRangesAreCoveredCheck() { } for _, boundCase := range boundCases { // positive - check(boundCase, []string{"a", "c"}, true) - check(boundCase, []string{"a", ""}, true) - check(boundCase, []string{"", "c"}, true) + check(boundCase, []string{"a", "c"}, -1, false) + check(boundCase, []string{"a", ""}, -1, false) + check(boundCase, []string{"", "c"}, -1, false) // negative - check(boundCase, []string{"a", "b"}, false) - check(boundCase, []string{"b", "c"}, false) - check(boundCase, []string{"b", ""}, false) - check(boundCase, []string{"", "b"}, false) + check(boundCase, []string{"a", "b"}, -1, true) + check(boundCase, []string{"b", "c"}, -1, true) + check(boundCase, []string{"b", ""}, -1, true) + check(boundCase, []string{"", "b"}, -1, true) // positive - check(boundCase, []string{"a", "b", "b", "c"}, true) - check(boundCase, []string{"", "b", "b", "c"}, true) - check(boundCase, []string{"a", "b", "b", ""}, true) - check(boundCase, []string{"", "b", "b", ""}, true) + check(boundCase, []string{"a", "b", "b", "c"}, -1, false) + check(boundCase, []string{"", "b", "b", "c"}, -1, false) + check(boundCase, []string{"a", "b", "b", ""}, -1, false) + check(boundCase, []string{"", "b", "b", ""}, -1, false) // negative - check(boundCase, []string{"a", "b", "b1", "c"}, false) - check(boundCase, []string{"", "b", "b1", "c"}, false) - check(boundCase, []string{"a", "b", "b1", ""}, false) - check(boundCase, []string{"", "b", "b1", ""}, false) - check(boundCase, []string{}, false) + check(boundCase, []string{"a", "b", "b1", "c"}, -1, true) + check(boundCase, []string{"", "b", "b1", "c"}, -1, true) + check(boundCase, []string{"a", "b", "b1", ""}, -1, true) + check(boundCase, []string{"", "b", "b1", ""}, -1, true) + check(boundCase, []string{}, -1, true) } nonContinuousCases := [][]string{ @@ -2731,16 +2731,16 @@ func (s *testRegionCacheSuite) TestRangesAreCoveredCheck() { } for _, nonContinuousCase := range nonContinuousCases { // positive - check(nonContinuousCase, []string{"a", "d"}, true) - check(nonContinuousCase, []string{"", "d"}, true) - check(nonContinuousCase, []string{"a", ""}, true) - check(nonContinuousCase, []string{"", ""}, true) + check(nonContinuousCase, []string{"a", "d"}, -1, false) + check(nonContinuousCase, []string{"", "d"}, -1, false) + check(nonContinuousCase, []string{"a", ""}, -1, false) + check(nonContinuousCase, []string{"", ""}, -1, false) // negative - check(nonContinuousCase, []string{"a", "b"}, false) - check(nonContinuousCase, []string{"b", "c"}, false) - check(nonContinuousCase, []string{"c", "d"}, false) - check(nonContinuousCase, []string{"", "b"}, false) - check(nonContinuousCase, []string{"c", ""}, false) + check(nonContinuousCase, []string{"a", "b"}, -1, true) + check(nonContinuousCase, []string{"b", "c"}, -1, true) + check(nonContinuousCase, []string{"c", "d"}, -1, true) + check(nonContinuousCase, []string{"", "b"}, -1, true) + check(nonContinuousCase, []string{"c", ""}, -1, true) } unboundCases := [][]string{ @@ -2750,18 +2750,18 @@ func (s *testRegionCacheSuite) TestRangesAreCoveredCheck() { } for _, unboundCase := range unboundCases { // positive - check(unboundCase, []string{"", ""}, true) + check(unboundCase, []string{"", ""}, -1, false) // negative - check(unboundCase, []string{"a", "c"}, false) - check(unboundCase, []string{"a", ""}, false) - check(unboundCase, []string{"", "c"}, false) + check(unboundCase, []string{"a", "c"}, -1, true) + check(unboundCase, []string{"a", ""}, -1, true) + check(unboundCase, []string{"", "c"}, -1, true) // positive - check(unboundCase, []string{"", "b", "b", ""}, true) + check(unboundCase, []string{"", "b", "b", ""}, -1, false) // negative - check(unboundCase, []string{"", "b", "b1", ""}, false) - check(unboundCase, []string{"a", "b", "b", ""}, false) - check(unboundCase, []string{"", "b", "b", "c"}, false) - check(unboundCase, []string{}, false) + check(unboundCase, []string{"", "b", "b1", ""}, -1, true) + check(unboundCase, []string{"a", "b", "b", ""}, -1, true) + check(unboundCase, []string{"", "b", "b", "c"}, -1, true) + check(unboundCase, []string{}, -1, true) } } From 36f7d6f648adf9e361d3876c17de9b0e8e2457c5 Mon Sep 17 00:00:00 2001 From: you06 Date: Wed, 3 Jul 2024 14:43:39 +0900 Subject: [PATCH 04/10] fix lint Signed-off-by: you06 --- internal/locate/region_cache.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go index e5cd79971..f688d1d73 100644 --- a/internal/locate/region_cache.go +++ b/internal/locate/region_cache.go @@ -2242,9 +2242,7 @@ func regionsHaveGap(ranges []pd.KeyRange, regionsInfo []*pd.Region, limit int) b if r.Meta == nil { return true } - if len(r.Meta.StartKey) == 0 { - checkKey = r.Meta.EndKey - } else if bytes.Compare(r.Meta.StartKey, checkKey) > 0 { + if len(r.Meta.StartKey) != 0 && bytes.Compare(r.Meta.StartKey, checkKey) > 0 { // there is a gap between returned region's start_key and current check key return true } From 81f2a7c528165dc0e54a26779f42516a7ee14375 Mon Sep 17 00:00:00 2001 From: you06 Date: Wed, 3 Jul 2024 14:51:28 +0900 Subject: [PATCH 05/10] add TODO for func Signed-off-by: you06 --- internal/locate/region_cache.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go index f688d1d73..a29812512 100644 --- a/internal/locate/region_cache.go +++ b/internal/locate/region_cache.go @@ -2229,6 +2229,9 @@ func (c *RegionCache) batchScanRegions(bo *retry.Backoffer, keyRanges []pd.KeyRa } } +// regionsHaveGap checks if the loaded regions can fully cover the key ranges. +// If there are any gaps between the regions, it returns true, then the requests might be retried. +// TODO: remove this function after PD client supports gap detection and handling it. func regionsHaveGap(ranges []pd.KeyRange, regionsInfo []*pd.Region, limit int) bool { if len(ranges) == 0 { return false From 6f07898078727a0adc839198ad37dfb20d0ff5c9 Mon Sep 17 00:00:00 2001 From: you06 Date: Wed, 3 Jul 2024 14:59:20 +0900 Subject: [PATCH 06/10] update tidb for integration test Signed-off-by: you06 --- integration_tests/go.mod | 20 ++++++++------------ integration_tests/go.sum | 30 ++++++++++++++++-------------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/integration_tests/go.mod b/integration_tests/go.mod index 9f971d40c..8e2f7d930 100644 --- a/integration_tests/go.mod +++ b/integration_tests/go.mod @@ -7,11 +7,11 @@ require ( github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f github.com/pingcap/failpoint v0.0.0-20240527053858-9b3b6e34194a github.com/pingcap/kvproto v0.0.0-20240620063548-118a4cab53e4 - github.com/pingcap/tidb v1.1.0-beta.0.20240430081142-7481aa6d0b8b + github.com/pingcap/tidb v1.1.0-beta.0.20240703042657-230bbc2ef5ef github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.9.0 github.com/tidwall/gjson v1.14.1 - github.com/tikv/client-go/v2 v2.0.8-0.20240621090319-d4f0f4cf12a9 + github.com/tikv/client-go/v2 v2.0.8-0.20240626064248-4a72526f6c30 github.com/tikv/pd/client v0.0.0-20240620115049-049de1761e56 go.uber.org/goleak v1.3.0 ) @@ -57,6 +57,7 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncw/directio v1.0.5 // indirect github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 // indirect github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef // indirect @@ -67,13 +68,13 @@ require ( github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d // indirect github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 // indirect - github.com/pingcap/tidb/pkg/parser v0.0.0-20240430081142-7481aa6d0b8b // indirect + github.com/pingcap/tidb/pkg/parser v0.0.0-20240703042657-230bbc2ef5ef // indirect github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect - github.com/prometheus/client_golang v1.19.0 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.53.0 // indirect + github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/qri-io/jsonpointer v0.1.1 // indirect github.com/qri-io/jsonschema v0.2.1 // indirect @@ -105,12 +106,12 @@ require ( golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/tools v0.22.0 // indirect gonum.org/v1/gonum v0.8.2 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect google.golang.org/grpc v1.63.2 // indirect - google.golang.org/protobuf v1.34.1 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -121,8 +122,3 @@ replace ( github.com/tikv/client-go/v2 => ../ ) - -replace ( - github.com/pingcap/tidb => github.com/you06/tidb v1.1.0-beta.0.20240624051137-006cbc9cc886 - github.com/pingcap/tidb/pkg/parser => github.com/you06/tidb/pkg/parser v0.0.0-20240624051137-006cbc9cc886 -) diff --git a/integration_tests/go.sum b/integration_tests/go.sum index 3b6579071..d50e94ff3 100644 --- a/integration_tests/go.sum +++ b/integration_tests/go.sum @@ -311,6 +311,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncw/directio v1.0.4/go.mod h1:CKGdcN7StAaqjT7Qack3lAXeX4pjnyc46YeqZH1yWVY= github.com/ncw/directio v1.0.5 h1:JSUBhdjEvVaJvOoyPAbcW0fnd0tvRXD76wEfZ1KcQz4= github.com/ncw/directio v1.0.5/go.mod h1:rX/pKEYkOXBGOggmcyJeJGloCkleSvphPx2eV3t6ROk= @@ -363,6 +365,10 @@ github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d h1:y3EueKVfVykdpTyfU github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA= github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 h1:T4pXRhBflzDeAhmOQHNPRRogMYxP13V7BkYw3ZsoSfE= github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5/go.mod h1:rlimy0GcTvjiJqvD5mXTRr8O2eNZPBrcUgiWVYp9530= +github.com/pingcap/tidb v1.1.0-beta.0.20240703042657-230bbc2ef5ef h1:qf0QuRpodj5Yvm22BuuBoGFaiolL7KdkEYdnQ7DUstg= +github.com/pingcap/tidb v1.1.0-beta.0.20240703042657-230bbc2ef5ef/go.mod h1:vfEq5Kh+kAxB6ROcwTubBBk1lzZmIHMVzUwFYwQSJIA= +github.com/pingcap/tidb/pkg/parser v0.0.0-20240703042657-230bbc2ef5ef h1:HhIocGusIROnXhcXQ9DQL/+RgJpGXGlOkI/IX25pNrA= +github.com/pingcap/tidb/pkg/parser v0.0.0-20240703042657-230bbc2ef5ef/go.mod h1:c/4la2yfv1vBYvtIG8WCDyDinLMDIUC5+zLRHiafY+Y= github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 h1:fVNBE06Rjec+EIHaYAKAHa/bIt5lnu3Zh9O6kV7ZAdg= github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -378,15 +384,15 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:Om github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= -github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= -github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= @@ -494,10 +500,6 @@ github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 h1:mr+7gGPUasLmH3/5Iv1zwQwiY0WgGO21Ym7Q4FVw+xs= github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17/go.mod h1:u9udtIEWeBkphB2isZ8V8xVIMWgcUobH+7FRMO/Ld6c= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/you06/tidb v1.1.0-beta.0.20240624051137-006cbc9cc886 h1:jw3Z7XTxtzPgqUqhMQHOlmETHvk+bxR9uG+30ZtWlPQ= -github.com/you06/tidb v1.1.0-beta.0.20240624051137-006cbc9cc886/go.mod h1:QBEHAvrZRsmpbrYcBkphRVf8tP8OaYPl2uetSb5PP64= -github.com/you06/tidb/pkg/parser v0.0.0-20240624051137-006cbc9cc886 h1:Eqw/mDIP4bH+g/KcYZngtA1X9zQ+mZn4CMDJKBumZI8= -github.com/you06/tidb/pkg/parser v0.0.0-20240624051137-006cbc9cc886/go.mod h1:c/4la2yfv1vBYvtIG8WCDyDinLMDIUC5+zLRHiafY+Y= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -608,8 +610,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -708,8 +710,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -748,8 +750,8 @@ google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9 h1:ATnmU8nL2NfIyTSiBvJVDIDIr3qBmeW+c7z7XU21eWs= google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9/go.mod h1:j5uROIAAgi3YmtiETMt1LW0d/lHqQ7wwrIY4uGRXLQ4= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From a1ced8d18a3e389b734b8b98ab524208a2b7b19a Mon Sep 17 00:00:00 2001 From: you06 Date: Wed, 3 Jul 2024 16:34:16 +0900 Subject: [PATCH 07/10] rename gap detection function Signed-off-by: you06 --- internal/locate/region_cache.go | 8 ++++---- internal/locate/region_cache_test.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go index a29812512..113588a54 100644 --- a/internal/locate/region_cache.go +++ b/internal/locate/region_cache.go @@ -2157,7 +2157,7 @@ func (c *RegionCache) scanRegions(bo *retry.Backoffer, startKey, endKey []byte, backoffErr = errors.Errorf("PD returned no region, limit: %d", limit) continue } - if regionsHaveGap([]pd.KeyRange{{StartKey: startKey, EndKey: endKey}}, regionsInfo, limit) { + if regionsHaveGapInRanges([]pd.KeyRange{{StartKey: startKey, EndKey: endKey}}, regionsInfo, limit) { backoffErr = errors.Errorf("PD returned regions have gaps, limit: %d", limit) continue } @@ -2218,7 +2218,7 @@ func (c *RegionCache) batchScanRegions(bo *retry.Backoffer, keyRanges []pd.KeyRa ) continue } - if regionsHaveGap(keyRanges, regionsInfo, limit) { + if regionsHaveGapInRanges(keyRanges, regionsInfo, limit) { backoffErr = errors.Errorf( "PD returned regions have gaps, range num: %d, limit: %d", len(keyRanges), limit, @@ -2229,10 +2229,10 @@ func (c *RegionCache) batchScanRegions(bo *retry.Backoffer, keyRanges []pd.KeyRa } } -// regionsHaveGap checks if the loaded regions can fully cover the key ranges. +// regionsHaveGapInRanges checks if the loaded regions can fully cover the key ranges. // If there are any gaps between the regions, it returns true, then the requests might be retried. // TODO: remove this function after PD client supports gap detection and handling it. -func regionsHaveGap(ranges []pd.KeyRange, regionsInfo []*pd.Region, limit int) bool { +func regionsHaveGapInRanges(ranges []pd.KeyRange, regionsInfo []*pd.Region, limit int) bool { if len(ranges) == 0 { return false } diff --git a/internal/locate/region_cache_test.go b/internal/locate/region_cache_test.go index 4ae0ee6c6..284173377 100644 --- a/internal/locate/region_cache_test.go +++ b/internal/locate/region_cache_test.go @@ -2693,7 +2693,7 @@ func (s *testRegionCacheSuite) TestRangesAreCoveredCheck() { EndKey: []byte(regions[i+1]), }}) } - s.Equal(expect, regionsHaveGap(rs, rgs, limit)) + s.Equal(expect, regionsHaveGapInRanges(rs, rgs, limit)) } boundCases := [][]string{ {"a", "c"}, From 6a0d73edda705a618c56cf687926c4631381acf7 Mon Sep 17 00:00:00 2001 From: you06 Date: Wed, 3 Jul 2024 17:05:41 +0900 Subject: [PATCH 08/10] address comment Signed-off-by: you06 --- internal/locate/region_cache.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go index 01a44ae71..e33b86364 100644 --- a/internal/locate/region_cache.go +++ b/internal/locate/region_cache.go @@ -2248,7 +2248,7 @@ func regionsHaveGapInRanges(ranges []pd.KeyRange, regionsInfo []*pd.Region, limi if r.Meta == nil { return true } - if len(r.Meta.StartKey) != 0 && bytes.Compare(r.Meta.StartKey, checkKey) > 0 { + if bytes.Compare(r.Meta.StartKey, checkKey) > 0 { // there is a gap between returned region's start_key and current check key return true } @@ -2257,7 +2257,8 @@ func regionsHaveGapInRanges(ranges []pd.KeyRange, regionsInfo []*pd.Region, limi return false } checkKey = r.Meta.EndKey - for len(ranges[checkIdx].EndKey) > 0 && bytes.Compare(checkKey, ranges[checkIdx].EndKey) >= 0 { + for len(ranges[c + eckIdx].EndKey) > 0 && bytes.Compare(checkKey, ranges[checkIdx].EndKey) >= 0 { // the end_key of returned region can cover multi ranges. checkIdx++ if checkIdx == len(ranges) { From f6688ebb196c14627798b6d0342960b9f31c2f79 Mon Sep 17 00:00:00 2001 From: you06 Date: Wed, 3 Jul 2024 17:06:38 +0900 Subject: [PATCH 09/10] address comment Signed-off-by: you06 --- internal/locate/region_cache.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go index e33b86364..151de21cf 100644 --- a/internal/locate/region_cache.go +++ b/internal/locate/region_cache.go @@ -2257,8 +2257,7 @@ func regionsHaveGapInRanges(ranges []pd.KeyRange, regionsInfo []*pd.Region, limi return false } checkKey = r.Meta.EndKey - for len(ranges[c - eckIdx].EndKey) > 0 && bytes.Compare(checkKey, ranges[checkIdx].EndKey) >= 0 { + for len(ranges[checkIdx].EndKey) > 0 && bytes.Compare(checkKey, ranges[checkIdx].EndKey) >= 0 { // the end_key of returned region can cover multi ranges. checkIdx++ if checkIdx == len(ranges) { From 3339583ddfa385580f794dee4fcb67bb8fffd89a Mon Sep 17 00:00:00 2001 From: you06 Date: Wed, 3 Jul 2024 17:49:38 +0900 Subject: [PATCH 10/10] add half bounded cases Signed-off-by: you06 --- internal/locate/region_cache_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/locate/region_cache_test.go b/internal/locate/region_cache_test.go index 9cc05f547..888b96d15 100644 --- a/internal/locate/region_cache_test.go +++ b/internal/locate/region_cache_test.go @@ -2773,6 +2773,15 @@ func (s *testRegionCacheSuite) TestRangesAreCoveredCheck() { check(unboundCase, []string{"", "b", "b", "c"}, -1, true) check(unboundCase, []string{}, -1, true) } + + // test half bounded ranges + check([]string{"", "b"}, []string{"", "a"}, -1, true) + check([]string{"", "b"}, []string{"", "a"}, 1, false) // it's just limitation reached + check([]string{"", "b"}, []string{"", "a"}, 2, true) + check([]string{"a", ""}, []string{"b", ""}, -1, true) + check([]string{"a", ""}, []string{"b", ""}, 1, true) + check([]string{"a", ""}, []string{"b", "c"}, 1, true) + check([]string{"a", ""}, []string{"a", ""}, -1, false) } func (s *testRegionCacheSuite) TestScanRegionsWithGaps() {