From d8f5844d664ff2a30e80d04308538e7030e30b55 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Sun, 15 Aug 2021 22:53:28 +0800 Subject: [PATCH 01/29] executor: add retriver for tidb_hot_regions_history Signed-off-by: IcePigZDB --- executor/builder.go | 8 + executor/memtable_reader.go | 337 ++++++++++++++++++++++++++++ executor/memtable_reader_test.go | 368 +++++++++++++++++++++++++++++++ util/pdapi/const.go | 1 + 4 files changed, 714 insertions(+) diff --git a/executor/builder.go b/executor/builder.go index c888facf4e369..d9b169fff5e5f 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1523,6 +1523,14 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo extractor: v.Extractor.(*plannercore.ClusterLogTableExtractor), }, } + case strings.ToLower(infoschema.TableTiDBHotRegionsHistory): + return &MemTableReaderExec{ + baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()), + table: v.Table, + retriever: &hotRegionsHistoryRetriver{ + extractor: v.Extractor.(*plannercore.HotRegionsHistoryTableExtractor), + }, + } case strings.ToLower(infoschema.TableInspectionResult): return &MemTableReaderExec{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()), diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 699a61bd2ce07..137ec4b3d2015 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -15,6 +15,7 @@ package executor import ( + "bytes" "container/heap" "context" "encoding/json" @@ -39,18 +40,21 @@ import ( plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/store/helper" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/pdapi" "github.com/pingcap/tidb/util/set" + "github.com/tikv/client-go/v2/tikv" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) const clusterLogBatchSize = 256 +const hotRegionsHistoryBatchSize = 256 type dummyCloser struct{} @@ -699,3 +703,336 @@ func (e *clusterLogRetriever) close() error { func (e *clusterLogRetriever) getRuntimeStats() execdetails.RuntimeStats { return nil } + +type hotRegionsStreamResult struct { + addr string + messages *HistoryHotRegions + err error +} + +type hotRegionsResponseHeap []hotRegionsStreamResult + +func (h hotRegionsResponseHeap) Len() int { + return len(h) +} + +func (h hotRegionsResponseHeap) Less(i, j int) bool { + lhs, rhs := h[i].messages.HistoryHotRegion[0], h[j].messages.HistoryHotRegion[0] + if lhs.UpdateTime != rhs.UpdateTime { + return lhs.UpdateTime < rhs.UpdateTime + } + return lhs.HotDegree < rhs.HotDegree +} + +func (h hotRegionsResponseHeap) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +func (h *hotRegionsResponseHeap) Push(x interface{}) { + *h = append(*h, x.(hotRegionsStreamResult)) +} + +func (h *hotRegionsResponseHeap) Pop() interface{} { + old := *h + n := len(old) + x := old[n-1] + *h = old[0 : n-1] + return x +} + +type hotRegionsHistoryRetriver struct { + isDrained bool + retrieving bool + heap *hotRegionsResponseHeap + extractor *plannercore.HotRegionsHistoryTableExtractor + cancel context.CancelFunc +} + +func (e *hotRegionsHistoryRetriver) close() error { + if e.cancel != nil { + e.cancel() + } + return nil +} + +func (e *hotRegionsHistoryRetriver) getRuntimeStats() execdetails.RuntimeStats { + return nil +} + +// HistoryHotRegionsRequest wrap conditions push down to PD. +type HistoryHotRegionsRequest struct { + StartTime int64 `json:"start_time,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + RegionIDs []uint64 `json:"region_ids,omitempty"` + StoreIDs []uint64 `json:"store_ids,omitempty"` + PeerIDs []uint64 `json:"peer_ids,omitempty"` + HotRegionTypes []string `json:"hot_region_types,omitempty"` + LowHotDegree int64 `json:"low_hot_degree,omitempty"` + HighHotDegree int64 `json:"high_hot_degree,omitempty"` + LowFlowBytes float64 `json:"low_flow_bytes,omitempty"` + HighFlowBytes float64 `json:"high_flow_bytes,omitempty"` + LowKeyRate float64 `json:"low_key_rate,omitempty"` + HighKeyRate float64 `json:"high_key_rate,omitempty"` + LowQueryRate float64 `json:"low_query_rate,omitempty"` + HighQueryRate float64 `json:"high_query_rate,omitempty"` +} + +// HistoryHotRegions records filtered hot regions stored in each PD +// it's the response of PD. +type HistoryHotRegions struct { + HistoryHotRegion []*HistoryHotRegion `json:"history_hot_region"` +} + +// HistoryHotRegion records each hot region's statistics +// it's the response of PD. +type HistoryHotRegion struct { + UpdateTime int64 `json:"update_time,omitempty"` + RegionID uint64 `json:"region_id,omitempty"` + PeerID uint64 `json:"peer_id,omitempty"` + StoreID uint64 `json:"store_id,omitempty"` + HotRegionType string `json:"hot_region_type,omitempty"` + HotDegree int64 `json:"hot_degree,omitempty"` + FlowBytes float64 `json:"flow_bytes,omitempty"` + KeyRate float64 `json:"key_rate,omitempty"` + QueryRate float64 `json:"query_rate,omitempty"` + StartKey []byte `json:"start_key,omitempty"` + EndKey []byte `json:"end_key,omitempty"` +} + +func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx sessionctx.Context) ([]chan hotRegionsStreamResult, error) { + // TODO check whether need it + if !hasPriv(sctx, mysql.ProcessPriv) { + return nil, plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS") + } + pdServers, err := infoschema.GetPDServerInfo(sctx) + if err != nil { + return nil, err + } + + // To avoid search hot regions interface overload, the user should specify the time range in normally SQL. + if e.extractor.StartTime == 0 { + return nil, errors.New("denied to scan hot regions, please specified the start time, such as `update_time > '2020-01-01 00:00:00'`") + } + if e.extractor.EndTime == 0 { + return nil, errors.New("denied to scan hot regions, please specified the end time, such as `update_time < '2020-01-01 00:00:00'`") + } + + hotRegionTypes := make([]string, 0, e.extractor.HotRegionTypes.Count()) + for typ := range e.extractor.HotRegionTypes { + hotRegionTypes = append(hotRegionTypes, typ) + } + + historyHotRegionsRequest := &HistoryHotRegionsRequest{ + StartTime: e.extractor.StartTime / 1000, // second in PD + EndTime: e.extractor.EndTime / 1000, + RegionIDs: e.extractor.RegionIDs, + StoreIDs: e.extractor.StoreIDs, + PeerIDs: e.extractor.PeerIDs, + HotRegionTypes: hotRegionTypes, + LowHotDegree: e.extractor.LowHotDegree, + HighHotDegree: e.extractor.HighHotDegree, + LowFlowBytes: e.extractor.LowFlowBytes, + HighFlowBytes: e.extractor.HighFlowBytes, + LowKeyRate: e.extractor.LowKeyRate, + HighKeyRate: e.extractor.HighKeyRate, + LowQueryRate: e.extractor.LowQueryRate, + HighQueryRate: e.extractor.HighQueryRate, + } + jsonBody, err := json.Marshal(historyHotRegionsRequest) + if err != nil { + return nil, err + } + body := bytes.NewBuffer(jsonBody) + return e.startRetrieving(ctx, sctx, pdServers, body) +} + +func (e *hotRegionsHistoryRetriver) startRetrieving( + ctx context.Context, + sctx sessionctx.Context, + serversInfo []infoschema.ServerInfo, + body *bytes.Buffer, +) ([]chan hotRegionsStreamResult, error) { + wg := sync.WaitGroup{} + var results []chan hotRegionsStreamResult + for _, srv := range serversInfo { + ch := make(chan hotRegionsStreamResult) + results = append(results, ch) + go func(address string, body *bytes.Buffer) { + util.WithRecovery(func() { + defer wg.Done() + url := fmt.Sprintf("%s://%s%s", util.InternalHTTPSchema(), address, pdapi.HotHistory) + req, err := http.NewRequest(http.MethodPost, url, body) + if err != nil { + ch <- hotRegionsStreamResult{err: errors.Trace(err)} + return + } + req.Header.Add("PD-Allow-follower-handle", "true") + resp, err := util.InternalHTTPClient().Do(req) + if err != nil { + ch <- hotRegionsStreamResult{err: errors.Trace(err)} + return + } + defer func() { + terror.Log(resp.Body.Close()) + }() + if resp.StatusCode != http.StatusOK { + ch <- hotRegionsStreamResult{err: errors.Errorf("request %s failed: %s", url, resp.Status)} + return + } + // var nested map[string]interface{} + var historyHotRegions HistoryHotRegions + if err = json.NewDecoder(resp.Body).Decode(&historyHotRegions); err != nil { + ch <- hotRegionsStreamResult{err: errors.Trace(err)} + return + } + ch <- hotRegionsStreamResult{addr: address, messages: &historyHotRegions} + }, nil) + }(srv.StatusAddr, body) + } + return results, nil +} + +func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionctx.Context) ([][]types.Datum, error) { + if e.extractor.SkipRequest || e.isDrained { + return nil, nil + } + + if !e.retrieving { + e.retrieving = true + results, err := e.initialize(ctx, sctx) + if err != nil { + e.isDrained = true + return nil, err + } + // initialize the heap + e.heap = &hotRegionsResponseHeap{} + for _, ch := range results { + result := <-ch + if result.err != nil || len(result.messages.HistoryHotRegion) == 0 { + if result.err != nil { + sctx.GetSessionVars().StmtCtx.AppendWarning(result.err) + } + continue + } + *e.heap = append(*e.heap, result) + } + heap.Init(e.heap) + } + // filter results by db_name, table_name, index_name, table_id, index_id and mearge the results + var finalRows [][]types.Datum + for e.heap.Len() > 0 && len(finalRows) < hotRegionsHistoryBatchSize { + minTimeItem := heap.Pop(e.heap).(hotRegionsStreamResult) + row, err := e.parseAndFilterBySchemaInfo(sctx, minTimeItem.messages.HistoryHotRegion[0]) + if err != nil { + return nil, err + } + if row != nil { + finalRows = append(finalRows, row) + } + minTimeItem.messages.HistoryHotRegion = minTimeItem.messages.HistoryHotRegion[1:] + // Current streaming result is drained, read the next to supply. + if len(minTimeItem.messages.HistoryHotRegion) != 0 { + heap.Push(e.heap, minTimeItem) + } + } + // All streams are drained + e.isDrained = e.heap.Len() == 0 + return finalRows, nil +} + +func (e *hotRegionsHistoryRetriver) filterBySchemaInfo(f *helper.FrameItem) bool { + // TODO Ignore this row can't find responding schema f. + if f == nil { + return false + } + if len(e.extractor.DBNames) != 0 && !e.extractor.DBNames.Exist(f.DBName) { + return false + } + if len(e.extractor.TableNames) != 0 && !e.extractor.TableNames.Exist(f.TableName) { + return false + } + if len(e.extractor.IndexNames) != 0 && !e.extractor.IndexNames.Exist(f.IndexName) { + return false + } + if len(e.extractor.TableIDs) != 0 { + tableIDset := set.NewInt64Set() + for _, tbl := range e.extractor.TableIDs { + tableIDset.Insert(int64(tbl)) + } + if !tableIDset.Exist(f.TableID) { + return false + } + } + if len(e.extractor.IndexIDs) != 0 { + indexIDset := set.NewInt64Set() + for _, idx := range e.extractor.IndexIDs { + indexIDset.Insert(int64(idx)) + } + if !indexIDset.Exist(f.IndexID) { + return false + } + } + return true +} + +func (e *hotRegionsHistoryRetriver) parseAndFilterBySchemaInfo(sctx sessionctx.Context, headMessage *HistoryHotRegion) ([]types.Datum, error) { + region := &tikv.KeyLocation{StartKey: headMessage.StartKey, EndKey: headMessage.EndKey} + hotRange, err := helper.NewRegionFrameRange(region) + if err != nil { + return nil, err + } + allSchemas := sctx.GetInfoSchema().(infoschema.InfoSchema).AllSchemas() + tikvStore, ok := sctx.GetStore().(helper.Storage) + if !ok { + return nil, errors.New("Information about hot region can be gotten only when the storage is TiKV") + } + tikvHelper := &helper.Helper{ + Store: tikvStore, + RegionCache: tikvStore.GetRegionCache(), + } + f := tikvHelper.FindTableIndexOfRegion(allSchemas, hotRange) + // keep this row or not + keep := e.filterBySchemaInfo(f) + if !keep { + return nil, nil + } + updateTime := time.Unix(headMessage.UpdateTime, 0) + row := make([]types.Datum, len(infoschema.TableTiDBHotRegionsHistoryCols)) + + row[0].SetString(updateTime.Format("2006/01/02 15:04:05"), mysql.DefaultCollationName) + row[1].SetString(strings.ToUpper(f.DBName), mysql.DefaultCollationName) + row[2].SetString(strings.ToUpper(f.TableName), mysql.DefaultCollationName) + row[3].SetInt64(f.TableID) + if f.IndexName != "" { + row[4].SetString(strings.ToUpper(f.IndexName), mysql.DefaultCollationName) + row[5].SetInt64(f.IndexID) + } else { + row[4].SetNull() + row[5].SetNull() + } + row[6].SetInt64(int64(headMessage.RegionID)) + row[7].SetInt64(int64(headMessage.StoreID)) + row[8].SetInt64(int64(headMessage.PeerID)) + row[9].SetString(strings.ToUpper(headMessage.HotRegionType), mysql.DefaultCollationName) + if headMessage.HotDegree != 0 { + row[10].SetInt64(headMessage.HotDegree) + } else { + row[10].SetNull() + } + if headMessage.FlowBytes != 0 { + row[11].SetFloat64(float64(headMessage.FlowBytes)) + } else { + row[11].SetNull() + } + if headMessage.KeyRate != 0 { + row[12].SetFloat64(float64(headMessage.KeyRate)) + } else { + row[12].SetNull() + } + if headMessage.QueryRate != 0 { + row[13].SetFloat64(float64(headMessage.QueryRate)) + } else { + row[13].SetNull() + } + return row, nil +} diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index 4440318b18d78..f3cfdc52ac696 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -16,9 +16,11 @@ package executor_test import ( "context" + "crypto/tls" "fmt" "log" "net" + "net/http" "net/http/httptest" "os" "path/filepath" @@ -33,11 +35,14 @@ import ( "github.com/pingcap/kvproto/pkg/diagnosticspb" "github.com/pingcap/sysutil" "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/store/helper" "github.com/pingcap/tidb/util/pdapi" "github.com/pingcap/tidb/util/testkit" pmodel "github.com/prometheus/common/model" + "github.com/soheilhy/cmux" "google.golang.org/grpc" ) @@ -942,3 +947,366 @@ func (s *testMemTableReaderSuite) TestTiDBClusterLogError(c *C) { c.Assert(err.Error(), Equals, "denied to scan full logs (use `SELECT * FROM cluster_log WHERE message LIKE '%'` explicitly if intentionally)") c.Assert(rs.Close(), IsNil) } + +var _ = SerialSuites(&testHotRegionsHistoryTableSuite{testInfoschemaTableSuiteBase: &testInfoschemaTableSuiteBase{}}) + +type testHotRegionsHistoryTableSuite struct { + *testInfoschemaTableSuiteBase + startTime time.Time +} + +type mockStoreWithMultiPD struct { + helper.Storage + hosts []string +} + +func (s *mockStoreWithMultiPD) EtcdAddrs() ([]string, error) { return s.hosts, nil } +func (s *mockStoreWithMultiPD) TLSConfig() *tls.Config { panic("not implemented") } +func (s *mockStoreWithMultiPD) StartGCWorker() error { panic("not implemented") } +func (s *mockStoreWithMultiPD) Name() string { return "mockStore" } +func (s *mockStoreWithMultiPD) Describe() string { return "" } + +func (s *testHotRegionsHistoryTableSuite) SetUpSuite(c *C) { + s.testInfoschemaTableSuiteBase.SetUpSuite(c) + store := &mockStoreWithMultiPD{ + s.store.(helper.Storage), + make([]string, 3), + } + + // start 3 PD server with hotRegionsServer and store them in s.store + for i := 0; i < 3; i++ { + listener, err := net.Listen("tcp", "127.0.0.1:0") + c.Assert(err, IsNil) + m := cmux.New(listener) + // server grpc and http on the same port + httpListener := m.Match(cmux.HTTP1Fast()) + httpServer, mockAddr := s.setUpMockPDHTTPServer(c, httpListener) + store.hosts[i] = mockAddr + c.Assert(httpServer, NotNil) + + go func() { + err = m.Serve() + }() + } + s.store = store + s.startTime = time.Now() +} + +var hotRegionsResponses = make(map[string]*executor.HistoryHotRegions, 3) + +func (s *testHotRegionsHistoryTableSuite) setUpMockPDHTTPServer(c *C, l net.Listener) (*http.Server, string) { + // mock PD http server + router := mux.NewRouter() + httpServer := &http.Server{ + Handler: router, + } + // mock PD API + router.Handle(pdapi.ClusterVersion, fn.Wrap(func() (string, error) { return "4.0.0-alpha", nil })) + router.Handle(pdapi.Status, fn.Wrap(func() (interface{}, error) { + return struct { + GitHash string `json:"git_hash"` + StartTimestamp int64 `json:"start_timestamp"` + }{ + GitHash: "mock-pd-githash", + StartTimestamp: s.startTime.Unix(), + }, nil + })) + // mock response + router.Handle(pdapi.HotHistory, fn.Wrap(func() (interface{}, error) { + return hotRegionsResponses[l.Addr().String()], nil + })) + go func() { + err := httpServer.Serve(l) + c.Assert(err, IsNil) + }() + return httpServer, l.Addr().String() +} + +func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { + var unixTimeMs = func(s string) int64 { + t, err := time.ParseInLocation("2006/01/02 15:04:05", s, time.Local) + c.Assert(err, IsNil) + return t.Unix() + } + fullHotRegions := [][]string{ + // Record Regions + // mysql table_id = 11 table_name = TABLES_PRIV + {"2019/10/10 10:10:11", "MYSQL", "TABLES_PRIV", "11", "", "", "1", "1", "11111", "READ", "99", "99", "99", "99"}, // start []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb } + {"2019/10/10 10:10:12", "MYSQL", "TABLES_PRIV", "11", "", "", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, // end []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff } + // mysql table_id =21 table_name = STATS_META + {"2019/10/10 10:10:13", "MYSQL", "STATS_META", "21", "", "", "3", "3", "33333", "READ", "99", "99", "99", "99"}, // start []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15 } + {"2019/10/10 10:10:14", "MYSQL", "STATS_META", "21", "", "", "4", "4", "44444", "WRITE", "99", "99", "99", "99"}, // end []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff } + // deleted schemas + {"2019/10/10 10:10:15", "UNKONW", "UNKONW", "131", "UNKONW", "", "5", "5", "55555", "READ", "99", "99", "99", "99"}, // start []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83 } + {"2019/10/10 10:10:16", "UNKONW", "UNKONW", "131", "UNKONW", "", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, // end []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff } + + // Index Regions + // mysql table_id = 11 table_name = TABLES_PRIV index_id = 1 index_name = PRIMARY + {"2019/10/10 10:10:17", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "1", "1", "11111", "READ", "99", "99", "99", "99"}, // start []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1} + {"2019/10/10 10:10:18", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, // end []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1} + // mysql table_id = 21 table_name = STATS_META index_id = 1 index_name = IDX_VER + {"2019/10/10 10:10:19", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "3", "3", "33333", "READ", "99", "99", "99", "99"}, // start []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1 } + {"2019/10/10 10:10:20", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "4", "4", "44444", "WRITE", "99", "99", "99", "99"}, // end []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1 } + // mysql table_id = 21 table_name = STATS_META index_id = 2 index_name = TBL + {"2019/10/10 10:10:21", "MYSQL", "STATS_META", "21", "TBL", "2", "5", "5", "55555", "READ", "99", "99", "99", "99"}, // start []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2 } + {"2019/10/10 10:10:22", "MYSQL", "STATS_META", "21", "TBL", "2", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, // end []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2 } + // deleted schemas + {"2019/10/10 10:10:23", "UNKONW", "UNKONW", "131", "UNKONW", "1", "5", "5", "55555", "READ", "99", "99", "99", "99"}, // start []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1 } + {"2019/10/10 10:10:24", "UNKONW", "UNKONW", "131", "UNKONW", "1", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, // end []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1 } + } + + c.Assert(fullHotRegions, NotNil) + + pdResps := []*executor.HistoryHotRegions{ + {HistoryHotRegion: []*executor.HistoryHotRegion{ + // Record Regions + // mysql table_id = 11 table_name = TABLES_PRIV + {UpdateTime: unixTimeMs("2019/10/10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + {UpdateTime: unixTimeMs("2019/10/10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + // mysql table_id =21 table_name = STATS_META + {UpdateTime: unixTimeMs("2019/10/10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + {UpdateTime: unixTimeMs("2019/10/10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + }}, + {HistoryHotRegion: []*executor.HistoryHotRegion{ + // deleted schemas + {UpdateTime: unixTimeMs("2019/10/10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + {UpdateTime: unixTimeMs("2019/10/10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + // Index Regions + // mysql table_id = 11 table_name = tables_priv index_id = 1 index_name = PRIMARY + {UpdateTime: unixTimeMs("2019/10/10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, + {UpdateTime: unixTimeMs("2019/10/10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, + }}, + { + HistoryHotRegion: []*executor.HistoryHotRegion{ + // mysql table_id = 21 table_name = STATS_META index_id = 1 index_name = IDX_VER + {UpdateTime: unixTimeMs("2019/10/10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, + {UpdateTime: unixTimeMs("2019/10/10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, + // mysql table_id = 21 table_name = STATS_META index_id = 2 index_name = TBL + {UpdateTime: unixTimeMs("2019/10/10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}}, + {UpdateTime: unixTimeMs("2019/10/10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}}, + // deleted schemas + {UpdateTime: unixTimeMs("2019/10/10 10:10:23"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, + {UpdateTime: unixTimeMs("2019/10/10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, + }, + }, + } + // mock http resp + store := s.store.(*mockStoreWithMultiPD) + for i, resp := range pdResps { + hotRegionsResponses[store.hosts[i]] = resp + } + + var cases = []struct { + conditions []string + reqCount int32 + expected [][]string + }{ + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + }, // time filtered by PD, assume response suit time range, and ignore deleted schemas + expected: [][]string{ + fullHotRegions[0], fullHotRegions[1], fullHotRegions[2], + fullHotRegions[3], + fullHotRegions[6], fullHotRegions[7], fullHotRegions[8], + fullHotRegions[9], fullHotRegions[10], fullHotRegions[11], + }, + }, + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + "table_id=11", + }, + expected: [][]string{ + fullHotRegions[0], fullHotRegions[1], fullHotRegions[6], fullHotRegions[7], + }, + }, + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + "table_name='TABLES_PRIV'", + }, + expected: [][]string{ + fullHotRegions[0], fullHotRegions[1], fullHotRegions[6], fullHotRegions[7], + }, + }, + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + "table_id=21", + "index_id=1", + }, + expected: [][]string{ + fullHotRegions[8], fullHotRegions[9], + }, + }, + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + "table_id=21", + "index_id=1", + "table_name='TABLES_PRIV'", + }, // table_id != table_name -> nil + expected: [][]string{}, + }, + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + "table_id=21", + "index_id=1", + "table_name='STATS_META'", + }, // table_id = table_name + expected: [][]string{ + fullHotRegions[8], fullHotRegions[9], + }, + }, + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + "table_id=21", + "index_id=1", + "index_name='UNKONW'", + }, // index_id != index_name -> nil + expected: [][]string{}, + }, + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + "table_id=21", + "index_id=1", + "index_name='IDX_VER'", + }, // index_id = index_name + expected: [][]string{ + fullHotRegions[8], fullHotRegions[9], + }, + }, + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + "index_id=1", + "index_name='IDX_VER'", + "table_id>=21", // unpushed down predicates 21>=21 + }, + expected: [][]string{ + fullHotRegions[8], fullHotRegions[9], + }, + }, + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + "index_id=1", + "index_name='IDX_VER'", + "table_id>21", // unpushed down predicates + }, // 21!>21 -> nil + expected: [][]string{}, + }, + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + "index_id=1", + "index_name='IDX_VER'", + "table_id>=21", // unpushed down predicates + "db_name='MYSQL'", + }, + expected: [][]string{ + fullHotRegions[8], fullHotRegions[9], + }, + }, + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + "index_id=1", + "index_name='IDX_VER'", + "table_id>=21", // unpushed down predicates + "db_name='MYSQL'", + "peer_id>=33334", + }, + expected: [][]string{ + fullHotRegions[9], + }, + }, + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + "index_id=1", + "index_name='IDX_VER'", + "table_id>=21", // unpushed down predicates + "db_name='UNKNOW'", + }, + expected: [][]string{}, + }, + } + // failpoint + fpName := "github.com/pingcap/tidb/executor/mockTiDBHotRegionsHistory" + c.Assert(failpoint.Enable(fpName, `return("")`), IsNil) + defer func() { c.Assert(failpoint.Disable(fpName), IsNil) }() + + tk := testkit.NewTestKit(c, s.store) + + for _, cas := range cases { + sql := "select * from information_schema.tidb_hot_regions_history" + if len(cas.conditions) > 0 { + sql = fmt.Sprintf("%s where %s", sql, strings.Join(cas.conditions, " and ")) + } + result := tk.MustQuery(sql) + // rows := result.Rows() + warnings := tk.Se.GetSessionVars().StmtCtx.GetWarnings() + c.Assert(len(warnings), Equals, 0, Commentf("unexpected warnigns: %+v", warnings)) + var expected []string + for _, row := range cas.expected { + expectedRow := row + expected = append(expected, strings.Join(expectedRow, " ")) + } + result.Check(testkit.Rows(expected...)) + } +} + +func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistoryError(c *C) { + tk := testkit.NewTestKit(c, s.store) + fpName := "github.com/pingcap/tidb/executor/mockTiDBHotRegionsHistory" + c.Assert(failpoint.Enable(fpName, `return("")`), IsNil) + defer func() { c.Assert(failpoint.Disable(fpName), IsNil) }() + + // Test without start time error + rs, err := tk.Exec("select * from information_schema.tidb_hot_regions_history") + c.Assert(err, IsNil) + _, err = session.ResultSetToStringSlice(context.Background(), tk.Se, rs) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "denied to scan hot regions, please specified the start time, such as `update_time > '2020-01-01 00:00:00'`") + c.Assert(rs.Close(), IsNil) + + // Test without end time error. + rs, err = tk.Exec("select * from information_schema.tidb_hot_regions_history where update_time>='2019/08/26 06:18:13.011'") + c.Assert(err, IsNil) + _, err = session.ResultSetToStringSlice(context.Background(), tk.Se, rs) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "denied to scan hot regions, please specified the end time, such as `update_time < '2020-01-01 00:00:00'`") + c.Assert(rs.Close(), IsNil) +} diff --git a/util/pdapi/const.go b/util/pdapi/const.go index 29428b477c878..ebf4d9be36aae 100644 --- a/util/pdapi/const.go +++ b/util/pdapi/const.go @@ -18,6 +18,7 @@ package pdapi const ( HotRead = "/pd/api/v1/hotspot/regions/read" HotWrite = "/pd/api/v1/hotspot/regions/write" + HotHistory = "/pd/api/v1/hotspot/regions/history" Regions = "/pd/api/v1/regions" RegionByID = "/pd/api/v1/region/id/" Stores = "/pd/api/v1/stores" From f87a1122b54d5e84f81a0ee62b080a652216db9d Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Mon, 16 Aug 2021 08:26:28 +0000 Subject: [PATCH 02/29] executor: unified use of ms in update_time, and move IntSet init to extractor --- executor/memtable_reader.go | 72 +++++++++++++++---------------------- 1 file changed, 28 insertions(+), 44 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 137ec4b3d2015..3c15cd0273201 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -748,17 +748,6 @@ type hotRegionsHistoryRetriver struct { cancel context.CancelFunc } -func (e *hotRegionsHistoryRetriver) close() error { - if e.cancel != nil { - e.cancel() - } - return nil -} - -func (e *hotRegionsHistoryRetriver) getRuntimeStats() execdetails.RuntimeStats { - return nil -} - // HistoryHotRegionsRequest wrap conditions push down to PD. type HistoryHotRegionsRequest struct { StartTime int64 `json:"start_time,omitempty"` @@ -777,13 +766,13 @@ type HistoryHotRegionsRequest struct { HighQueryRate float64 `json:"high_query_rate,omitempty"` } -// HistoryHotRegions records filtered hot regions stored in each PD +// HistoryHotRegions records filtered hot regions stored in each PD. // it's the response of PD. type HistoryHotRegions struct { HistoryHotRegion []*HistoryHotRegion `json:"history_hot_region"` } -// HistoryHotRegion records each hot region's statistics +// HistoryHotRegion records each hot region's statistics. // it's the response of PD. type HistoryHotRegion struct { UpdateTime int64 `json:"update_time,omitempty"` @@ -800,7 +789,6 @@ type HistoryHotRegion struct { } func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx sessionctx.Context) ([]chan hotRegionsStreamResult, error) { - // TODO check whether need it if !hasPriv(sctx, mysql.ProcessPriv) { return nil, plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS") } @@ -823,8 +811,8 @@ func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx session } historyHotRegionsRequest := &HistoryHotRegionsRequest{ - StartTime: e.extractor.StartTime / 1000, // second in PD - EndTime: e.extractor.EndTime / 1000, + StartTime: e.extractor.StartTime, + EndTime: e.extractor.EndTime, RegionIDs: e.extractor.RegionIDs, StoreIDs: e.extractor.StoreIDs, PeerIDs: e.extractor.PeerIDs, @@ -879,7 +867,6 @@ func (e *hotRegionsHistoryRetriver) startRetrieving( ch <- hotRegionsStreamResult{err: errors.Errorf("request %s failed: %s", url, resp.Status)} return } - // var nested map[string]interface{} var historyHotRegions HistoryHotRegions if err = json.NewDecoder(resp.Body).Decode(&historyHotRegions); err != nil { ch <- hotRegionsStreamResult{err: errors.Trace(err)} @@ -904,7 +891,7 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct e.isDrained = true return nil, err } - // initialize the heap + // Initialize the heap e.heap = &hotRegionsResponseHeap{} for _, ch := range results { result := <-ch @@ -918,7 +905,7 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct } heap.Init(e.heap) } - // filter results by db_name, table_name, index_name, table_id, index_id and mearge the results + // Filter results by db_name, table_name, index_name, table_id, index_id and merge the results. var finalRows [][]types.Datum for e.heap.Len() > 0 && len(finalRows) < hotRegionsHistoryBatchSize { minTimeItem := heap.Pop(e.heap).(hotRegionsStreamResult) @@ -930,7 +917,7 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct finalRows = append(finalRows, row) } minTimeItem.messages.HistoryHotRegion = minTimeItem.messages.HistoryHotRegion[1:] - // Current streaming result is drained, read the next to supply. + // Fetch next message item if len(minTimeItem.messages.HistoryHotRegion) != 0 { heap.Push(e.heap, minTimeItem) } @@ -941,36 +928,24 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct } func (e *hotRegionsHistoryRetriver) filterBySchemaInfo(f *helper.FrameItem) bool { - // TODO Ignore this row can't find responding schema f. + // Ignore this row can't find responding schema f. if f == nil { return false } - if len(e.extractor.DBNames) != 0 && !e.extractor.DBNames.Exist(f.DBName) { + if e.extractor.DBNames.Count() != 0 && !e.extractor.DBNames.Exist(f.DBName) { return false } - if len(e.extractor.TableNames) != 0 && !e.extractor.TableNames.Exist(f.TableName) { + if e.extractor.TableNames.Count() != 0 && !e.extractor.TableNames.Exist(f.TableName) { return false } - if len(e.extractor.IndexNames) != 0 && !e.extractor.IndexNames.Exist(f.IndexName) { + if e.extractor.IndexNames.Count() != 0 && !e.extractor.IndexNames.Exist(f.IndexName) { return false } - if len(e.extractor.TableIDs) != 0 { - tableIDset := set.NewInt64Set() - for _, tbl := range e.extractor.TableIDs { - tableIDset.Insert(int64(tbl)) - } - if !tableIDset.Exist(f.TableID) { - return false - } + if e.extractor.TableIDs.Count() != 0 && !e.extractor.TableIDs.Exist(f.TableID) { + return false } - if len(e.extractor.IndexIDs) != 0 { - indexIDset := set.NewInt64Set() - for _, idx := range e.extractor.IndexIDs { - indexIDset.Insert(int64(idx)) - } - if !indexIDset.Exist(f.IndexID) { - return false - } + if e.extractor.IndexIDs.Count() != 0 && !e.extractor.IndexIDs.Exist(f.IndexID) { + return false } return true } @@ -991,14 +966,12 @@ func (e *hotRegionsHistoryRetriver) parseAndFilterBySchemaInfo(sctx sessionctx.C RegionCache: tikvStore.GetRegionCache(), } f := tikvHelper.FindTableIndexOfRegion(allSchemas, hotRange) - // keep this row or not - keep := e.filterBySchemaInfo(f) - if !keep { + // Keep this row or not + if !e.filterBySchemaInfo(f) { return nil, nil } updateTime := time.Unix(headMessage.UpdateTime, 0) row := make([]types.Datum, len(infoschema.TableTiDBHotRegionsHistoryCols)) - row[0].SetString(updateTime.Format("2006/01/02 15:04:05"), mysql.DefaultCollationName) row[1].SetString(strings.ToUpper(f.DBName), mysql.DefaultCollationName) row[2].SetString(strings.ToUpper(f.TableName), mysql.DefaultCollationName) @@ -1036,3 +1009,14 @@ func (e *hotRegionsHistoryRetriver) parseAndFilterBySchemaInfo(sctx sessionctx.C } return row, nil } + +func (e *hotRegionsHistoryRetriver) close() error { + if e.cancel != nil { + e.cancel() + } + return nil +} + +func (e *hotRegionsHistoryRetriver) getRuntimeStats() execdetails.RuntimeStats { + return nil +} From 644b4066c41e7056c5e7eb48d33fa89974396c62 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Mon, 16 Aug 2021 11:14:04 +0000 Subject: [PATCH 03/29] executor: fix time formate bug Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 3c15cd0273201..313a6636c0050 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -970,7 +970,7 @@ func (e *hotRegionsHistoryRetriver) parseAndFilterBySchemaInfo(sctx sessionctx.C if !e.filterBySchemaInfo(f) { return nil, nil } - updateTime := time.Unix(headMessage.UpdateTime, 0) + updateTime := time.Unix(headMessage.UpdateTime/1000, (headMessage.UpdateTime%1000)*int64(time.Millisecond)) row := make([]types.Datum, len(infoschema.TableTiDBHotRegionsHistoryCols)) row[0].SetString(updateTime.Format("2006/01/02 15:04:05"), mysql.DefaultCollationName) row[1].SetString(strings.ToUpper(f.DBName), mysql.DefaultCollationName) From 9067e136c8b074e785e9bbae6805320eb4f661df Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Mon, 16 Aug 2021 11:51:16 +0000 Subject: [PATCH 04/29] executor: fix history hot regions memtable reader test time bug Signed-off-by: IcePigZDB --- executor/memtable_reader_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index f3cfdc52ac696..7b1d72752ce7d 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -1026,7 +1026,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { var unixTimeMs = func(s string) int64 { t, err := time.ParseInLocation("2006/01/02 15:04:05", s, time.Local) c.Assert(err, IsNil) - return t.Unix() + return t.UnixNano() / int64(time.Millisecond) } fullHotRegions := [][]string{ // Record Regions From 47247382e3f9c1b857f1dc0e31f24ecd35c2f401 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Wed, 18 Aug 2021 07:39:41 +0000 Subject: [PATCH 05/29] executor: call DecodeBytes for region range key, and update test range key Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 5 +- executor/memtable_reader_test.go | 117 +++++++++++++++++-------------- 2 files changed, 68 insertions(+), 54 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 313a6636c0050..8b428f2544083 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -44,6 +44,7 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/pdapi" "github.com/pingcap/tidb/util/set" @@ -951,7 +952,9 @@ func (e *hotRegionsHistoryRetriver) filterBySchemaInfo(f *helper.FrameItem) bool } func (e *hotRegionsHistoryRetriver) parseAndFilterBySchemaInfo(sctx sessionctx.Context, headMessage *HistoryHotRegion) ([]types.Datum, error) { - region := &tikv.KeyLocation{StartKey: headMessage.StartKey, EndKey: headMessage.EndKey} + _, startKey, _ := codec.DecodeBytes(headMessage.StartKey, []byte{}) + _, endKey, _ := codec.DecodeBytes(headMessage.EndKey, []byte{}) + region := &tikv.KeyLocation{StartKey: startKey, EndKey: endKey} hotRange, err := helper.NewRegionFrameRange(region) if err != nil { return nil, err diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index 7b1d72752ce7d..020523cd8744e 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -1030,29 +1030,28 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { } fullHotRegions := [][]string{ // Record Regions - // mysql table_id = 11 table_name = TABLES_PRIV - {"2019/10/10 10:10:11", "MYSQL", "TABLES_PRIV", "11", "", "", "1", "1", "11111", "READ", "99", "99", "99", "99"}, // start []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb } - {"2019/10/10 10:10:12", "MYSQL", "TABLES_PRIV", "11", "", "", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, // end []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff } - // mysql table_id =21 table_name = STATS_META - {"2019/10/10 10:10:13", "MYSQL", "STATS_META", "21", "", "", "3", "3", "33333", "READ", "99", "99", "99", "99"}, // start []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15 } - {"2019/10/10 10:10:14", "MYSQL", "STATS_META", "21", "", "", "4", "4", "44444", "WRITE", "99", "99", "99", "99"}, // end []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff } - // deleted schemas - {"2019/10/10 10:10:15", "UNKONW", "UNKONW", "131", "UNKONW", "", "5", "5", "55555", "READ", "99", "99", "99", "99"}, // start []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83 } - {"2019/10/10 10:10:16", "UNKONW", "UNKONW", "131", "UNKONW", "", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, // end []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff } - + // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV + {"2019/10/10 10:10:11", "MYSQL", "TABLES_PRIV", "11", "", "", "1", "1", "11111", "READ", "99", "99", "99", "99"}, + {"2019/10/10 10:10:12", "MYSQL", "TABLES_PRIV", "11", "", "", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, + // mysql table_id = 21, record_id = 1, table_name = STATS_META + {"2019/10/10 10:10:13", "MYSQL", "STATS_META", "21", "", "", "3", "3", "33333", "READ", "99", "99", "99", "99"}, + {"2019/10/10 10:10:14", "MYSQL", "STATS_META", "21", "", "", "4", "4", "44444", "WRITE", "99", "99", "99", "99"}, + // table_id = 131, record_id=1, deleted schema + {"2019/10/10 10:10:15", "UNKONW", "UNKONW", "131", "UNKONW", "", "5", "5", "55555", "READ", "99", "99", "99", "99"}, + {"2019/10/10 10:10:16", "UNKONW", "UNKONW", "131", "UNKONW", "", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, // Index Regions - // mysql table_id = 11 table_name = TABLES_PRIV index_id = 1 index_name = PRIMARY - {"2019/10/10 10:10:17", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "1", "1", "11111", "READ", "99", "99", "99", "99"}, // start []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1} - {"2019/10/10 10:10:18", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, // end []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1} - // mysql table_id = 21 table_name = STATS_META index_id = 1 index_name = IDX_VER - {"2019/10/10 10:10:19", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "3", "3", "33333", "READ", "99", "99", "99", "99"}, // start []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1 } - {"2019/10/10 10:10:20", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "4", "4", "44444", "WRITE", "99", "99", "99", "99"}, // end []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1 } - // mysql table_id = 21 table_name = STATS_META index_id = 2 index_name = TBL - {"2019/10/10 10:10:21", "MYSQL", "STATS_META", "21", "TBL", "2", "5", "5", "55555", "READ", "99", "99", "99", "99"}, // start []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2 } - {"2019/10/10 10:10:22", "MYSQL", "STATS_META", "21", "TBL", "2", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, // end []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2 } - // deleted schemas - {"2019/10/10 10:10:23", "UNKONW", "UNKONW", "131", "UNKONW", "1", "5", "5", "55555", "READ", "99", "99", "99", "99"}, // start []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1 } - {"2019/10/10 10:10:24", "UNKONW", "UNKONW", "131", "UNKONW", "1", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, // end []byte{ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1 } + // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY + {"2019/10/10 10:10:17", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "1", "1", "11111", "READ", "99", "99", "99", "99"}, + {"2019/10/10 10:10:18", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, + // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER + {"2019/10/10 10:10:19", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "3", "3", "33333", "READ", "99", "99", "99", "99"}, + {"2019/10/10 10:10:20", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "4", "4", "44444", "WRITE", "99", "99", "99", "99"}, + // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL + {"2019/10/10 10:10:21", "MYSQL", "STATS_META", "21", "TBL", "2", "5", "5", "55555", "READ", "99", "99", "99", "99"}, + {"2019/10/10 10:10:22", "MYSQL", "STATS_META", "21", "TBL", "2", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, + // table_id = 131, index_id = 1, index_value = 1, deleted schema + {"2019/10/10 10:10:23", "UNKONW", "UNKONW", "131", "UNKONW", "1", "5", "5", "55555", "READ", "99", "99", "99", "99"}, + {"2019/10/10 10:10:24", "UNKONW", "UNKONW", "131", "UNKONW", "1", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, } c.Assert(fullHotRegions, NotNil) @@ -1060,49 +1059,61 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { pdResps := []*executor.HistoryHotRegions{ {HistoryHotRegion: []*executor.HistoryHotRegion{ // Record Regions - // mysql table_id = 11 table_name = TABLES_PRIV + // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV {UpdateTime: unixTimeMs("2019/10/10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, {UpdateTime: unixTimeMs("2019/10/10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, - // mysql table_id =21 table_name = STATS_META + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + // mysql table_id = 21, record_id = 1, table_name = STATS_META {UpdateTime: unixTimeMs("2019/10/10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, {UpdateTime: unixTimeMs("2019/10/10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }}, {HistoryHotRegion: []*executor.HistoryHotRegion{ - // deleted schemas + // table_id = 131, record_id=1, deleted schema {UpdateTime: unixTimeMs("2019/10/10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, {UpdateTime: unixTimeMs("2019/10/10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x72, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // Index Regions - // mysql table_id = 11 table_name = tables_priv index_id = 1 index_name = PRIMARY + // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY {UpdateTime: unixTimeMs("2019/10/10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, {UpdateTime: unixTimeMs("2019/10/10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + }}, + {HistoryHotRegion: []*executor.HistoryHotRegion{ + // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER + {UpdateTime: unixTimeMs("2019/10/10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + {UpdateTime: unixTimeMs("2019/10/10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL + {UpdateTime: unixTimeMs("2019/10/10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + {UpdateTime: unixTimeMs("2019/10/10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + // table_id = 131, index_id = 1, index_value = 1, deleted schema + {UpdateTime: unixTimeMs("2019/10/10 10:10:23"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + {UpdateTime: unixTimeMs("2019/10/10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }}, - { - HistoryHotRegion: []*executor.HistoryHotRegion{ - // mysql table_id = 21 table_name = STATS_META index_id = 1 index_name = IDX_VER - {UpdateTime: unixTimeMs("2019/10/10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, - {UpdateTime: unixTimeMs("2019/10/10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, - // mysql table_id = 21 table_name = STATS_META index_id = 2 index_name = TBL - {UpdateTime: unixTimeMs("2019/10/10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}}, - {UpdateTime: unixTimeMs("2019/10/10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}}, - // deleted schemas - {UpdateTime: unixTimeMs("2019/10/10 10:10:23"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, - {UpdateTime: unixTimeMs("2019/10/10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, - }, - }, } // mock http resp store := s.store.(*mockStoreWithMultiPD) From 78a9484fcfaa07313d11c9425dd88b279040f4a1 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Thu, 19 Aug 2021 13:04:54 +0000 Subject: [PATCH 06/29] executor: change UPDATE_TIME type to TypeTimestamp and add timezone check Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 9 +++- executor/memtable_reader_test.go | 70 +++++++++++++++++++------------- 2 files changed, 48 insertions(+), 31 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 8b428f2544083..345a8600f34c7 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -973,9 +973,14 @@ func (e *hotRegionsHistoryRetriver) parseAndFilterBySchemaInfo(sctx sessionctx.C if !e.filterBySchemaInfo(f) { return nil, nil } - updateTime := time.Unix(headMessage.UpdateTime/1000, (headMessage.UpdateTime%1000)*int64(time.Millisecond)) row := make([]types.Datum, len(infoschema.TableTiDBHotRegionsHistoryCols)) - row[0].SetString(updateTime.Format("2006/01/02 15:04:05"), mysql.DefaultCollationName) + updateTimestamp := time.Unix(headMessage.UpdateTime/1000, (headMessage.UpdateTime%1000)*int64(time.Millisecond)) + tz := sctx.GetSessionVars().Location() + if updateTimestamp.Location() != tz { + updateTimestamp.In(tz) + } + updateTime := types.NewTime(types.FromGoTime(updateTimestamp), mysql.TypeTimestamp, types.MinFsp) + row[0].SetMysqlTime(updateTime) row[1].SetString(strings.ToUpper(f.DBName), mysql.DefaultCollationName) row[2].SetString(strings.ToUpper(f.TableName), mysql.DefaultCollationName) row[3].SetInt64(f.TableID) diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index 020523cd8744e..3befca4fdc1ed 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -1024,34 +1024,34 @@ func (s *testHotRegionsHistoryTableSuite) setUpMockPDHTTPServer(c *C, l net.List func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { var unixTimeMs = func(s string) int64 { - t, err := time.ParseInLocation("2006/01/02 15:04:05", s, time.Local) + t, err := time.ParseInLocation("2006-01-02 15:04:05", s, time.Local) c.Assert(err, IsNil) return t.UnixNano() / int64(time.Millisecond) } fullHotRegions := [][]string{ // Record Regions // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV - {"2019/10/10 10:10:11", "MYSQL", "TABLES_PRIV", "11", "", "", "1", "1", "11111", "READ", "99", "99", "99", "99"}, - {"2019/10/10 10:10:12", "MYSQL", "TABLES_PRIV", "11", "", "", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:11", "MYSQL", "TABLES_PRIV", "11", "", "", "1", "1", "11111", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:12", "MYSQL", "TABLES_PRIV", "11", "", "", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, // mysql table_id = 21, record_id = 1, table_name = STATS_META - {"2019/10/10 10:10:13", "MYSQL", "STATS_META", "21", "", "", "3", "3", "33333", "READ", "99", "99", "99", "99"}, - {"2019/10/10 10:10:14", "MYSQL", "STATS_META", "21", "", "", "4", "4", "44444", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:13", "MYSQL", "STATS_META", "21", "", "", "3", "3", "33333", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:14", "MYSQL", "STATS_META", "21", "", "", "4", "4", "44444", "WRITE", "99", "99", "99", "99"}, // table_id = 131, record_id=1, deleted schema - {"2019/10/10 10:10:15", "UNKONW", "UNKONW", "131", "UNKONW", "", "5", "5", "55555", "READ", "99", "99", "99", "99"}, - {"2019/10/10 10:10:16", "UNKONW", "UNKONW", "131", "UNKONW", "", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:15", "UNKONW", "UNKONW", "131", "UNKONW", "", "5", "5", "55555", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:16", "UNKONW", "UNKONW", "131", "UNKONW", "", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, // Index Regions // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY - {"2019/10/10 10:10:17", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "1", "1", "11111", "READ", "99", "99", "99", "99"}, - {"2019/10/10 10:10:18", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:17", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "1", "1", "11111", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:18", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER - {"2019/10/10 10:10:19", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "3", "3", "33333", "READ", "99", "99", "99", "99"}, - {"2019/10/10 10:10:20", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "4", "4", "44444", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:19", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "3", "3", "33333", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:20", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "4", "4", "44444", "WRITE", "99", "99", "99", "99"}, // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL - {"2019/10/10 10:10:21", "MYSQL", "STATS_META", "21", "TBL", "2", "5", "5", "55555", "READ", "99", "99", "99", "99"}, - {"2019/10/10 10:10:22", "MYSQL", "STATS_META", "21", "TBL", "2", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:21", "MYSQL", "STATS_META", "21", "TBL", "2", "5", "5", "55555", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:22", "MYSQL", "STATS_META", "21", "TBL", "2", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, // table_id = 131, index_id = 1, index_value = 1, deleted schema - {"2019/10/10 10:10:23", "UNKONW", "UNKONW", "131", "UNKONW", "1", "5", "5", "55555", "READ", "99", "99", "99", "99"}, - {"2019/10/10 10:10:24", "UNKONW", "UNKONW", "131", "UNKONW", "1", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:23", "UNKONW", "UNKONW", "131", "UNKONW", "1", "5", "5", "55555", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:24", "UNKONW", "UNKONW", "131", "UNKONW", "1", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, } c.Assert(fullHotRegions, NotNil) @@ -1060,57 +1060,57 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { {HistoryHotRegion: []*executor.HistoryHotRegion{ // Record Regions // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV - {UpdateTime: unixTimeMs("2019/10/10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019/10/10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21, record_id = 1, table_name = STATS_META - {UpdateTime: unixTimeMs("2019/10/10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019/10/10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }}, {HistoryHotRegion: []*executor.HistoryHotRegion{ // table_id = 131, record_id=1, deleted schema - {UpdateTime: unixTimeMs("2019/10/10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019/10/10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // Index Regions // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY - {UpdateTime: unixTimeMs("2019/10/10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019/10/10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }}, {HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER - {UpdateTime: unixTimeMs("2019/10/10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019/10/10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL - {UpdateTime: unixTimeMs("2019/10/10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019/10/10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // table_id = 131, index_id = 1, index_value = 1, deleted schema - {UpdateTime: unixTimeMs("2019/10/10 10:10:23"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:23"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019/10/10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }}, @@ -1138,6 +1138,18 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { fullHotRegions[9], fullHotRegions[10], fullHotRegions[11], }, }, + { + conditions: []string{ + "update_time>=TIMESTAMP('2019-10-10 10:10:10')", + "update_time<=TIMESTAMP('2019-10-11 10:10:10')", + }, // test support of timestamp + expected: [][]string{ + fullHotRegions[0], fullHotRegions[1], fullHotRegions[2], + fullHotRegions[3], + fullHotRegions[6], fullHotRegions[7], fullHotRegions[8], + fullHotRegions[9], fullHotRegions[10], fullHotRegions[11], + }, + }, { conditions: []string{ "update_time>='2019-10-10 10:10:10'", From ff0c630add5342ef7db8b9394bc85f7f04ee717b Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Sun, 22 Aug 2021 07:05:04 +0000 Subject: [PATCH 07/29] executor: devide read and write hot types into two http request to fix results order overlap Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 125 +++++++++------- executor/memtable_reader_test.go | 250 +++++++++++++++++++------------ 2 files changed, 225 insertions(+), 150 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 345a8600f34c7..94b2bd26f96b3 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -789,6 +789,13 @@ type HistoryHotRegion struct { EndKey []byte `json:"end_key,omitempty"` } +const ( + // HotRegionTypeREAD hot read region. + HotRegionTypeREAD = "READ" + // HotRegionTypeWRITE hot write region. + HotRegionTypeWRITE = "WRITE" +) + func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx sessionctx.Context) ([]chan hotRegionsStreamResult, error) { if !hasPriv(sctx, mysql.ProcessPriv) { return nil, plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS") @@ -806,76 +813,82 @@ func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx session return nil, errors.New("denied to scan hot regions, please specified the end time, such as `update_time < '2020-01-01 00:00:00'`") } + // Divide read write into two request because of time range ovelap, + // because PD use [type,time] as key of hot regions. + if e.extractor.HotRegionTypes.Count() == 0 { + e.extractor.HotRegionTypes.Insert(HotRegionTypeREAD) + e.extractor.HotRegionTypes.Insert(HotRegionTypeWRITE) + } hotRegionTypes := make([]string, 0, e.extractor.HotRegionTypes.Count()) for typ := range e.extractor.HotRegionTypes { hotRegionTypes = append(hotRegionTypes, typ) } - + // set hotType before request historyHotRegionsRequest := &HistoryHotRegionsRequest{ - StartTime: e.extractor.StartTime, - EndTime: e.extractor.EndTime, - RegionIDs: e.extractor.RegionIDs, - StoreIDs: e.extractor.StoreIDs, - PeerIDs: e.extractor.PeerIDs, - HotRegionTypes: hotRegionTypes, - LowHotDegree: e.extractor.LowHotDegree, - HighHotDegree: e.extractor.HighHotDegree, - LowFlowBytes: e.extractor.LowFlowBytes, - HighFlowBytes: e.extractor.HighFlowBytes, - LowKeyRate: e.extractor.LowKeyRate, - HighKeyRate: e.extractor.HighKeyRate, - LowQueryRate: e.extractor.LowQueryRate, - HighQueryRate: e.extractor.HighQueryRate, - } - jsonBody, err := json.Marshal(historyHotRegionsRequest) - if err != nil { - return nil, err - } - body := bytes.NewBuffer(jsonBody) - return e.startRetrieving(ctx, sctx, pdServers, body) + StartTime: e.extractor.StartTime, + EndTime: e.extractor.EndTime, + RegionIDs: e.extractor.RegionIDs, + StoreIDs: e.extractor.StoreIDs, + PeerIDs: e.extractor.PeerIDs, + LowHotDegree: e.extractor.LowHotDegree, + HighHotDegree: e.extractor.HighHotDegree, + LowFlowBytes: e.extractor.LowFlowBytes, + HighFlowBytes: e.extractor.HighFlowBytes, + LowKeyRate: e.extractor.LowKeyRate, + HighKeyRate: e.extractor.HighKeyRate, + LowQueryRate: e.extractor.LowQueryRate, + HighQueryRate: e.extractor.HighQueryRate, + } + return e.startRetrieving(ctx, sctx, pdServers, historyHotRegionsRequest) } func (e *hotRegionsHistoryRetriver) startRetrieving( ctx context.Context, sctx sessionctx.Context, serversInfo []infoschema.ServerInfo, - body *bytes.Buffer, + req *HistoryHotRegionsRequest, ) ([]chan hotRegionsStreamResult, error) { - wg := sync.WaitGroup{} var results []chan hotRegionsStreamResult for _, srv := range serversInfo { - ch := make(chan hotRegionsStreamResult) - results = append(results, ch) - go func(address string, body *bytes.Buffer) { - util.WithRecovery(func() { - defer wg.Done() - url := fmt.Sprintf("%s://%s%s", util.InternalHTTPSchema(), address, pdapi.HotHistory) - req, err := http.NewRequest(http.MethodPost, url, body) - if err != nil { - ch <- hotRegionsStreamResult{err: errors.Trace(err)} - return - } - req.Header.Add("PD-Allow-follower-handle", "true") - resp, err := util.InternalHTTPClient().Do(req) - if err != nil { - ch <- hotRegionsStreamResult{err: errors.Trace(err)} - return - } - defer func() { - terror.Log(resp.Body.Close()) - }() - if resp.StatusCode != http.StatusOK { - ch <- hotRegionsStreamResult{err: errors.Errorf("request %s failed: %s", url, resp.Status)} - return - } - var historyHotRegions HistoryHotRegions - if err = json.NewDecoder(resp.Body).Decode(&historyHotRegions); err != nil { - ch <- hotRegionsStreamResult{err: errors.Trace(err)} - return - } - ch <- hotRegionsStreamResult{addr: address, messages: &historyHotRegions} - }, nil) - }(srv.StatusAddr, body) + for typ := range e.extractor.HotRegionTypes { + req.HotRegionTypes = []string{typ} + jsonBody, err := json.Marshal(req) + if err != nil { + return nil, err + } + body := bytes.NewBuffer(jsonBody) + ch := make(chan hotRegionsStreamResult) + results = append(results, ch) + go func(address string, body *bytes.Buffer) { + util.WithRecovery(func() { + url := fmt.Sprintf("%s://%s%s", util.InternalHTTPSchema(), address, pdapi.HotHistory) + req, err := http.NewRequest(http.MethodPost, url, body) + if err != nil { + ch <- hotRegionsStreamResult{err: errors.Trace(err)} + return + } + req.Header.Add("PD-Allow-follower-handle", "true") + resp, err := util.InternalHTTPClient().Do(req) + if err != nil { + ch <- hotRegionsStreamResult{err: errors.Trace(err)} + return + } + defer func() { + terror.Log(resp.Body.Close()) + }() + if resp.StatusCode != http.StatusOK { + ch <- hotRegionsStreamResult{err: errors.Errorf("request %s failed: %s", url, resp.Status)} + return + } + var historyHotRegions HistoryHotRegions + if err = json.NewDecoder(resp.Body).Decode(&historyHotRegions); err != nil { + ch <- hotRegionsStreamResult{err: errors.Trace(err)} + return + } + ch <- hotRegionsStreamResult{addr: address, messages: &historyHotRegions} + }, nil) + }(srv.StatusAddr, body) + } } return results, nil } diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index 3befca4fdc1ed..f00641b984374 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -17,7 +17,9 @@ package executor_test import ( "context" "crypto/tls" + "encoding/json" "fmt" + "io" "log" "net" "net/http" @@ -42,7 +44,6 @@ import ( "github.com/pingcap/tidb/util/pdapi" "github.com/pingcap/tidb/util/testkit" pmodel "github.com/prometheus/common/model" - "github.com/soheilhy/cmux" "google.golang.org/grpc" ) @@ -960,6 +961,8 @@ type mockStoreWithMultiPD struct { hosts []string } +var hotRegionsResponses = make(map[string]*executor.HistoryHotRegions, 3) + func (s *mockStoreWithMultiPD) EtcdAddrs() ([]string, error) { return s.hosts, nil } func (s *mockStoreWithMultiPD) TLSConfig() *tls.Config { panic("not implemented") } func (s *mockStoreWithMultiPD) StartGCWorker() error { panic("not implemented") } @@ -972,34 +975,61 @@ func (s *testHotRegionsHistoryTableSuite) SetUpSuite(c *C) { s.store.(helper.Storage), make([]string, 3), } - // start 3 PD server with hotRegionsServer and store them in s.store for i := 0; i < 3; i++ { - listener, err := net.Listen("tcp", "127.0.0.1:0") - c.Assert(err, IsNil) - m := cmux.New(listener) - // server grpc and http on the same port - httpListener := m.Match(cmux.HTTP1Fast()) - httpServer, mockAddr := s.setUpMockPDHTTPServer(c, httpListener) + httpServer, mockAddr := s.setUpMockPDHTTPServer(c) store.hosts[i] = mockAddr c.Assert(httpServer, NotNil) - - go func() { - err = m.Serve() - }() } s.store = store s.startTime = time.Now() } -var hotRegionsResponses = make(map[string]*executor.HistoryHotRegions, 3) +func writeJSONError(w http.ResponseWriter, code int, prefix string, err error) { + type errorResponse struct { + Error string `json:"error"` + } + + w.WriteHeader(code) + + if err != nil { + prefix += ": " + err.Error() + } + _ = json.NewEncoder(w).Encode(errorResponse{Error: prefix}) +} + +func hisHotRegionsHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + data, err := io.ReadAll(r.Body) + if err != nil { + writeJSONError(w, http.StatusInternalServerError, "unable to read req", err) + return + } + r.Body.Close() + req := &executor.HistoryHotRegionsRequest{} + err = json.Unmarshal(data, req) + if err != nil { + writeJSONError(w, http.StatusInternalServerError, "unable to serialize req", err) + return + } + resp := &executor.HistoryHotRegions{} + for _, typ := range req.HotRegionTypes { + resp.HistoryHotRegion = append(resp.HistoryHotRegion, hotRegionsResponses[strings.ToUpper(typ)+r.Host].HistoryHotRegion...) + } + w.WriteHeader(http.StatusOK) + jsonResp, err := json.Marshal(resp) + if err != nil { + writeJSONError(w, http.StatusInternalServerError, "unable to marshal resp", err) + return + } + w.Write(jsonResp) +} -func (s *testHotRegionsHistoryTableSuite) setUpMockPDHTTPServer(c *C, l net.Listener) (*http.Server, string) { +func (s *testHotRegionsHistoryTableSuite) setUpMockPDHTTPServer(c *C) (*httptest.Server, string) { // mock PD http server router := mux.NewRouter() - httpServer := &http.Server{ - Handler: router, - } + server := httptest.NewServer(router) + mockAddr := strings.TrimPrefix(server.URL, "http://") // mock PD API router.Handle(pdapi.ClusterVersion, fn.Wrap(func() (string, error) { return "4.0.0-alpha", nil })) router.Handle(pdapi.Status, fn.Wrap(func() (interface{}, error) { @@ -1011,15 +1041,9 @@ func (s *testHotRegionsHistoryTableSuite) setUpMockPDHTTPServer(c *C, l net.List StartTimestamp: s.startTime.Unix(), }, nil })) - // mock response - router.Handle(pdapi.HotHistory, fn.Wrap(func() (interface{}, error) { - return hotRegionsResponses[l.Addr().String()], nil - })) - go func() { - err := httpServer.Serve(l) - c.Assert(err, IsNil) - }() - return httpServer, l.Addr().String() + // mock hisory hot regions response + router.HandleFunc(pdapi.HotHistory, hisHotRegionsHandler) + return server, mockAddr } func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { @@ -1029,7 +1053,6 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { return t.UnixNano() / int64(time.Millisecond) } fullHotRegions := [][]string{ - // Record Regions // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV {"2019-10-10 10:10:11", "MYSQL", "TABLES_PRIV", "11", "", "", "1", "1", "11111", "READ", "99", "99", "99", "99"}, {"2019-10-10 10:10:12", "MYSQL", "TABLES_PRIV", "11", "", "", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, @@ -1039,7 +1062,6 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { // table_id = 131, record_id=1, deleted schema {"2019-10-10 10:10:15", "UNKONW", "UNKONW", "131", "UNKONW", "", "5", "5", "55555", "READ", "99", "99", "99", "99"}, {"2019-10-10 10:10:16", "UNKONW", "UNKONW", "131", "UNKONW", "", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, - // Index Regions // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY {"2019-10-10 10:10:17", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "1", "1", "11111", "READ", "99", "99", "99", "99"}, {"2019-10-10 10:10:18", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, @@ -1056,69 +1078,93 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { c.Assert(fullHotRegions, NotNil) - pdResps := []*executor.HistoryHotRegions{ - {HistoryHotRegion: []*executor.HistoryHotRegion{ - // Record Regions - // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV - {UpdateTime: unixTimeMs("2019-10-10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019-10-10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - // mysql table_id = 21, record_id = 1, table_name = STATS_META - {UpdateTime: unixTimeMs("2019-10-10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019-10-10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - }}, - {HistoryHotRegion: []*executor.HistoryHotRegion{ - // table_id = 131, record_id=1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019-10-10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - // Index Regions - // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY - {UpdateTime: unixTimeMs("2019-10-10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019-10-10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - }}, - {HistoryHotRegion: []*executor.HistoryHotRegion{ - // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER - {UpdateTime: unixTimeMs("2019-10-10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019-10-10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL - {UpdateTime: unixTimeMs("2019-10-10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019-10-10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - // table_id = 131, index_id = 1, index_value = 1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:23"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - {UpdateTime: unixTimeMs("2019-10-10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, - StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, - EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, - }}, - } - // mock http resp - store := s.store.(*mockStoreWithMultiPD) - for i, resp := range pdResps { - hotRegionsResponses[store.hosts[i]] = resp + pdResps := []map[string]*executor.HistoryHotRegions{ + { + executor.HotRegionTypeREAD: { + HistoryHotRegion: []*executor.HistoryHotRegion{ + // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV + {UpdateTime: unixTimeMs("2019-10-10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + // mysql table_id = 21, record_id = 1, table_name = STATS_META + {UpdateTime: unixTimeMs("2019-10-10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + }, + }, + executor.HotRegionTypeWRITE: { + HistoryHotRegion: []*executor.HistoryHotRegion{ + // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV + {UpdateTime: unixTimeMs("2019-10-10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + // mysql table_id = 21, record_id = 1, table_name = STATS_META + {UpdateTime: unixTimeMs("2019-10-10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + }, + }, + }, + { + executor.HotRegionTypeREAD: { + HistoryHotRegion: []*executor.HistoryHotRegion{ + // table_id = 131, record_id=1, deleted schema + {UpdateTime: unixTimeMs("2019-10-10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY + {UpdateTime: unixTimeMs("2019-10-10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + }, + }, + executor.HotRegionTypeWRITE: { + HistoryHotRegion: []*executor.HistoryHotRegion{ + // table_id = 131, record_id=1, deleted schema + {UpdateTime: unixTimeMs("2019-10-10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY + {UpdateTime: unixTimeMs("2019-10-10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + }, + }, + }, + { + executor.HotRegionTypeREAD: { + HistoryHotRegion: []*executor.HistoryHotRegion{ + // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER + {UpdateTime: unixTimeMs("2019-10-10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL + {UpdateTime: unixTimeMs("2019-10-10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + // table_id = 131, index_id = 1, index_value = 1, deleted schema + {UpdateTime: unixTimeMs("2019-10-10 10:10:23"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + }, + }, + executor.HotRegionTypeWRITE: { + HistoryHotRegion: []*executor.HistoryHotRegion{ + // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER + {UpdateTime: unixTimeMs("2019-10-10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL + {UpdateTime: unixTimeMs("2019-10-10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + // table_id = 131, index_id = 1, index_value = 1, deleted schema + {UpdateTime: unixTimeMs("2019-10-10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, + EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, + }, + }, + }, } var cases = []struct { @@ -1126,6 +1172,18 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { reqCount int32 expected [][]string }{ + { + conditions: []string{ + "update_time>='2019-10-10 10:10:10'", + "update_time<='2019-10-11 10:10:10'", + "table_id=21", + "index_id=1", + "table_name='STATS_META'", + }, // table_id = table_name + expected: [][]string{ + fullHotRegions[8], fullHotRegions[9], + }, + }, { conditions: []string{ "update_time>='2019-10-10 10:10:10'", @@ -1290,16 +1348,20 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { fpName := "github.com/pingcap/tidb/executor/mockTiDBHotRegionsHistory" c.Assert(failpoint.Enable(fpName, `return("")`), IsNil) defer func() { c.Assert(failpoint.Disable(fpName), IsNil) }() - + // mock http resp + store := s.store.(*mockStoreWithMultiPD) + for i, resp := range pdResps { + for k, v := range resp { + hotRegionsResponses[k+store.hosts[i]] = v + } + } tk := testkit.NewTestKit(c, s.store) - for _, cas := range cases { sql := "select * from information_schema.tidb_hot_regions_history" if len(cas.conditions) > 0 { sql = fmt.Sprintf("%s where %s", sql, strings.Join(cas.conditions, " and ")) } result := tk.MustQuery(sql) - // rows := result.Rows() warnings := tk.Se.GetSessionVars().StmtCtx.GetWarnings() c.Assert(len(warnings), Equals, 0, Commentf("unexpected warnigns: %+v", warnings)) var expected []string From faf8f5212f23fd7e4b3ff8e7769ef1bfe5532098 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Sun, 22 Aug 2021 08:28:51 +0000 Subject: [PATCH 08/29] executor: add is_leader in request and update retriver test Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 31 ++++++++++++------ executor/memtable_reader_test.go | 56 ++++++++++++++++---------------- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 94b2bd26f96b3..b599f0f6075e3 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -756,6 +756,7 @@ type HistoryHotRegionsRequest struct { RegionIDs []uint64 `json:"region_ids,omitempty"` StoreIDs []uint64 `json:"store_ids,omitempty"` PeerIDs []uint64 `json:"peer_ids,omitempty"` + IsLeader bool `json:"is_leader,omitempty"` HotRegionTypes []string `json:"hot_region_types,omitempty"` LowHotDegree int64 `json:"low_hot_degree,omitempty"` HighHotDegree int64 `json:"high_hot_degree,omitempty"` @@ -778,8 +779,9 @@ type HistoryHotRegions struct { type HistoryHotRegion struct { UpdateTime int64 `json:"update_time,omitempty"` RegionID uint64 `json:"region_id,omitempty"` - PeerID uint64 `json:"peer_id,omitempty"` StoreID uint64 `json:"store_id,omitempty"` + PeerID uint64 `json:"peer_id,omitempty"` + IsLeader bool `json:"is_leader,omitempty"` HotRegionType string `json:"hot_region_type,omitempty"` HotDegree int64 `json:"hot_degree,omitempty"` FlowBytes float64 `json:"flow_bytes,omitempty"` @@ -839,6 +841,10 @@ func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx session LowQueryRate: e.extractor.LowQueryRate, HighQueryRate: e.extractor.HighQueryRate, } + // Not set isLeader when roles' length equal to 0(no condition), 2(leader or follower) + if e.extractor.Roles.Count() == 1 { + historyHotRegionsRequest.IsLeader = e.extractor.Roles.Exist(1) + } return e.startRetrieving(ctx, sctx, pdServers, historyHotRegionsRequest) } @@ -1007,26 +1013,31 @@ func (e *hotRegionsHistoryRetriver) parseAndFilterBySchemaInfo(sctx sessionctx.C row[6].SetInt64(int64(headMessage.RegionID)) row[7].SetInt64(int64(headMessage.StoreID)) row[8].SetInt64(int64(headMessage.PeerID)) - row[9].SetString(strings.ToUpper(headMessage.HotRegionType), mysql.DefaultCollationName) + if headMessage.IsLeader { + row[9].SetInt64(1) + } else { + row[9].SetInt64(0) + } + row[10].SetString(strings.ToUpper(headMessage.HotRegionType), mysql.DefaultCollationName) if headMessage.HotDegree != 0 { - row[10].SetInt64(headMessage.HotDegree) + row[11].SetInt64(headMessage.HotDegree) } else { - row[10].SetNull() + row[11].SetNull() } if headMessage.FlowBytes != 0 { - row[11].SetFloat64(float64(headMessage.FlowBytes)) + row[12].SetFloat64(float64(headMessage.FlowBytes)) } else { - row[11].SetNull() + row[12].SetNull() } if headMessage.KeyRate != 0 { - row[12].SetFloat64(float64(headMessage.KeyRate)) + row[13].SetFloat64(float64(headMessage.KeyRate)) } else { - row[12].SetNull() + row[13].SetNull() } if headMessage.QueryRate != 0 { - row[13].SetFloat64(float64(headMessage.QueryRate)) + row[14].SetFloat64(float64(headMessage.QueryRate)) } else { - row[13].SetNull() + row[14].SetNull() } return row, nil } diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index f00641b984374..2ceba050650b8 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -1054,26 +1054,26 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { } fullHotRegions := [][]string{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV - {"2019-10-10 10:10:11", "MYSQL", "TABLES_PRIV", "11", "", "", "1", "1", "11111", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:12", "MYSQL", "TABLES_PRIV", "11", "", "", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:11", "MYSQL", "TABLES_PRIV", "11", "", "", "1", "1", "11111", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:12", "MYSQL", "TABLES_PRIV", "11", "", "", "2", "2", "22222", "0", "WRITE", "99", "99", "99", "99"}, // mysql table_id = 21, record_id = 1, table_name = STATS_META - {"2019-10-10 10:10:13", "MYSQL", "STATS_META", "21", "", "", "3", "3", "33333", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:14", "MYSQL", "STATS_META", "21", "", "", "4", "4", "44444", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:13", "MYSQL", "STATS_META", "21", "", "", "3", "3", "33333", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:14", "MYSQL", "STATS_META", "21", "", "", "4", "4", "44444", "0", "WRITE", "99", "99", "99", "99"}, // table_id = 131, record_id=1, deleted schema - {"2019-10-10 10:10:15", "UNKONW", "UNKONW", "131", "UNKONW", "", "5", "5", "55555", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:16", "UNKONW", "UNKONW", "131", "UNKONW", "", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:15", "UNKONW", "UNKONW", "131", "UNKONW", "", "5", "5", "55555", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:16", "UNKONW", "UNKONW", "131", "UNKONW", "", "6", "6", "66666", "0", "WRITE", "99", "99", "99", "99"}, // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY - {"2019-10-10 10:10:17", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "1", "1", "11111", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:18", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "2", "2", "22222", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:17", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "1", "1", "11111", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:18", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "2", "2", "22222", "0", "WRITE", "99", "99", "99", "99"}, // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER - {"2019-10-10 10:10:19", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "3", "3", "33333", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:20", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "4", "4", "44444", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:19", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "3", "3", "33333", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:20", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "4", "4", "44444", "0", "WRITE", "99", "99", "99", "99"}, // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL - {"2019-10-10 10:10:21", "MYSQL", "STATS_META", "21", "TBL", "2", "5", "5", "55555", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:22", "MYSQL", "STATS_META", "21", "TBL", "2", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:21", "MYSQL", "STATS_META", "21", "TBL", "2", "5", "5", "55555", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:22", "MYSQL", "STATS_META", "21", "TBL", "2", "6", "6", "66666", "0", "WRITE", "99", "99", "99", "99"}, // table_id = 131, index_id = 1, index_value = 1, deleted schema - {"2019-10-10 10:10:23", "UNKONW", "UNKONW", "131", "UNKONW", "1", "5", "5", "55555", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:24", "UNKONW", "UNKONW", "131", "UNKONW", "1", "6", "6", "66666", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:23", "UNKONW", "UNKONW", "131", "UNKONW", "1", "5", "5", "55555", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:24", "UNKONW", "UNKONW", "131", "UNKONW", "1", "6", "6", "66666", "0", "WRITE", "99", "99", "99", "99"}, } c.Assert(fullHotRegions, NotNil) @@ -1083,11 +1083,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeREAD: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV - {UpdateTime: unixTimeMs("2019-10-10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21, record_id = 1, table_name = STATS_META - {UpdateTime: unixTimeMs("2019-10-10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1095,11 +1095,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeWRITE: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV - {UpdateTime: unixTimeMs("2019-10-10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21, record_id = 1, table_name = STATS_META - {UpdateTime: unixTimeMs("2019-10-10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1109,11 +1109,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeREAD: { HistoryHotRegion: []*executor.HistoryHotRegion{ // table_id = 131, record_id=1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY - {UpdateTime: unixTimeMs("2019-10-10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1121,11 +1121,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeWRITE: { HistoryHotRegion: []*executor.HistoryHotRegion{ // table_id = 131, record_id=1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY - {UpdateTime: unixTimeMs("2019-10-10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1135,15 +1135,15 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeREAD: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER - {UpdateTime: unixTimeMs("2019-10-10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL - {UpdateTime: unixTimeMs("2019-10-10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // table_id = 131, index_id = 1, index_value = 1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:23"), RegionID: 5, StoreID: 5, PeerID: 55555, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:23"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1151,15 +1151,15 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeWRITE: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER - {UpdateTime: unixTimeMs("2019-10-10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL - {UpdateTime: unixTimeMs("2019-10-10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // table_id = 131, index_id = 1, index_value = 1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, From 0330dc6484eee1d3b29a49b029ffce2e3febcf40 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Sun, 22 Aug 2021 10:31:25 +0000 Subject: [PATCH 09/29] executor: remove debug test case Signed-off-by: IcePigZDB --- executor/memtable_reader_test.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index 2ceba050650b8..15a461041efb2 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -1172,18 +1172,6 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { reqCount int32 expected [][]string }{ - { - conditions: []string{ - "update_time>='2019-10-10 10:10:10'", - "update_time<='2019-10-11 10:10:10'", - "table_id=21", - "index_id=1", - "table_name='STATS_META'", - }, // table_id = table_name - expected: [][]string{ - fullHotRegions[8], fullHotRegions[9], - }, - }, { conditions: []string{ "update_time>='2019-10-10 10:10:10'", From 42a5c616a4c391f2c030f60e1e6b7f0001d2daa0 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Sun, 22 Aug 2021 12:08:57 +0000 Subject: [PATCH 10/29] executor: close httpServers after test down Signed-off-by: IcePigZDB --- executor/memtable_reader_test.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index 15a461041efb2..e5e72fe17dc4b 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -953,7 +953,8 @@ var _ = SerialSuites(&testHotRegionsHistoryTableSuite{testInfoschemaTableSuiteBa type testHotRegionsHistoryTableSuite struct { *testInfoschemaTableSuiteBase - startTime time.Time + httpServers []*httptest.Server + startTime time.Time } type mockStoreWithMultiPD struct { @@ -978,8 +979,9 @@ func (s *testHotRegionsHistoryTableSuite) SetUpSuite(c *C) { // start 3 PD server with hotRegionsServer and store them in s.store for i := 0; i < 3; i++ { httpServer, mockAddr := s.setUpMockPDHTTPServer(c) - store.hosts[i] = mockAddr c.Assert(httpServer, NotNil) + s.httpServers = append(s.httpServers, httpServer) + store.hosts[i] = mockAddr } s.store = store s.startTime = time.Now() @@ -989,9 +991,7 @@ func writeJSONError(w http.ResponseWriter, code int, prefix string, err error) { type errorResponse struct { Error string `json:"error"` } - w.WriteHeader(code) - if err != nil { prefix += ": " + err.Error() } @@ -1046,6 +1046,13 @@ func (s *testHotRegionsHistoryTableSuite) setUpMockPDHTTPServer(c *C) (*httptest return server, mockAddr } +func (s *testHotRegionsHistoryTableSuite) TearDownSuite(c *C) { + for _, server := range s.httpServers { + server.Close() + } + s.testInfoschemaTableSuiteBase.TearDownSuite(c) +} + func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { var unixTimeMs = func(s string) int64 { t, err := time.ParseInLocation("2006-01-02 15:04:05", s, time.Local) From c8e440da4fa764c1bda4d99f306af76c685a4808 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Mon, 23 Aug 2021 02:28:48 +0000 Subject: [PATCH 11/29] executor: roles from intset to slice Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 16 +++++----------- executor/memtable_reader_test.go | 28 ++++++++++++++-------------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index b599f0f6075e3..b2498fa51cfeb 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -756,7 +756,7 @@ type HistoryHotRegionsRequest struct { RegionIDs []uint64 `json:"region_ids,omitempty"` StoreIDs []uint64 `json:"store_ids,omitempty"` PeerIDs []uint64 `json:"peer_ids,omitempty"` - IsLeader bool `json:"is_leader,omitempty"` + Roles []uint64 `json:"roles,omitempty"` HotRegionTypes []string `json:"hot_region_types,omitempty"` LowHotDegree int64 `json:"low_hot_degree,omitempty"` HighHotDegree int64 `json:"high_hot_degree,omitempty"` @@ -781,7 +781,7 @@ type HistoryHotRegion struct { RegionID uint64 `json:"region_id,omitempty"` StoreID uint64 `json:"store_id,omitempty"` PeerID uint64 `json:"peer_id,omitempty"` - IsLeader bool `json:"is_leader,omitempty"` + IsLeader int64 `json:"is_leader,omitempty"` HotRegionType string `json:"hot_region_type,omitempty"` HotDegree int64 `json:"hot_degree,omitempty"` FlowBytes float64 `json:"flow_bytes,omitempty"` @@ -832,6 +832,7 @@ func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx session RegionIDs: e.extractor.RegionIDs, StoreIDs: e.extractor.StoreIDs, PeerIDs: e.extractor.PeerIDs, + Roles: e.extractor.Roles, LowHotDegree: e.extractor.LowHotDegree, HighHotDegree: e.extractor.HighHotDegree, LowFlowBytes: e.extractor.LowFlowBytes, @@ -841,10 +842,7 @@ func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx session LowQueryRate: e.extractor.LowQueryRate, HighQueryRate: e.extractor.HighQueryRate, } - // Not set isLeader when roles' length equal to 0(no condition), 2(leader or follower) - if e.extractor.Roles.Count() == 1 { - historyHotRegionsRequest.IsLeader = e.extractor.Roles.Exist(1) - } + return e.startRetrieving(ctx, sctx, pdServers, historyHotRegionsRequest) } @@ -1013,11 +1011,7 @@ func (e *hotRegionsHistoryRetriver) parseAndFilterBySchemaInfo(sctx sessionctx.C row[6].SetInt64(int64(headMessage.RegionID)) row[7].SetInt64(int64(headMessage.StoreID)) row[8].SetInt64(int64(headMessage.PeerID)) - if headMessage.IsLeader { - row[9].SetInt64(1) - } else { - row[9].SetInt64(0) - } + row[9].SetInt64(headMessage.IsLeader) row[10].SetString(strings.ToUpper(headMessage.HotRegionType), mysql.DefaultCollationName) if headMessage.HotDegree != 0 { row[11].SetInt64(headMessage.HotDegree) diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index e5e72fe17dc4b..314487e91d4c9 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -1090,11 +1090,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeREAD: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV - {UpdateTime: unixTimeMs("2019-10-10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21, record_id = 1, table_name = STATS_META - {UpdateTime: unixTimeMs("2019-10-10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1102,11 +1102,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeWRITE: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV - {UpdateTime: unixTimeMs("2019-10-10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21, record_id = 1, table_name = STATS_META - {UpdateTime: unixTimeMs("2019-10-10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1116,11 +1116,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeREAD: { HistoryHotRegion: []*executor.HistoryHotRegion{ // table_id = 131, record_id=1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY - {UpdateTime: unixTimeMs("2019-10-10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1128,11 +1128,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeWRITE: { HistoryHotRegion: []*executor.HistoryHotRegion{ // table_id = 131, record_id=1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY - {UpdateTime: unixTimeMs("2019-10-10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1142,15 +1142,15 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeREAD: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER - {UpdateTime: unixTimeMs("2019-10-10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL - {UpdateTime: unixTimeMs("2019-10-10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // table_id = 131, index_id = 1, index_value = 1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:23"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:23"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1158,15 +1158,15 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeWRITE: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER - {UpdateTime: unixTimeMs("2019-10-10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL - {UpdateTime: unixTimeMs("2019-10-10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // table_id = 131, index_id = 1, index_value = 1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, From 034533d283014fdb47103c5408d2bd0b2f85bb49 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Mon, 23 Aug 2021 11:49:03 +0000 Subject: [PATCH 12/29] execuotr: use bool for IsLeader Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 9 +++++++-- executor/memtable_reader_test.go | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index b2498fa51cfeb..7ba6662cc9385 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -781,7 +781,7 @@ type HistoryHotRegion struct { RegionID uint64 `json:"region_id,omitempty"` StoreID uint64 `json:"store_id,omitempty"` PeerID uint64 `json:"peer_id,omitempty"` - IsLeader int64 `json:"is_leader,omitempty"` + IsLeader bool `json:"is_leader,omitempty"` HotRegionType string `json:"hot_region_type,omitempty"` HotDegree int64 `json:"hot_degree,omitempty"` FlowBytes float64 `json:"flow_bytes,omitempty"` @@ -1011,7 +1011,12 @@ func (e *hotRegionsHistoryRetriver) parseAndFilterBySchemaInfo(sctx sessionctx.C row[6].SetInt64(int64(headMessage.RegionID)) row[7].SetInt64(int64(headMessage.StoreID)) row[8].SetInt64(int64(headMessage.PeerID)) - row[9].SetInt64(headMessage.IsLeader) + if headMessage.IsLeader { + row[9].SetInt64(1) + } else { + row[9].SetInt64(0) + } + row[10].SetString(strings.ToUpper(headMessage.HotRegionType), mysql.DefaultCollationName) if headMessage.HotDegree != 0 { row[11].SetInt64(headMessage.HotDegree) diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index 314487e91d4c9..e5e72fe17dc4b 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -1090,11 +1090,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeREAD: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV - {UpdateTime: unixTimeMs("2019-10-10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21, record_id = 1, table_name = STATS_META - {UpdateTime: unixTimeMs("2019-10-10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1102,11 +1102,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeWRITE: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV - {UpdateTime: unixTimeMs("2019-10-10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21, record_id = 1, table_name = STATS_META - {UpdateTime: unixTimeMs("2019-10-10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1116,11 +1116,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeREAD: { HistoryHotRegion: []*executor.HistoryHotRegion{ // table_id = 131, record_id=1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY - {UpdateTime: unixTimeMs("2019-10-10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1128,11 +1128,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeWRITE: { HistoryHotRegion: []*executor.HistoryHotRegion{ // table_id = 131, record_id=1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY - {UpdateTime: unixTimeMs("2019-10-10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1142,15 +1142,15 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeREAD: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER - {UpdateTime: unixTimeMs("2019-10-10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL - {UpdateTime: unixTimeMs("2019-10-10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // table_id = 131, index_id = 1, index_value = 1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:23"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: 1, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:23"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1158,15 +1158,15 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeWRITE: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER - {UpdateTime: unixTimeMs("2019-10-10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL - {UpdateTime: unixTimeMs("2019-10-10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // table_id = 131, index_id = 1, index_value = 1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: 0, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, From 6ba28be34a235d09b97eebe9fdcecd4f88d6bc80 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Tue, 24 Aug 2021 08:57:47 +0000 Subject: [PATCH 13/29] executor: use http.MethodGet Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 7ba6662cc9385..0e0f9f2b1367f 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -866,7 +866,7 @@ func (e *hotRegionsHistoryRetriver) startRetrieving( go func(address string, body *bytes.Buffer) { util.WithRecovery(func() { url := fmt.Sprintf("%s://%s%s", util.InternalHTTPSchema(), address, pdapi.HotHistory) - req, err := http.NewRequest(http.MethodPost, url, body) + req, err := http.NewRequest(http.MethodGet, url, body) if err != nil { ch <- hotRegionsStreamResult{err: errors.Trace(err)} return From 800879818c895b6b55334a34828bf868f8d55230 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Wed, 25 Aug 2021 03:22:01 +0000 Subject: [PATCH 14/29] executor: remove extraction unnecessary columns Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 120 +++++++++++++----------------------- 1 file changed, 44 insertions(+), 76 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 0e0f9f2b1367f..7935639390689 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -758,14 +758,6 @@ type HistoryHotRegionsRequest struct { PeerIDs []uint64 `json:"peer_ids,omitempty"` Roles []uint64 `json:"roles,omitempty"` HotRegionTypes []string `json:"hot_region_types,omitempty"` - LowHotDegree int64 `json:"low_hot_degree,omitempty"` - HighHotDegree int64 `json:"high_hot_degree,omitempty"` - LowFlowBytes float64 `json:"low_flow_bytes,omitempty"` - HighFlowBytes float64 `json:"high_flow_bytes,omitempty"` - LowKeyRate float64 `json:"low_key_rate,omitempty"` - HighKeyRate float64 `json:"high_key_rate,omitempty"` - LowQueryRate float64 `json:"low_query_rate,omitempty"` - HighQueryRate float64 `json:"high_query_rate,omitempty"` } // HistoryHotRegions records filtered hot regions stored in each PD. @@ -827,20 +819,12 @@ func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx session } // set hotType before request historyHotRegionsRequest := &HistoryHotRegionsRequest{ - StartTime: e.extractor.StartTime, - EndTime: e.extractor.EndTime, - RegionIDs: e.extractor.RegionIDs, - StoreIDs: e.extractor.StoreIDs, - PeerIDs: e.extractor.PeerIDs, - Roles: e.extractor.Roles, - LowHotDegree: e.extractor.LowHotDegree, - HighHotDegree: e.extractor.HighHotDegree, - LowFlowBytes: e.extractor.LowFlowBytes, - HighFlowBytes: e.extractor.HighFlowBytes, - LowKeyRate: e.extractor.LowKeyRate, - HighKeyRate: e.extractor.HighKeyRate, - LowQueryRate: e.extractor.LowQueryRate, - HighQueryRate: e.extractor.HighQueryRate, + StartTime: e.extractor.StartTime, + EndTime: e.extractor.EndTime, + RegionIDs: e.extractor.RegionIDs, + StoreIDs: e.extractor.StoreIDs, + PeerIDs: e.extractor.PeerIDs, + Roles: e.extractor.Roles, } return e.startRetrieving(ctx, sctx, pdServers, historyHotRegionsRequest) @@ -923,11 +907,21 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct } heap.Init(e.heap) } - // Filter results by db_name, table_name, index_name, table_id, index_id and merge the results. + // Merge the results var finalRows [][]types.Datum + allSchemas := sctx.GetInfoSchema().(infoschema.InfoSchema).AllSchemas() + tikvStore, ok := sctx.GetStore().(helper.Storage) + tz := sctx.GetSessionVars().Location() + if !ok { + return nil, errors.New("Information about hot region can be gotten only when the storage is TiKV") + } + tikvHelper := &helper.Helper{ + Store: tikvStore, + RegionCache: tikvStore.GetRegionCache(), + } for e.heap.Len() > 0 && len(finalRows) < hotRegionsHistoryBatchSize { minTimeItem := heap.Pop(e.heap).(hotRegionsStreamResult) - row, err := e.parseAndFilterBySchemaInfo(sctx, minTimeItem.messages.HistoryHotRegion[0]) + row, err := e.getHotRegionRowWithSchemaInfo(minTimeItem.messages.HistoryHotRegion[0], tikvHelper, allSchemas, tz) if err != nil { return nil, err } @@ -945,54 +939,28 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct return finalRows, nil } -func (e *hotRegionsHistoryRetriver) filterBySchemaInfo(f *helper.FrameItem) bool { - // Ignore this row can't find responding schema f. - if f == nil { - return false - } - if e.extractor.DBNames.Count() != 0 && !e.extractor.DBNames.Exist(f.DBName) { - return false - } - if e.extractor.TableNames.Count() != 0 && !e.extractor.TableNames.Exist(f.TableName) { - return false - } - if e.extractor.IndexNames.Count() != 0 && !e.extractor.IndexNames.Exist(f.IndexName) { - return false - } - if e.extractor.TableIDs.Count() != 0 && !e.extractor.TableIDs.Exist(f.TableID) { - return false - } - if e.extractor.IndexIDs.Count() != 0 && !e.extractor.IndexIDs.Exist(f.IndexID) { - return false - } - return true -} - -func (e *hotRegionsHistoryRetriver) parseAndFilterBySchemaInfo(sctx sessionctx.Context, headMessage *HistoryHotRegion) ([]types.Datum, error) { - _, startKey, _ := codec.DecodeBytes(headMessage.StartKey, []byte{}) - _, endKey, _ := codec.DecodeBytes(headMessage.EndKey, []byte{}) +func (e *hotRegionsHistoryRetriver) getHotRegionRowWithSchemaInfo( + hisHotRegion *HistoryHotRegion, + tikvHelper *helper.Helper, + allSchemas []*model.DBInfo, + tz *time.Location, +) ([]types.Datum, error) { + _, startKey, _ := codec.DecodeBytes(hisHotRegion.StartKey, []byte{}) + _, endKey, _ := codec.DecodeBytes(hisHotRegion.EndKey, []byte{}) region := &tikv.KeyLocation{StartKey: startKey, EndKey: endKey} hotRange, err := helper.NewRegionFrameRange(region) if err != nil { return nil, err } - allSchemas := sctx.GetInfoSchema().(infoschema.InfoSchema).AllSchemas() - tikvStore, ok := sctx.GetStore().(helper.Storage) - if !ok { - return nil, errors.New("Information about hot region can be gotten only when the storage is TiKV") - } - tikvHelper := &helper.Helper{ - Store: tikvStore, - RegionCache: tikvStore.GetRegionCache(), - } + f := tikvHelper.FindTableIndexOfRegion(allSchemas, hotRange) - // Keep this row or not - if !e.filterBySchemaInfo(f) { + // Ignore row without coresponding schema f. + if f == nil { return nil, nil } row := make([]types.Datum, len(infoschema.TableTiDBHotRegionsHistoryCols)) - updateTimestamp := time.Unix(headMessage.UpdateTime/1000, (headMessage.UpdateTime%1000)*int64(time.Millisecond)) - tz := sctx.GetSessionVars().Location() + updateTimestamp := time.Unix(hisHotRegion.UpdateTime/1000, (hisHotRegion.UpdateTime%1000)*int64(time.Millisecond)) + if updateTimestamp.Location() != tz { updateTimestamp.In(tz) } @@ -1008,33 +976,33 @@ func (e *hotRegionsHistoryRetriver) parseAndFilterBySchemaInfo(sctx sessionctx.C row[4].SetNull() row[5].SetNull() } - row[6].SetInt64(int64(headMessage.RegionID)) - row[7].SetInt64(int64(headMessage.StoreID)) - row[8].SetInt64(int64(headMessage.PeerID)) - if headMessage.IsLeader { + row[6].SetInt64(int64(hisHotRegion.RegionID)) + row[7].SetInt64(int64(hisHotRegion.StoreID)) + row[8].SetInt64(int64(hisHotRegion.PeerID)) + if hisHotRegion.IsLeader { row[9].SetInt64(1) } else { row[9].SetInt64(0) } - row[10].SetString(strings.ToUpper(headMessage.HotRegionType), mysql.DefaultCollationName) - if headMessage.HotDegree != 0 { - row[11].SetInt64(headMessage.HotDegree) + row[10].SetString(strings.ToUpper(hisHotRegion.HotRegionType), mysql.DefaultCollationName) + if hisHotRegion.HotDegree != 0 { + row[11].SetInt64(hisHotRegion.HotDegree) } else { row[11].SetNull() } - if headMessage.FlowBytes != 0 { - row[12].SetFloat64(float64(headMessage.FlowBytes)) + if hisHotRegion.FlowBytes != 0 { + row[12].SetFloat64(float64(hisHotRegion.FlowBytes)) } else { row[12].SetNull() } - if headMessage.KeyRate != 0 { - row[13].SetFloat64(float64(headMessage.KeyRate)) + if hisHotRegion.KeyRate != 0 { + row[13].SetFloat64(float64(hisHotRegion.KeyRate)) } else { row[13].SetNull() } - if headMessage.QueryRate != 0 { - row[14].SetFloat64(float64(headMessage.QueryRate)) + if hisHotRegion.QueryRate != 0 { + row[14].SetFloat64(float64(hisHotRegion.QueryRate)) } else { row[14].SetNull() } From 95093e923af10df2f2a22a1f99b40cf1a9918c43 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Wed, 25 Aug 2021 05:13:12 +0000 Subject: [PATCH 15/29] executor: review from @rleungx Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 46 +++++++++++++++++--------------- executor/memtable_reader_test.go | 12 ++++----- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 7935639390689..1ed0537b080cc 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -705,13 +705,13 @@ func (e *clusterLogRetriever) getRuntimeStats() execdetails.RuntimeStats { return nil } -type hotRegionsStreamResult struct { +type hotRegionsResult struct { addr string messages *HistoryHotRegions err error } -type hotRegionsResponseHeap []hotRegionsStreamResult +type hotRegionsResponseHeap []hotRegionsResult func (h hotRegionsResponseHeap) Len() int { return len(h) @@ -730,7 +730,7 @@ func (h hotRegionsResponseHeap) Swap(i, j int) { } func (h *hotRegionsResponseHeap) Push(x interface{}) { - *h = append(*h, x.(hotRegionsStreamResult)) + *h = append(*h, x.(hotRegionsResult)) } func (h *hotRegionsResponseHeap) Pop() interface{} { @@ -784,13 +784,13 @@ type HistoryHotRegion struct { } const ( - // HotRegionTypeREAD hot read region. - HotRegionTypeREAD = "READ" - // HotRegionTypeWRITE hot write region. - HotRegionTypeWRITE = "WRITE" + // HotRegionTypeRead hot read region. + HotRegionTypeRead = "READ" + // HotRegionTypeWrite hot write region. + HotRegionTypeWrite = "WRITE" ) -func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx sessionctx.Context) ([]chan hotRegionsStreamResult, error) { +func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx sessionctx.Context) ([]chan hotRegionsResult, error) { if !hasPriv(sctx, mysql.ProcessPriv) { return nil, plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS") } @@ -810,8 +810,8 @@ func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx session // Divide read write into two request because of time range ovelap, // because PD use [type,time] as key of hot regions. if e.extractor.HotRegionTypes.Count() == 0 { - e.extractor.HotRegionTypes.Insert(HotRegionTypeREAD) - e.extractor.HotRegionTypes.Insert(HotRegionTypeWRITE) + e.extractor.HotRegionTypes.Insert(HotRegionTypeRead) + e.extractor.HotRegionTypes.Insert(HotRegionTypeWrite) } hotRegionTypes := make([]string, 0, e.extractor.HotRegionTypes.Count()) for typ := range e.extractor.HotRegionTypes { @@ -835,8 +835,8 @@ func (e *hotRegionsHistoryRetriver) startRetrieving( sctx sessionctx.Context, serversInfo []infoschema.ServerInfo, req *HistoryHotRegionsRequest, -) ([]chan hotRegionsStreamResult, error) { - var results []chan hotRegionsStreamResult +) ([]chan hotRegionsResult, error) { + var results []chan hotRegionsResult for _, srv := range serversInfo { for typ := range e.extractor.HotRegionTypes { req.HotRegionTypes = []string{typ} @@ -845,37 +845,39 @@ func (e *hotRegionsHistoryRetriver) startRetrieving( return nil, err } body := bytes.NewBuffer(jsonBody) - ch := make(chan hotRegionsStreamResult) + ch := make(chan hotRegionsResult) results = append(results, ch) - go func(address string, body *bytes.Buffer) { + go func(ch chan hotRegionsResult, address string, body *bytes.Buffer) { util.WithRecovery(func() { + defer close(ch) + url := fmt.Sprintf("%s://%s%s", util.InternalHTTPSchema(), address, pdapi.HotHistory) req, err := http.NewRequest(http.MethodGet, url, body) if err != nil { - ch <- hotRegionsStreamResult{err: errors.Trace(err)} + ch <- hotRegionsResult{err: errors.Trace(err)} return } req.Header.Add("PD-Allow-follower-handle", "true") resp, err := util.InternalHTTPClient().Do(req) if err != nil { - ch <- hotRegionsStreamResult{err: errors.Trace(err)} + ch <- hotRegionsResult{err: errors.Trace(err)} return } defer func() { terror.Log(resp.Body.Close()) }() if resp.StatusCode != http.StatusOK { - ch <- hotRegionsStreamResult{err: errors.Errorf("request %s failed: %s", url, resp.Status)} + ch <- hotRegionsResult{err: errors.Errorf("request %s failed: %s", url, resp.Status)} return } var historyHotRegions HistoryHotRegions if err = json.NewDecoder(resp.Body).Decode(&historyHotRegions); err != nil { - ch <- hotRegionsStreamResult{err: errors.Trace(err)} + ch <- hotRegionsResult{err: errors.Trace(err)} return } - ch <- hotRegionsStreamResult{addr: address, messages: &historyHotRegions} + ch <- hotRegionsResult{addr: address, messages: &historyHotRegions} }, nil) - }(srv.StatusAddr, body) + }(ch, srv.StatusAddr, body) } } return results, nil @@ -920,7 +922,7 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct RegionCache: tikvStore.GetRegionCache(), } for e.heap.Len() > 0 && len(finalRows) < hotRegionsHistoryBatchSize { - minTimeItem := heap.Pop(e.heap).(hotRegionsStreamResult) + minTimeItem := heap.Pop(e.heap).(hotRegionsResult) row, err := e.getHotRegionRowWithSchemaInfo(minTimeItem.messages.HistoryHotRegion[0], tikvHelper, allSchemas, tz) if err != nil { return nil, err @@ -954,7 +956,7 @@ func (e *hotRegionsHistoryRetriver) getHotRegionRowWithSchemaInfo( } f := tikvHelper.FindTableIndexOfRegion(allSchemas, hotRange) - // Ignore row without coresponding schema f. + // Ignore row without corresponding schema f. if f == nil { return nil, nil } diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index e5e72fe17dc4b..fac6891fd3c1c 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -1087,7 +1087,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { pdResps := []map[string]*executor.HistoryHotRegions{ { - executor.HotRegionTypeREAD: { + executor.HotRegionTypeRead: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV {UpdateTime: unixTimeMs("2019-10-10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, @@ -1099,7 +1099,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, }, - executor.HotRegionTypeWRITE: { + executor.HotRegionTypeWrite: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV {UpdateTime: unixTimeMs("2019-10-10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, @@ -1113,7 +1113,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { }, }, { - executor.HotRegionTypeREAD: { + executor.HotRegionTypeRead: { HistoryHotRegion: []*executor.HistoryHotRegion{ // table_id = 131, record_id=1, deleted schema {UpdateTime: unixTimeMs("2019-10-10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, @@ -1125,7 +1125,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, }, - executor.HotRegionTypeWRITE: { + executor.HotRegionTypeWrite: { HistoryHotRegion: []*executor.HistoryHotRegion{ // table_id = 131, record_id=1, deleted schema {UpdateTime: unixTimeMs("2019-10-10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, @@ -1139,7 +1139,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { }, }, { - executor.HotRegionTypeREAD: { + executor.HotRegionTypeRead: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER {UpdateTime: unixTimeMs("2019-10-10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, @@ -1155,7 +1155,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, }, - executor.HotRegionTypeWRITE: { + executor.HotRegionTypeWrite: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER {UpdateTime: unixTimeMs("2019-10-10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, From cb2cbedc690687f1b6965055810d4e17443e2419 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Wed, 25 Aug 2021 12:58:52 +0000 Subject: [PATCH 16/29] exectuor:change PD-Allow-follower-handle to false Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 1ed0537b080cc..2404b353943aa 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -857,7 +857,7 @@ func (e *hotRegionsHistoryRetriver) startRetrieving( ch <- hotRegionsResult{err: errors.Trace(err)} return } - req.Header.Add("PD-Allow-follower-handle", "true") + req.Header.Add("PD-Allow-follower-handle", "false") resp, err := util.InternalHTTPClient().Do(req) if err != nil { ch <- hotRegionsResult{err: errors.Trace(err)} From 0490b497ea7ec0ffa3caffb99e2a5e7a2540de1d Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Sat, 28 Aug 2021 03:52:28 +0000 Subject: [PATCH 17/29] executor: add is_learner field Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 69 +++++++++++++++++++++++--------- executor/memtable_reader_test.go | 54 ++++++++++++------------- 2 files changed, 78 insertions(+), 45 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 2404b353943aa..a3d89c47f40eb 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -756,8 +756,9 @@ type HistoryHotRegionsRequest struct { RegionIDs []uint64 `json:"region_ids,omitempty"` StoreIDs []uint64 `json:"store_ids,omitempty"` PeerIDs []uint64 `json:"peer_ids,omitempty"` - Roles []uint64 `json:"roles,omitempty"` - HotRegionTypes []string `json:"hot_region_types,omitempty"` + IsLearners []bool `json:"is_learners,omitempty"` + IsLeaders []bool `json:"is_leaders,omitempty"` + HotRegionTypes []string `json:"hot_region_type,omitempty"` } // HistoryHotRegions records filtered hot regions stored in each PD. @@ -773,6 +774,7 @@ type HistoryHotRegion struct { RegionID uint64 `json:"region_id,omitempty"` StoreID uint64 `json:"store_id,omitempty"` PeerID uint64 `json:"peer_id,omitempty"` + IsLearner bool `json:"is_learner,omitempty"` IsLeader bool `json:"is_leader,omitempty"` HotRegionType string `json:"hot_region_type,omitempty"` HotDegree int64 `json:"hot_degree,omitempty"` @@ -817,14 +819,40 @@ func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx session for typ := range e.extractor.HotRegionTypes { hotRegionTypes = append(hotRegionTypes, typ) } + + // Request all if no conditions + if len(e.extractor.IsLearners) == 0 { + e.extractor.IsLearners = []uint64{0, 1} + } + if len(e.extractor.IsLeaders) == 0 { + e.extractor.IsLeaders = []uint64{0, 1} + } + // Change uint to ture false slice, + var isLearners, isLeaders []bool + for _, l := range e.extractor.IsLearners { + if l == 1 { + isLearners = append(isLearners, true) + } else if l == 0 { + isLearners = append(isLearners, false) + } + } + for _, l := range e.extractor.IsLeaders { + if l == 1 { + isLeaders = append(isLeaders, true) + } else if l == 0 { + isLeaders = append(isLeaders, false) + } + } + // set hotType before request historyHotRegionsRequest := &HistoryHotRegionsRequest{ - StartTime: e.extractor.StartTime, - EndTime: e.extractor.EndTime, - RegionIDs: e.extractor.RegionIDs, - StoreIDs: e.extractor.StoreIDs, - PeerIDs: e.extractor.PeerIDs, - Roles: e.extractor.Roles, + StartTime: e.extractor.StartTime, + EndTime: e.extractor.EndTime, + RegionIDs: e.extractor.RegionIDs, + StoreIDs: e.extractor.StoreIDs, + PeerIDs: e.extractor.PeerIDs, + IsLearners: isLearners, + IsLeaders: isLeaders, } return e.startRetrieving(ctx, sctx, pdServers, historyHotRegionsRequest) @@ -981,32 +1009,37 @@ func (e *hotRegionsHistoryRetriver) getHotRegionRowWithSchemaInfo( row[6].SetInt64(int64(hisHotRegion.RegionID)) row[7].SetInt64(int64(hisHotRegion.StoreID)) row[8].SetInt64(int64(hisHotRegion.PeerID)) - if hisHotRegion.IsLeader { + if hisHotRegion.IsLearner { row[9].SetInt64(1) } else { row[9].SetInt64(0) } + if hisHotRegion.IsLeader { + row[10].SetInt64(1) + } else { + row[10].SetInt64(0) + } - row[10].SetString(strings.ToUpper(hisHotRegion.HotRegionType), mysql.DefaultCollationName) + row[11].SetString(strings.ToUpper(hisHotRegion.HotRegionType), mysql.DefaultCollationName) if hisHotRegion.HotDegree != 0 { - row[11].SetInt64(hisHotRegion.HotDegree) + row[12].SetInt64(hisHotRegion.HotDegree) } else { - row[11].SetNull() + row[12].SetNull() } if hisHotRegion.FlowBytes != 0 { - row[12].SetFloat64(float64(hisHotRegion.FlowBytes)) + row[13].SetFloat64(float64(hisHotRegion.FlowBytes)) } else { - row[12].SetNull() + row[13].SetNull() } if hisHotRegion.KeyRate != 0 { - row[13].SetFloat64(float64(hisHotRegion.KeyRate)) + row[14].SetFloat64(float64(hisHotRegion.KeyRate)) } else { - row[13].SetNull() + row[14].SetNull() } if hisHotRegion.QueryRate != 0 { - row[14].SetFloat64(float64(hisHotRegion.QueryRate)) + row[15].SetFloat64(float64(hisHotRegion.QueryRate)) } else { - row[14].SetNull() + row[15].SetNull() } return row, nil } diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index fac6891fd3c1c..01dc20c611ec8 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -1061,26 +1061,26 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { } fullHotRegions := [][]string{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV - {"2019-10-10 10:10:11", "MYSQL", "TABLES_PRIV", "11", "", "", "1", "1", "11111", "1", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:12", "MYSQL", "TABLES_PRIV", "11", "", "", "2", "2", "22222", "0", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:11", "MYSQL", "TABLES_PRIV", "11", "", "", "1", "1", "11111", "0", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:12", "MYSQL", "TABLES_PRIV", "11", "", "", "2", "2", "22222", "0", "0", "WRITE", "99", "99", "99", "99"}, // mysql table_id = 21, record_id = 1, table_name = STATS_META - {"2019-10-10 10:10:13", "MYSQL", "STATS_META", "21", "", "", "3", "3", "33333", "1", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:14", "MYSQL", "STATS_META", "21", "", "", "4", "4", "44444", "0", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:13", "MYSQL", "STATS_META", "21", "", "", "3", "3", "33333", "0", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:14", "MYSQL", "STATS_META", "21", "", "", "4", "4", "44444", "0", "0", "WRITE", "99", "99", "99", "99"}, // table_id = 131, record_id=1, deleted schema - {"2019-10-10 10:10:15", "UNKONW", "UNKONW", "131", "UNKONW", "", "5", "5", "55555", "1", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:16", "UNKONW", "UNKONW", "131", "UNKONW", "", "6", "6", "66666", "0", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:15", "UNKONW", "UNKONW", "131", "UNKONW", "", "5", "5", "55555", "0", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:16", "UNKONW", "UNKONW", "131", "UNKONW", "", "6", "6", "66666", "0", "0", "WRITE", "99", "99", "99", "99"}, // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY - {"2019-10-10 10:10:17", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "1", "1", "11111", "1", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:18", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "2", "2", "22222", "0", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:17", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "1", "1", "11111", "0", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:18", "MYSQL", "TABLES_PRIV", "11", "PRIMARY", "1", "2", "2", "22222", "0", "0", "WRITE", "99", "99", "99", "99"}, // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER - {"2019-10-10 10:10:19", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "3", "3", "33333", "1", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:20", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "4", "4", "44444", "0", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:19", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "3", "3", "33333", "0", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:20", "MYSQL", "STATS_META", "21", "IDX_VER", "1", "4", "4", "44444", "0", "0", "WRITE", "99", "99", "99", "99"}, // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL - {"2019-10-10 10:10:21", "MYSQL", "STATS_META", "21", "TBL", "2", "5", "5", "55555", "1", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:22", "MYSQL", "STATS_META", "21", "TBL", "2", "6", "6", "66666", "0", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:21", "MYSQL", "STATS_META", "21", "TBL", "2", "5", "5", "55555", "0", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:22", "MYSQL", "STATS_META", "21", "TBL", "2", "6", "6", "66666", "0", "0", "WRITE", "99", "99", "99", "99"}, // table_id = 131, index_id = 1, index_value = 1, deleted schema - {"2019-10-10 10:10:23", "UNKONW", "UNKONW", "131", "UNKONW", "1", "5", "5", "55555", "1", "READ", "99", "99", "99", "99"}, - {"2019-10-10 10:10:24", "UNKONW", "UNKONW", "131", "UNKONW", "1", "6", "6", "66666", "0", "WRITE", "99", "99", "99", "99"}, + {"2019-10-10 10:10:23", "UNKONW", "UNKONW", "131", "UNKONW", "1", "5", "5", "55555", "0", "1", "READ", "99", "99", "99", "99"}, + {"2019-10-10 10:10:24", "UNKONW", "UNKONW", "131", "UNKONW", "1", "6", "6", "66666", "0", "0", "WRITE", "99", "99", "99", "99"}, } c.Assert(fullHotRegions, NotNil) @@ -1090,11 +1090,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeRead: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV - {UpdateTime: unixTimeMs("2019-10-10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLearner: false, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21, record_id = 1, table_name = STATS_META - {UpdateTime: unixTimeMs("2019-10-10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:13"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLearner: false, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1102,11 +1102,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeWrite: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV - {UpdateTime: unixTimeMs("2019-10-10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLearner: false, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21, record_id = 1, table_name = STATS_META - {UpdateTime: unixTimeMs("2019-10-10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:14"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLearner: false, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1116,11 +1116,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeRead: { HistoryHotRegion: []*executor.HistoryHotRegion{ // table_id = 131, record_id=1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLearner: false, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY - {UpdateTime: unixTimeMs("2019-10-10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:17"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLearner: false, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1128,11 +1128,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeWrite: { HistoryHotRegion: []*executor.HistoryHotRegion{ // table_id = 131, record_id=1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLearner: false, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 11, index_id = 1, index_value = 1, table_name = TABLES_PRIV, index_name = PRIMARY - {UpdateTime: unixTimeMs("2019-10-10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:18"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLearner: false, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, @@ -1142,11 +1142,11 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeRead: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER - {UpdateTime: unixTimeMs("2019-10-10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLearner: false, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL - {UpdateTime: unixTimeMs("2019-10-10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:21"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLearner: false, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // table_id = 131, index_id = 1, index_value = 1, deleted schema @@ -1158,15 +1158,15 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { executor.HotRegionTypeWrite: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER - {UpdateTime: unixTimeMs("2019-10-10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLearner: false, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // mysql table_id = 21 ,index_id = 2, index_value = 1, table_name = STATS_META, index_name = TBL - {UpdateTime: unixTimeMs("2019-10-10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:22"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLearner: false, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, // table_id = 131, index_id = 1, index_value = 1, deleted schema - {UpdateTime: unixTimeMs("2019-10-10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, + {UpdateTime: unixTimeMs("2019-10-10 10:10:24"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLearner: false, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, StartKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}, EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, From faf03b0caf2bab9d3c945aad26f1aa218c5a616c Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Wed, 22 Sep 2021 12:11:08 +0000 Subject: [PATCH 18/29] executor: formate retriver Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index a3d89c47f40eb..9828142bcc0fd 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -940,8 +940,8 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct // Merge the results var finalRows [][]types.Datum allSchemas := sctx.GetInfoSchema().(infoschema.InfoSchema).AllSchemas() - tikvStore, ok := sctx.GetStore().(helper.Storage) tz := sctx.GetSessionVars().Location() + tikvStore, ok := sctx.GetStore().(helper.Storage) if !ok { return nil, errors.New("Information about hot region can be gotten only when the storage is TiKV") } From df7af2df93b06c57d1768a9ff1b82853945270f2 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Wed, 22 Sep 2021 12:19:06 +0000 Subject: [PATCH 19/29] executor: add init of cancel Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 9828142bcc0fd..44ea7d39adf3c 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -864,6 +864,9 @@ func (e *hotRegionsHistoryRetriver) startRetrieving( serversInfo []infoschema.ServerInfo, req *HistoryHotRegionsRequest, ) ([]chan hotRegionsResult, error) { + // The retrieve progress may be abort + ctx, e.cancel = context.WithCancel(ctx) + var results []chan hotRegionsResult for _, srv := range serversInfo { for typ := range e.extractor.HotRegionTypes { From 27f8e4b6fcaca1267ad80e675af064b4033402ae Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Thu, 23 Sep 2021 09:20:48 +0000 Subject: [PATCH 20/29] executor: formate Signed-off-by: IcePigZDB --- executor/memtable_reader_test.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index 01dc20c611ec8..7180d21012122 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -949,14 +949,6 @@ func (s *testMemTableReaderSuite) TestTiDBClusterLogError(c *C) { c.Assert(rs.Close(), IsNil) } -var _ = SerialSuites(&testHotRegionsHistoryTableSuite{testInfoschemaTableSuiteBase: &testInfoschemaTableSuiteBase{}}) - -type testHotRegionsHistoryTableSuite struct { - *testInfoschemaTableSuiteBase - httpServers []*httptest.Server - startTime time.Time -} - type mockStoreWithMultiPD struct { helper.Storage hosts []string @@ -970,6 +962,14 @@ func (s *mockStoreWithMultiPD) StartGCWorker() error { panic("not implem func (s *mockStoreWithMultiPD) Name() string { return "mockStore" } func (s *mockStoreWithMultiPD) Describe() string { return "" } +var _ = SerialSuites(&testHotRegionsHistoryTableSuite{testInfoschemaTableSuiteBase: &testInfoschemaTableSuiteBase{}}) + +type testHotRegionsHistoryTableSuite struct { + *testInfoschemaTableSuiteBase + httpServers []*httptest.Server + startTime time.Time +} + func (s *testHotRegionsHistoryTableSuite) SetUpSuite(c *C) { s.testInfoschemaTableSuiteBase.SetUpSuite(c) store := &mockStoreWithMultiPD{ @@ -1083,8 +1083,6 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { {"2019-10-10 10:10:24", "UNKONW", "UNKONW", "131", "UNKONW", "1", "6", "6", "66666", "0", "0", "WRITE", "99", "99", "99", "99"}, } - c.Assert(fullHotRegions, NotNil) - pdResps := []map[string]*executor.HistoryHotRegions{ { executor.HotRegionTypeRead: { @@ -1339,10 +1337,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { expected: [][]string{}, }, } - // failpoint - fpName := "github.com/pingcap/tidb/executor/mockTiDBHotRegionsHistory" - c.Assert(failpoint.Enable(fpName, `return("")`), IsNil) - defer func() { c.Assert(failpoint.Disable(fpName), IsNil) }() + // mock http resp store := s.store.(*mockStoreWithMultiPD) for i, resp := range pdResps { From f8b7416bcb0156c3463dd8319b6e78485b71116c Mon Sep 17 00:00:00 2001 From: db <39407623+IcePigZDB@users.noreply.github.com> Date: Fri, 8 Oct 2021 11:11:51 +0800 Subject: [PATCH 21/29] Update executor/memtable_reader.go Co-authored-by: ShuNing --- executor/memtable_reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 44ea7d39adf3c..1e8302c588a7e 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -809,7 +809,7 @@ func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx session return nil, errors.New("denied to scan hot regions, please specified the end time, such as `update_time < '2020-01-01 00:00:00'`") } - // Divide read write into two request because of time range ovelap, + // Divide read-write into two requests because of time range overlap, // because PD use [type,time] as key of hot regions. if e.extractor.HotRegionTypes.Count() == 0 { e.extractor.HotRegionTypes.Insert(HotRegionTypeRead) From bbc6ef8b8517f716971494673644ac86dd9e3f2c Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Fri, 8 Oct 2021 03:34:15 +0000 Subject: [PATCH 22/29] executor: simplfy code and move channel close to retrive func Signed-off-by: IcePigZDB Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 34 ++++++---------------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 1e8302c588a7e..464d423737bd9 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -742,11 +742,11 @@ func (h *hotRegionsResponseHeap) Pop() interface{} { } type hotRegionsHistoryRetriver struct { + dummyCloser isDrained bool retrieving bool heap *hotRegionsResponseHeap extractor *plannercore.HotRegionsHistoryTableExtractor - cancel context.CancelFunc } // HistoryHotRegionsRequest wrap conditions push down to PD. @@ -830,18 +830,10 @@ func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx session // Change uint to ture false slice, var isLearners, isLeaders []bool for _, l := range e.extractor.IsLearners { - if l == 1 { - isLearners = append(isLearners, true) - } else if l == 0 { - isLearners = append(isLearners, false) - } + isLearners = append(isLearners, l == 1) } for _, l := range e.extractor.IsLeaders { - if l == 1 { - isLeaders = append(isLeaders, true) - } else if l == 0 { - isLeaders = append(isLeaders, false) - } + isLeaders = append(isLeaders, l == 1) } // set hotType before request @@ -861,14 +853,12 @@ func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx session func (e *hotRegionsHistoryRetriver) startRetrieving( ctx context.Context, sctx sessionctx.Context, - serversInfo []infoschema.ServerInfo, + pdServers []infoschema.ServerInfo, req *HistoryHotRegionsRequest, ) ([]chan hotRegionsResult, error) { - // The retrieve progress may be abort - ctx, e.cancel = context.WithCancel(ctx) var results []chan hotRegionsResult - for _, srv := range serversInfo { + for _, srv := range pdServers { for typ := range e.extractor.HotRegionTypes { req.HotRegionTypes = []string{typ} jsonBody, err := json.Marshal(req) @@ -880,8 +870,6 @@ func (e *hotRegionsHistoryRetriver) startRetrieving( results = append(results, ch) go func(ch chan hotRegionsResult, address string, body *bytes.Buffer) { util.WithRecovery(func() { - defer close(ch) - url := fmt.Sprintf("%s://%s%s", util.InternalHTTPSchema(), address, pdapi.HotHistory) req, err := http.NewRequest(http.MethodGet, url, body) if err != nil { @@ -929,6 +917,7 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct // Initialize the heap e.heap = &hotRegionsResponseHeap{} for _, ch := range results { + defer close(ch) result := <-ch if result.err != nil || len(result.messages.HistoryHotRegion) == 0 { if result.err != nil { @@ -1046,14 +1035,3 @@ func (e *hotRegionsHistoryRetriver) getHotRegionRowWithSchemaInfo( } return row, nil } - -func (e *hotRegionsHistoryRetriver) close() error { - if e.cancel != nil { - e.cancel() - } - return nil -} - -func (e *hotRegionsHistoryRetriver) getRuntimeStats() execdetails.RuntimeStats { - return nil -} From d78ae5c8b67393ad011a293f846897f604ed3ef3 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Sun, 10 Oct 2021 13:44:34 +0000 Subject: [PATCH 23/29] executor: move close channel to startRetriving Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 464d423737bd9..84bf7da0090de 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -870,6 +870,7 @@ func (e *hotRegionsHistoryRetriver) startRetrieving( results = append(results, ch) go func(ch chan hotRegionsResult, address string, body *bytes.Buffer) { util.WithRecovery(func() { + defer close(ch) url := fmt.Sprintf("%s://%s%s", util.InternalHTTPSchema(), address, pdapi.HotHistory) req, err := http.NewRequest(http.MethodGet, url, body) if err != nil { @@ -917,7 +918,6 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct // Initialize the heap e.heap = &hotRegionsResponseHeap{} for _, ch := range results { - defer close(ch) result := <-ch if result.err != nil || len(result.messages.HistoryHotRegion) == 0 { if result.err != nil { From 6d6ed522ce18e84951b6547e6a89dbfb9290b92a Mon Sep 17 00:00:00 2001 From: db <39407623+IcePigZDB@users.noreply.github.com> Date: Fri, 15 Oct 2021 20:43:55 +0800 Subject: [PATCH 24/29] Update executor/memtable_reader.go Co-authored-by: ShuNing --- executor/memtable_reader.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 408193932ecbf..2644e6e0b3c95 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -916,12 +916,12 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct e.heap = &hotRegionsResponseHeap{} for _, ch := range results { result := <-ch - if result.err != nil || len(result.messages.HistoryHotRegion) == 0 { - if result.err != nil { + if result.err != nil { sctx.GetSessionVars().StmtCtx.AppendWarning(result.err) } - continue - } + if result.err != nil || len(result.messages.HistoryHotRegion) == 0 { + continue + } *e.heap = append(*e.heap, result) } heap.Init(e.heap) From 1077f702b4df828378815c366b3e7e139d312278 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Sun, 17 Oct 2021 04:31:37 +0000 Subject: [PATCH 25/29] move supplement of HotTypes, IsLearners and IsLeaders to extractor Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 53 +--- executor/memtable_reader_test.go | 17 +- planner/core/memtable_predicate_extractor.go | 37 ++- .../core/memtable_predicate_extractor_test.go | 292 ++++++++++++------ planner/core/util.go | 13 + 5 files changed, 252 insertions(+), 160 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 2644e6e0b3c95..baa1863fe59d9 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -782,13 +782,6 @@ type HistoryHotRegion struct { EndKey []byte `json:"end_key,omitempty"` } -const ( - // HotRegionTypeRead hot read region. - HotRegionTypeRead = "READ" - // HotRegionTypeWrite hot write region. - HotRegionTypeWrite = "WRITE" -) - func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx sessionctx.Context) ([]chan hotRegionsResult, error) { if !hasPriv(sctx, mysql.ProcessPriv) { return nil, plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS") @@ -806,42 +799,14 @@ func (e *hotRegionsHistoryRetriver) initialize(ctx context.Context, sctx session return nil, errors.New("denied to scan hot regions, please specified the end time, such as `update_time < '2020-01-01 00:00:00'`") } - // Divide read-write into two requests because of time range overlap, - // because PD use [type,time] as key of hot regions. - if e.extractor.HotRegionTypes.Count() == 0 { - e.extractor.HotRegionTypes.Insert(HotRegionTypeRead) - e.extractor.HotRegionTypes.Insert(HotRegionTypeWrite) - } - hotRegionTypes := make([]string, 0, e.extractor.HotRegionTypes.Count()) - for typ := range e.extractor.HotRegionTypes { - hotRegionTypes = append(hotRegionTypes, typ) - } - - // Request all if no conditions - if len(e.extractor.IsLearners) == 0 { - e.extractor.IsLearners = []uint64{0, 1} - } - if len(e.extractor.IsLeaders) == 0 { - e.extractor.IsLeaders = []uint64{0, 1} - } - // Change uint to ture false slice, - var isLearners, isLeaders []bool - for _, l := range e.extractor.IsLearners { - isLearners = append(isLearners, l == 1) - } - for _, l := range e.extractor.IsLeaders { - isLeaders = append(isLeaders, l == 1) - } - - // set hotType before request historyHotRegionsRequest := &HistoryHotRegionsRequest{ StartTime: e.extractor.StartTime, EndTime: e.extractor.EndTime, RegionIDs: e.extractor.RegionIDs, StoreIDs: e.extractor.StoreIDs, PeerIDs: e.extractor.PeerIDs, - IsLearners: isLearners, - IsLeaders: isLeaders, + IsLearners: e.extractor.IsLearners, + IsLeaders: e.extractor.IsLeaders, } return e.startRetrieving(ctx, sctx, pdServers, historyHotRegionsRequest) @@ -874,7 +839,7 @@ func (e *hotRegionsHistoryRetriver) startRetrieving( ch <- hotRegionsResult{err: errors.Trace(err)} return } - req.Header.Add("PD-Allow-follower-handle", "false") + req.Header.Add("PD-Allow-follower-handle", "true") resp, err := util.InternalHTTPClient().Do(req) if err != nil { ch <- hotRegionsResult{err: errors.Trace(err)} @@ -916,12 +881,12 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct e.heap = &hotRegionsResponseHeap{} for _, ch := range results { result := <-ch - if result.err != nil { - sctx.GetSessionVars().StmtCtx.AppendWarning(result.err) - } - if result.err != nil || len(result.messages.HistoryHotRegion) == 0 { - continue - } + if result.err != nil { + sctx.GetSessionVars().StmtCtx.AppendWarning(result.err) + } + if result.err != nil || len(result.messages.HistoryHotRegion) == 0 { + continue + } *e.heap = append(*e.heap, result) } heap.Init(e.heap) diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index cd3aef87b43f9..6de671afbc2d4 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -39,6 +39,7 @@ import ( "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/store/helper" "github.com/pingcap/tidb/util/pdapi" @@ -1028,7 +1029,7 @@ func hisHotRegionsHandler(w http.ResponseWriter, r *http.Request) { } resp := &executor.HistoryHotRegions{} for _, typ := range req.HotRegionTypes { - resp.HistoryHotRegion = append(resp.HistoryHotRegion, hotRegionsResponses[strings.ToUpper(typ)+r.Host].HistoryHotRegion...) + resp.HistoryHotRegion = append(resp.HistoryHotRegion, hotRegionsResponses[typ+r.Host].HistoryHotRegion...) } w.WriteHeader(http.StatusOK) jsonResp, err := json.Marshal(resp) @@ -1099,7 +1100,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { pdResps := []map[string]*executor.HistoryHotRegions{ { - executor.HotRegionTypeRead: { + core.HotRegionTypeRead: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV {UpdateTime: unixTimeMs("2019-10-10 10:10:11"), RegionID: 1, StoreID: 1, PeerID: 11111, IsLearner: false, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, @@ -1111,7 +1112,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, }, - executor.HotRegionTypeWrite: { + core.HotRegionTypeWrite: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 11, record_id = 1, table_name = TABLES_PRIV {UpdateTime: unixTimeMs("2019-10-10 10:10:12"), RegionID: 2, StoreID: 2, PeerID: 22222, IsLearner: false, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, @@ -1125,7 +1126,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { }, }, { - executor.HotRegionTypeRead: { + core.HotRegionTypeRead: { HistoryHotRegion: []*executor.HistoryHotRegion{ // table_id = 131, record_id=1, deleted schema {UpdateTime: unixTimeMs("2019-10-10 10:10:15"), RegionID: 5, StoreID: 5, PeerID: 55555, IsLearner: false, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, @@ -1137,7 +1138,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xb, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, }, - executor.HotRegionTypeWrite: { + core.HotRegionTypeWrite: { HistoryHotRegion: []*executor.HistoryHotRegion{ // table_id = 131, record_id=1, deleted schema {UpdateTime: unixTimeMs("2019-10-10 10:10:16"), RegionID: 6, StoreID: 6, PeerID: 66666, IsLearner: false, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, @@ -1151,7 +1152,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { }, }, { - executor.HotRegionTypeRead: { + core.HotRegionTypeRead: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER {UpdateTime: unixTimeMs("2019-10-10 10:10:19"), RegionID: 3, StoreID: 3, PeerID: 33333, IsLearner: false, IsLeader: true, HotRegionType: "READ", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, @@ -1167,7 +1168,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { EndKey: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}}, }, }, - executor.HotRegionTypeWrite: { + core.HotRegionTypeWrite: { HistoryHotRegion: []*executor.HistoryHotRegion{ // mysql table_id = 21 ,index_id = 1, index_value = 1, table_name = STATS_META, index_name = IDX_VER {UpdateTime: unixTimeMs("2019-10-10 10:10:20"), RegionID: 4, StoreID: 4, PeerID: 44444, IsLearner: false, IsLeader: false, HotRegionType: "WRITE", HotDegree: 99, FlowBytes: 99, KeyRate: 99, QueryRate: 99, @@ -1367,7 +1368,7 @@ func (s *testHotRegionsHistoryTableSuite) TestTiDBHotRegionsHistory(c *C) { } result := tk.MustQuery(sql) warnings := tk.Se.GetSessionVars().StmtCtx.GetWarnings() - c.Assert(len(warnings), Equals, 0, Commentf("unexpected warnigns: %+v", warnings)) + c.Assert(len(warnings), Equals, 0, Commentf("unexpected warnigns: %+v, sql: %s", warnings, sql)) var expected []string for _, row := range cas.expected { expectedRow := row diff --git a/planner/core/memtable_predicate_extractor.go b/planner/core/memtable_predicate_extractor.go index 7fd2e787dae11..fe355e4c9cae9 100644 --- a/planner/core/memtable_predicate_extractor.go +++ b/planner/core/memtable_predicate_extractor.go @@ -697,6 +697,13 @@ func (e *ClusterLogTableExtractor) explainInfo(p *PhysicalMemTable) string { return s } +const ( + // HotRegionTypeRead hot read region. + HotRegionTypeRead = "read" + // HotRegionTypeWrite hot write region. + HotRegionTypeWrite = "write" +) + // HotRegionsHistoryTableExtractor is used to extract some predicates of `tidb_hot_regions_history` type HotRegionsHistoryTableExtractor struct { extractHelper @@ -723,8 +730,8 @@ type HotRegionsHistoryTableExtractor struct { // e.g: // 1. SELECT * FROM tidb_hot_regions_history WHERE is_learner=1 // 2. SELECT * FROM tidb_hot_regions_history WHERE is_learner in (0,1) -> request all - IsLearners []uint64 - IsLeaders []uint64 + IsLearners []bool + IsLeaders []bool // HotRegionTypes represents all hot region types we should filter in PD to reduce network IO. // e.g: @@ -754,11 +761,25 @@ func (e *HotRegionsHistoryTableExtractor) Extract( // Extract the is_learner/is_leader columns. remained, isLearnerSkipRequest, isLearners := e.extractCol(schema, names, remained, "is_learner", false) remained, isLeaderSkipRequest, isLeaders := e.extractCol(schema, names, remained, "is_leader", false) - e.IsLearners, e.IsLeaders = e.parseUint64(isLearners), e.parseUint64(isLeaders) + isLearnersUint64, isLeadersUint64 := e.parseUint64(isLearners), e.parseUint64(isLeaders) e.SkipRequest = isLearnerSkipRequest || isLeaderSkipRequest if e.SkipRequest { return nil } + // Request all if no conditions + if len(isLearnersUint64) == 0 { + isLearnersUint64 = []uint64{0, 1} + } + if len(isLeadersUint64) == 0 { + isLeadersUint64 = []uint64{0, 1} + } + // Change uint to ture false slice + for _, l := range isLearnersUint64 { + e.IsLearners = append(e.IsLearners, l == 1) + } + for _, l := range isLeadersUint64 { + e.IsLeaders = append(e.IsLeaders, l == 1) + } // Extract the `type` column. remained, typeSkipRequest, types := e.extractCol(schema, names, remained, "type", false) @@ -767,6 +788,12 @@ func (e *HotRegionsHistoryTableExtractor) Extract( if e.SkipRequest { return nil } + // Divide read-write into two requests because of time range overlap, + // PD use [type,time] as key of hot regions. + if e.HotRegionTypes.Count() == 0 { + e.HotRegionTypes.Insert(HotRegionTypeRead) + e.HotRegionTypes.Insert(HotRegionTypeWrite) + } remained, startTime, endTime := e.extractTimeRange(ctx, schema, names, remained, "update_time", ctx.GetSessionVars().StmtCtx.TimeZone) // The time unit for search hot regions is millisecond. @@ -808,10 +835,10 @@ func (e *HotRegionsHistoryTableExtractor) explainInfo(p *PhysicalMemTable) strin r.WriteString(fmt.Sprintf("peer_ids:[%s], ", extractStringFromUint64Slice(e.PeerIDs))) } if len(e.IsLearners) > 0 { - r.WriteString(fmt.Sprintf("roles:[%s], ", extractStringFromUint64Slice(e.IsLearners))) + r.WriteString(fmt.Sprintf("roles:[%s], ", extractStringFromBoolSlice(e.IsLearners))) } if len(e.IsLeaders) > 0 { - r.WriteString(fmt.Sprintf("roles:[%s], ", extractStringFromUint64Slice(e.IsLeaders))) + r.WriteString(fmt.Sprintf("roles:[%s], ", extractStringFromBoolSlice(e.IsLeaders))) } if len(e.HotRegionTypes) > 0 { r.WriteString(fmt.Sprintf("hot_region_types:[%s], ", extractStringFromStringSet(e.HotRegionTypes))) diff --git a/planner/core/memtable_predicate_extractor_test.go b/planner/core/memtable_predicate_extractor_test.go index 15f468d33a5c3..a24c4a25f05db 100644 --- a/planner/core/memtable_predicate_extractor_test.go +++ b/planner/core/memtable_predicate_extractor_test.go @@ -1058,7 +1058,7 @@ func (s *extractorSuite) TestTiDBHotRegionsHistoryTableExtractor(c *C) { skipRequest bool startTime, endTime int64 regionIDs, storeIDs, peerIDs []uint64 - isLearners, isLeaders []uint64 + isLearners, isLeaders []bool hotRegionTypes set.StringSet }{ // Test full data, it will not call Extract() and executor(retriver) will panic and remind user to add conditions to save network IO. @@ -1071,114 +1071,180 @@ func (s *extractorSuite) TestTiDBHotRegionsHistoryTableExtractor(c *C) { sql: "select * from information_schema.tidb_hot_regions_history where update_time='2019-10-10 10::10'", }, { - sql: "select * from information_schema.tidb_hot_regions_history where update_time='2019-10-10 10:10:10'", - startTime: timestamp(c, "2019-10-10 10:10:10"), - endTime: timestamp(c, "2019-10-10 10:10:10"), + sql: "select * from information_schema.tidb_hot_regions_history where update_time='2019-10-10 10:10:10'", + startTime: timestamp(c, "2019-10-10 10:10:10"), + endTime: timestamp(c, "2019-10-10 10:10:10"), + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10' and update_time<='2019-10-11 10:10:10'", - startTime: timestamp(c, "2019-10-10 10:10:10"), - endTime: timestamp(c, "2019-10-11 10:10:10"), + sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10' and update_time<='2019-10-11 10:10:10'", + startTime: timestamp(c, "2019-10-10 10:10:10"), + endTime: timestamp(c, "2019-10-11 10:10:10"), + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where update_time>'2019-10-10 10:10:10' and update_time<'2019-10-11 10:10:10'", - startTime: timestamp(c, "2019-10-10 10:10:10") + 1, - endTime: timestamp(c, "2019-10-11 10:10:10") - 1, + sql: "select * from information_schema.tidb_hot_regions_history where update_time>'2019-10-10 10:10:10' and update_time<'2019-10-11 10:10:10'", + startTime: timestamp(c, "2019-10-10 10:10:10") + 1, + endTime: timestamp(c, "2019-10-11 10:10:10") - 1, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10' and update_time<'2019-10-11 10:10:10'", - startTime: timestamp(c, "2019-10-10 10:10:10"), - endTime: timestamp(c, "2019-10-11 10:10:10") - 1, + sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10' and update_time<'2019-10-11 10:10:10'", + startTime: timestamp(c, "2019-10-10 10:10:10"), + endTime: timestamp(c, "2019-10-11 10:10:10") - 1, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-12 10:10:10' and update_time<'2019-10-11 10:10:10'", - startTime: timestamp(c, "2019-10-12 10:10:10"), - endTime: timestamp(c, "2019-10-11 10:10:10") - 1, - skipRequest: true, + sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-12 10:10:10' and update_time<'2019-10-11 10:10:10'", + startTime: timestamp(c, "2019-10-12 10:10:10"), + endTime: timestamp(c, "2019-10-11 10:10:10") - 1, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), + skipRequest: true, }, { - sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10'", - startTime: timestamp(c, "2019-10-10 10:10:10"), + sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10'", + startTime: timestamp(c, "2019-10-10 10:10:10"), + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10' and update_time>='2019-10-11 10:10:10' and update_time>='2019-10-12 10:10:10'", - startTime: timestamp(c, "2019-10-12 10:10:10"), + sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10' and update_time>='2019-10-11 10:10:10' and update_time>='2019-10-12 10:10:10'", + startTime: timestamp(c, "2019-10-12 10:10:10"), + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10' and update_time>='2019-10-11 10:10:10' and update_time>='2019-10-12 10:10:10' and update_time='2019-10-13 10:10:10'", - startTime: timestamp(c, "2019-10-13 10:10:10"), - endTime: timestamp(c, "2019-10-13 10:10:10"), + sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10' and update_time>='2019-10-11 10:10:10' and update_time>='2019-10-12 10:10:10' and update_time='2019-10-13 10:10:10'", + startTime: timestamp(c, "2019-10-13 10:10:10"), + endTime: timestamp(c, "2019-10-13 10:10:10"), + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where update_time<='2019-10-10 10:10:10' and update_time='2019-10-13 10:10:10'", - startTime: timestamp(c, "2019-10-13 10:10:10"), - endTime: timestamp(c, "2019-10-10 10:10:10"), - skipRequest: true, + sql: "select * from information_schema.tidb_hot_regions_history where update_time<='2019-10-10 10:10:10' and update_time='2019-10-13 10:10:10'", + startTime: timestamp(c, "2019-10-13 10:10:10"), + endTime: timestamp(c, "2019-10-10 10:10:10"), + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), + skipRequest: true, }, { - sql: "select * from information_schema.tidb_hot_regions_history where update_time='2019-10-10 10:10:10' and update_time<='2019-10-13 10:10:10'", - startTime: timestamp(c, "2019-10-10 10:10:10"), - endTime: timestamp(c, "2019-10-10 10:10:10"), + sql: "select * from information_schema.tidb_hot_regions_history where update_time='2019-10-10 10:10:10' and update_time<='2019-10-13 10:10:10'", + startTime: timestamp(c, "2019-10-10 10:10:10"), + endTime: timestamp(c, "2019-10-10 10:10:10"), + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, // Test `region_id`, `store_id`, `peer_id` columns. { - sql: "select * from information_schema.tidb_hot_regions_history where region_id=100", - regionIDs: []uint64{100}, + sql: "select * from information_schema.tidb_hot_regions_history where region_id=100", + regionIDs: []uint64{100}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where 100=region_id", - regionIDs: []uint64{100}, + sql: "select * from information_schema.tidb_hot_regions_history where 100=region_id", + regionIDs: []uint64{100}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where 100=region_id or region_id=101", - regionIDs: []uint64{100, 101}, + sql: "select * from information_schema.tidb_hot_regions_history where 100=region_id or region_id=101", + regionIDs: []uint64{100, 101}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where 100=region_id or region_id=101 or region_id=102 or 103 = region_id", - regionIDs: []uint64{100, 101, 102, 103}, + sql: "select * from information_schema.tidb_hot_regions_history where 100=region_id or region_id=101 or region_id=102 or 103 = region_id", + regionIDs: []uint64{100, 101, 102, 103}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where (region_id=100 or region_id=101) and (store_id=200 or store_id=201)", - regionIDs: []uint64{100, 101}, - storeIDs: []uint64{200, 201}, + sql: "select * from information_schema.tidb_hot_regions_history where (region_id=100 or region_id=101) and (store_id=200 or store_id=201)", + regionIDs: []uint64{100, 101}, + storeIDs: []uint64{200, 201}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where region_id in (100, 101)", - regionIDs: []uint64{100, 101}, + sql: "select * from information_schema.tidb_hot_regions_history where region_id in (100, 101)", + regionIDs: []uint64{100, 101}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where region_id in (100, 101) and store_id=200", - regionIDs: []uint64{100, 101}, - storeIDs: []uint64{200}, + sql: "select * from information_schema.tidb_hot_regions_history where region_id in (100, 101) and store_id=200", + regionIDs: []uint64{100, 101}, + storeIDs: []uint64{200}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where region_id in (100, 101) and store_id in (200, 201)", - regionIDs: []uint64{100, 101}, - storeIDs: []uint64{200, 201}, + sql: "select * from information_schema.tidb_hot_regions_history where region_id in (100, 101) and store_id in (200, 201)", + regionIDs: []uint64{100, 101}, + storeIDs: []uint64{200, 201}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where region_id=100 and store_id in (200, 201)", - regionIDs: []uint64{100}, - storeIDs: []uint64{200, 201}, + sql: "select * from information_schema.tidb_hot_regions_history where region_id=100 and store_id in (200, 201)", + regionIDs: []uint64{100}, + storeIDs: []uint64{200, 201}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where region_id=100 and store_id=200", - regionIDs: []uint64{100}, - storeIDs: []uint64{200}, + sql: "select * from information_schema.tidb_hot_regions_history where region_id=100 and store_id=200", + regionIDs: []uint64{100}, + storeIDs: []uint64{200}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { sql: "select * from information_schema.tidb_hot_regions_history where region_id=100 and region_id=101", skipRequest: true, }, { - sql: "select * from information_schema.tidb_hot_regions_history where region_id=100 and region_id in (100,101)", - regionIDs: []uint64{100}, + sql: "select * from information_schema.tidb_hot_regions_history where region_id=100 and region_id in (100,101)", + regionIDs: []uint64{100}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where region_id=100 and region_id in (100,101) and store_id=200 and store_id in (200,201)", - regionIDs: []uint64{100}, - storeIDs: []uint64{200}, + sql: "select * from information_schema.tidb_hot_regions_history where region_id=100 and region_id in (100,101) and store_id=200 and store_id in (200,201)", + regionIDs: []uint64{100}, + storeIDs: []uint64{200}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { sql: "select * from information_schema.tidb_hot_regions_history where region_id=100 and region_id in (101,102)", @@ -1193,52 +1259,61 @@ func (s *extractorSuite) TestTiDBHotRegionsHistoryTableExtractor(c *C) { skipRequest: true, }, { - sql: `select * from information_schema.tidb_hot_regions_history - where region_id=100 and region_id in (100,101) + sql: `select * from information_schema.tidb_hot_regions_history + where region_id=100 and region_id in (100,101) and store_id=200 and store_id in (201,202)`, skipRequest: true, }, { - sql: "select * from information_schema.tidb_hot_regions_history where region_id in (100,101) and region_id in (101,102)", - regionIDs: []uint64{101}, + sql: "select * from information_schema.tidb_hot_regions_history where region_id in (100,101) and region_id in (101,102)", + regionIDs: []uint64{101}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: `select * from information_schema.tidb_hot_regions_history - where region_id in (100,101) - and region_id in (101,102) - and store_id in (200,201) + sql: `select * from information_schema.tidb_hot_regions_history + where region_id in (100,101) + and region_id in (101,102) + and store_id in (200,201) and store_id in (201,202)`, - regionIDs: []uint64{101}, - storeIDs: []uint64{201}, + regionIDs: []uint64{101}, + storeIDs: []uint64{201}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: `select * from information_schema.tidb_hot_regions_history - where region_id in (100,101) - and region_id in (101,102) - and store_id in (200,201) + sql: `select * from information_schema.tidb_hot_regions_history + where region_id in (100,101) + and region_id in (101,102) + and store_id in (200,201) and store_id in (201,202) and peer_id in (3000,3001) and peer_id in (3001,3002)`, - regionIDs: []uint64{101}, - storeIDs: []uint64{201}, - peerIDs: []uint64{3001}, - }, - { - sql: `select * from information_schema.tidb_hot_regions_history - where region_id in (100,101) - and region_id in (100,102) - and region_id in (102,103) + regionIDs: []uint64{101}, + storeIDs: []uint64{201}, + peerIDs: []uint64{3001}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), + }, + { + sql: `select * from information_schema.tidb_hot_regions_history + where region_id in (100,101) + and region_id in (100,102) + and region_id in (102,103) and region_id in (103,104)`, skipRequest: true, }, // Test `type` column. { sql: "select * from information_schema.tidb_hot_regions_history where type='read'", - hotRegionTypes: set.NewStringSet("read"), + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead), }, { sql: "select * from information_schema.tidb_hot_regions_history where type in('read')", - hotRegionTypes: set.NewStringSet("read"), + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead), }, { sql: "select * from information_schema.tidb_hot_regions_history where type='read' and type='write'", @@ -1246,11 +1321,11 @@ func (s *extractorSuite) TestTiDBHotRegionsHistoryTableExtractor(c *C) { }, { sql: "select * from information_schema.tidb_hot_regions_history where type in ('read', 'write')", - hotRegionTypes: set.NewStringSet("read", "write"), + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { sql: "select * from information_schema.tidb_hot_regions_history where type='read' and type in ('read', 'write')", - hotRegionTypes: set.NewStringSet("read"), + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead), }, { sql: "select * from information_schema.tidb_hot_regions_history where type in ('read') and type in ('write')", @@ -1258,33 +1333,45 @@ func (s *extractorSuite) TestTiDBHotRegionsHistoryTableExtractor(c *C) { }, // Test `is_learner`, `is_leaeder` columns. { - sql: "select * from information_schema.tidb_hot_regions_history where is_learner=1", - isLearners: []uint64{1}, + sql: "select * from information_schema.tidb_hot_regions_history where is_learner=1", + isLearners: []bool{true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where is_leader=0", - isLeaders: []uint64{0}, + sql: "select * from information_schema.tidb_hot_regions_history where is_leader=0", + isLearners: []bool{false, true}, + isLeaders: []bool{false}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where is_learner=true", - isLearners: []uint64{1}, + sql: "select * from information_schema.tidb_hot_regions_history where is_learner=true", + isLearners: []bool{true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { sql: "select * from information_schema.tidb_hot_regions_history where is_learner in(0,1)", - isLearners: []uint64{0, 1}, + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, }, { - sql: "select * from information_schema.tidb_hot_regions_history where is_learner in(true,false)", - isLearners: []uint64{0, 1}, + sql: "select * from information_schema.tidb_hot_regions_history where is_learner in(true,false)", + isLearners: []bool{false, true}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where is_learner in(3,4)", - isLearners: []uint64{3, 4}, + sql: "select * from information_schema.tidb_hot_regions_history where is_learner in(3,4)", + isLearners: []bool{false, false}, + isLeaders: []bool{false, true}, + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { - sql: "select * from information_schema.tidb_hot_regions_history where is_learner in(3,4) and is_leader in(0,1,true,false,3,4)", - isLearners: []uint64{3, 4}, - isLeaders: []uint64{0, 1, 3, 4}, + sql: "select * from information_schema.tidb_hot_regions_history where is_learner in(3,4) and is_leader in(0,1,true,false,3,4)", + hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), + isLearners: []bool{false, false}, + isLeaders: []bool{false, true, false, false}, }, { sql: "select * from information_schema.tidb_hot_regions_history where is_learner=1 and is_learner=0", @@ -1322,7 +1409,6 @@ func (s *extractorSuite) TestTiDBHotRegionsHistoryTableExtractor(c *C) { if ca.hotRegionTypes.Count() > 0 { c.Assert(hotRegionsHistoryExtractor.HotRegionTypes, DeepEquals, ca.hotRegionTypes, Commentf("SQL: %v", ca.sql)) } - // ues length to avoid case uint64{} != uint64(nil) if len(ca.regionIDs) > 0 { c.Assert(hotRegionsHistoryExtractor.RegionIDs, DeepEquals, ca.regionIDs, Commentf("SQL: %v", ca.sql)) } diff --git a/planner/core/util.go b/planner/core/util.go index 517bb24bf7864..480c4bdfabd0c 100644 --- a/planner/core/util.go +++ b/planner/core/util.go @@ -308,6 +308,19 @@ func extractStringFromUint64Slice(slice []uint64) string { return strings.Join(l, ",") } +// extractStringFromBoolSlice helps extract string info from bool slice. +func extractStringFromBoolSlice(slice []bool) string { + if len(slice) < 1 { + return "" + } + l := make([]string, 0, len(slice)) + for _, k := range slice { + l = append(l, fmt.Sprintf(`%t`, k)) + } + sort.Strings(l) + return strings.Join(l, ",") +} + func tableHasDirtyContent(ctx sessionctx.Context, tableInfo *model.TableInfo) bool { pi := tableInfo.GetPartitionInfo() if pi == nil { From c8d8c11df18ecd0836771e7e2a58a7071bd3cbd6 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Wed, 20 Oct 2021 08:31:52 +0000 Subject: [PATCH 26/29] planner: keep role bool slice unique Signed-off-by: IcePigZDB Co-authored-by: ShuNing --- planner/core/memtable_predicate_extractor.go | 47 +++++++++++-------- .../core/memtable_predicate_extractor_test.go | 6 +-- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/planner/core/memtable_predicate_extractor.go b/planner/core/memtable_predicate_extractor.go index fe355e4c9cae9..353d09d5d74ab 100644 --- a/planner/core/memtable_predicate_extractor.go +++ b/planner/core/memtable_predicate_extractor.go @@ -541,6 +541,24 @@ func (helper extractHelper) convertToTime(t int64) time.Time { return time.Unix(0, t) } +func (helper extractHelper) extractRoleBoolSliceFromUint64Slice(uint64Slice []uint64) []bool { + if len(uint64Slice) == 0 { + return []bool{false, true} + } + var res []bool + // use to keep res unique + b := make(map[bool]struct{}, 2) + for _, l := range uint64Slice { + tmpBool := (l == 1) + _, ok := b[tmpBool] + if !ok { + b[tmpBool] = struct{}{} + res = append(res, tmpBool) + } + } + return res +} + // ClusterTableExtractor is used to extract some predicates of cluster table. type ClusterTableExtractor struct { extractHelper @@ -748,7 +766,7 @@ func (e *HotRegionsHistoryTableExtractor) Extract( names []*types.FieldName, predicates []expression.Expression, ) []expression.Expression { - // Extract the `region_id/store_id/peer_id` columns. + // Extract the `region_id/store_id/peer_id` columns remained, regionIDSkipRequest, regionIDs := e.extractCol(schema, names, predicates, "region_id", false) remained, storeIDSkipRequest, storeIDs := e.extractCol(schema, names, remained, "store_id", false) remained, peerIDSkipRequest, peerIDs := e.extractCol(schema, names, remained, "peer_id", false) @@ -758,7 +776,7 @@ func (e *HotRegionsHistoryTableExtractor) Extract( return nil } - // Extract the is_learner/is_leader columns. + // Extract the is_learner/is_leader columns remained, isLearnerSkipRequest, isLearners := e.extractCol(schema, names, remained, "is_learner", false) remained, isLeaderSkipRequest, isLeaders := e.extractCol(schema, names, remained, "is_leader", false) isLearnersUint64, isLeadersUint64 := e.parseUint64(isLearners), e.parseUint64(isLeaders) @@ -766,22 +784,11 @@ func (e *HotRegionsHistoryTableExtractor) Extract( if e.SkipRequest { return nil } - // Request all if no conditions - if len(isLearnersUint64) == 0 { - isLearnersUint64 = []uint64{0, 1} - } - if len(isLeadersUint64) == 0 { - isLeadersUint64 = []uint64{0, 1} - } - // Change uint to ture false slice - for _, l := range isLearnersUint64 { - e.IsLearners = append(e.IsLearners, l == 1) - } - for _, l := range isLeadersUint64 { - e.IsLeaders = append(e.IsLeaders, l == 1) - } + // + e.IsLearners = e.extractRoleBoolSliceFromUint64Slice(isLearnersUint64) + e.IsLeaders = e.extractRoleBoolSliceFromUint64Slice(isLeadersUint64) - // Extract the `type` column. + // Extract the `type` column remained, typeSkipRequest, types := e.extractCol(schema, names, remained, "type", false) e.HotRegionTypes = types e.SkipRequest = typeSkipRequest @@ -796,7 +803,7 @@ func (e *HotRegionsHistoryTableExtractor) Extract( } remained, startTime, endTime := e.extractTimeRange(ctx, schema, names, remained, "update_time", ctx.GetSessionVars().StmtCtx.TimeZone) - // The time unit for search hot regions is millisecond. + // The time unit for search hot regions is millisecond startTime = startTime / int64(time.Millisecond) endTime = endTime / int64(time.Millisecond) e.StartTime = startTime @@ -835,10 +842,10 @@ func (e *HotRegionsHistoryTableExtractor) explainInfo(p *PhysicalMemTable) strin r.WriteString(fmt.Sprintf("peer_ids:[%s], ", extractStringFromUint64Slice(e.PeerIDs))) } if len(e.IsLearners) > 0 { - r.WriteString(fmt.Sprintf("roles:[%s], ", extractStringFromBoolSlice(e.IsLearners))) + r.WriteString(fmt.Sprintf("learner_roles:[%s], ", extractStringFromBoolSlice(e.IsLearners))) } if len(e.IsLeaders) > 0 { - r.WriteString(fmt.Sprintf("roles:[%s], ", extractStringFromBoolSlice(e.IsLeaders))) + r.WriteString(fmt.Sprintf("leader_roles:[%s], ", extractStringFromBoolSlice(e.IsLeaders))) } if len(e.HotRegionTypes) > 0 { r.WriteString(fmt.Sprintf("hot_region_types:[%s], ", extractStringFromStringSet(e.HotRegionTypes))) diff --git a/planner/core/memtable_predicate_extractor_test.go b/planner/core/memtable_predicate_extractor_test.go index a24c4a25f05db..311b9ae4a8838 100644 --- a/planner/core/memtable_predicate_extractor_test.go +++ b/planner/core/memtable_predicate_extractor_test.go @@ -1363,15 +1363,15 @@ func (s *extractorSuite) TestTiDBHotRegionsHistoryTableExtractor(c *C) { }, { sql: "select * from information_schema.tidb_hot_regions_history where is_learner in(3,4)", - isLearners: []bool{false, false}, + isLearners: []bool{false}, isLeaders: []bool{false, true}, hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { sql: "select * from information_schema.tidb_hot_regions_history where is_learner in(3,4) and is_leader in(0,1,true,false,3,4)", hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), - isLearners: []bool{false, false}, - isLeaders: []bool{false, true, false, false}, + isLearners: []bool{false}, + isLeaders: []bool{false, true}, }, { sql: "select * from information_schema.tidb_hot_regions_history where is_learner=1 and is_learner=0", From 4ddc37fa6e3111657c5653c89ed29322d92e0970 Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Wed, 20 Oct 2021 08:35:44 +0000 Subject: [PATCH 27/29] planner: update comment Signed-off-by: IcePigZDB --- planner/core/memtable_predicate_extractor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/memtable_predicate_extractor.go b/planner/core/memtable_predicate_extractor.go index 353d09d5d74ab..ebac26022c19e 100644 --- a/planner/core/memtable_predicate_extractor.go +++ b/planner/core/memtable_predicate_extractor.go @@ -784,7 +784,7 @@ func (e *HotRegionsHistoryTableExtractor) Extract( if e.SkipRequest { return nil } - // + // uint64 slice to unique bool slice e.IsLearners = e.extractRoleBoolSliceFromUint64Slice(isLearnersUint64) e.IsLeaders = e.extractRoleBoolSliceFromUint64Slice(isLeadersUint64) From e05a2489cc7803e555cf528945879dc4830c284f Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Wed, 17 Nov 2021 08:00:32 +0000 Subject: [PATCH 28/29] fix conflict due to the remove of pdapi.clusterVersion Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 6 +++--- executor/memtable_reader_test.go | 3 ++- util/pdapi/const.go | 16 ++++++++-------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index baa1863fe59d9..cb92487a8fbeb 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -981,17 +981,17 @@ func (e *hotRegionsHistoryRetriver) getHotRegionRowWithSchemaInfo( row[12].SetNull() } if hisHotRegion.FlowBytes != 0 { - row[13].SetFloat64(float64(hisHotRegion.FlowBytes)) + row[13].SetFloat64(hisHotRegion.FlowBytes) } else { row[13].SetNull() } if hisHotRegion.KeyRate != 0 { - row[14].SetFloat64(float64(hisHotRegion.KeyRate)) + row[14].SetFloat64(hisHotRegion.KeyRate) } else { row[14].SetNull() } if hisHotRegion.QueryRate != 0 { - row[15].SetFloat64(float64(hisHotRegion.QueryRate)) + row[15].SetFloat64(hisHotRegion.QueryRate) } else { row[15].SetNull() } diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index 6de671afbc2d4..9fa51faaa3737 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -1046,12 +1046,13 @@ func (s *testHotRegionsHistoryTableSuite) setUpMockPDHTTPServer(c *C) (*httptest server := httptest.NewServer(router) mockAddr := strings.TrimPrefix(server.URL, "http://") // mock PD API - router.Handle(pdapi.ClusterVersion, fn.Wrap(func() (string, error) { return "4.0.0-alpha", nil })) router.Handle(pdapi.Status, fn.Wrap(func() (interface{}, error) { return struct { + Version string `json:"version"` GitHash string `json:"git_hash"` StartTimestamp int64 `json:"start_timestamp"` }{ + Version: "4.0.0-alpha", GitHash: "mock-pd-githash", StartTimestamp: s.startTime.Unix(), }, nil diff --git a/util/pdapi/const.go b/util/pdapi/const.go index 09cde183b300f..0ce75f5d5a6a0 100644 --- a/util/pdapi/const.go +++ b/util/pdapi/const.go @@ -16,12 +16,12 @@ package pdapi // The following constants are the APIs of PD server. const ( - HotRead = "/pd/api/v1/hotspot/regions/read" - HotWrite = "/pd/api/v1/hotspot/regions/write" - HotHistory = "/pd/api/v1/hotspot/regions/history" - Regions = "/pd/api/v1/regions" - RegionByID = "/pd/api/v1/region/id/" - Stores = "/pd/api/v1/stores" - Status = "/pd/api/v1/status" - Config = "/pd/api/v1/config" + HotRead = "/pd/api/v1/hotspot/regions/read" + HotWrite = "/pd/api/v1/hotspot/regions/write" + HotHistory = "/pd/api/v1/hotspot/regions/history" + Regions = "/pd/api/v1/regions" + RegionByID = "/pd/api/v1/region/id/" + Stores = "/pd/api/v1/stores" + Status = "/pd/api/v1/status" + Config = "/pd/api/v1/config" ) From e4d113f4dceeb0831d93856fc42c3462ce1841dc Mon Sep 17 00:00:00 2001 From: IcePigZDB Date: Wed, 17 Nov 2021 10:33:11 +0000 Subject: [PATCH 29/29] address comment Signed-off-by: IcePigZDB --- executor/memtable_reader.go | 6 +++--- planner/core/memtable_predicate_extractor.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index cb92487a8fbeb..1f014e53d26ae 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -881,10 +881,10 @@ func (e *hotRegionsHistoryRetriver) retrieve(ctx context.Context, sctx sessionct e.heap = &hotRegionsResponseHeap{} for _, ch := range results { result := <-ch - if result.err != nil { - sctx.GetSessionVars().StmtCtx.AppendWarning(result.err) - } if result.err != nil || len(result.messages.HistoryHotRegion) == 0 { + if result.err != nil { + sctx.GetSessionVars().StmtCtx.AppendWarning(result.err) + } continue } *e.heap = append(*e.heap, result) diff --git a/planner/core/memtable_predicate_extractor.go b/planner/core/memtable_predicate_extractor.go index ebac26022c19e..923d025ac7d13 100644 --- a/planner/core/memtable_predicate_extractor.go +++ b/planner/core/memtable_predicate_extractor.go @@ -541,7 +541,7 @@ func (helper extractHelper) convertToTime(t int64) time.Time { return time.Unix(0, t) } -func (helper extractHelper) extractRoleBoolSliceFromUint64Slice(uint64Slice []uint64) []bool { +func (helper extractHelper) convertToBoolSlice(uint64Slice []uint64) []bool { if len(uint64Slice) == 0 { return []bool{false, true} } @@ -785,8 +785,8 @@ func (e *HotRegionsHistoryTableExtractor) Extract( return nil } // uint64 slice to unique bool slice - e.IsLearners = e.extractRoleBoolSliceFromUint64Slice(isLearnersUint64) - e.IsLeaders = e.extractRoleBoolSliceFromUint64Slice(isLeadersUint64) + e.IsLearners = e.convertToBoolSlice(isLearnersUint64) + e.IsLeaders = e.convertToBoolSlice(isLeadersUint64) // Extract the `type` column remained, typeSkipRequest, types := e.extractCol(schema, names, remained, "type", false)