From 761b34b8b23af8b7d525e2c08bfc4b4ea12ad51f Mon Sep 17 00:00:00 2001 From: yisaer Date: Tue, 31 Jan 2023 14:59:31 +0800 Subject: [PATCH 1/6] fix --- executor/historical_stats_test.go | 30 ++++++++++ statistics/handle/dump.go | 94 ++++++++++++++++++------------- 2 files changed, 86 insertions(+), 38 deletions(-) diff --git a/executor/historical_stats_test.go b/executor/historical_stats_test.go index becb1e82212f8..9ddd34655369f 100644 --- a/executor/historical_stats_test.go +++ b/executor/historical_stats_test.go @@ -365,3 +365,33 @@ PARTITION p0 VALUES LESS THAN (6) require.NotNil(t, jsTable.Partitions["p0"]) require.NotNil(t, jsTable.Partitions["global"]) } + +func TestDumpHistoricalStatsFallback(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set global tidb_enable_historical_stats = 0") + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec(`CREATE TABLE t (a int, b int, index idx(b)) +PARTITION BY RANGE ( a ) ( +PARTITION p0 VALUES LESS THAN (6) +)`) + // dump historical stats + tk.MustExec("analyze table t") + is := dom.InfoSchema() + tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + require.NotNil(t, tbl) + + // dump historical stats + hsWorker := dom.GetHistoricalStatsWorker() + tblID := hsWorker.GetOneHistoricalStatsTable() + // assert no historical stats task generated + require.Equal(t, tblID, int64(-1)) + tk.MustExec("set global tidb_enable_historical_stats = 1") + h := dom.StatsHandle() + jt, err := h.DumpHistoricalStatsBySnapshot("test", tbl.Meta(), oracle.GoTimeToTS(time.Now())) + require.NoError(t, err) + require.NotNil(t, jt) + require.False(t, jt.IsHistoricalStats) +} diff --git a/statistics/handle/dump.go b/statistics/handle/dump.go index da8603ea90573..c6a17e86ffd95 100644 --- a/statistics/handle/dump.go +++ b/statistics/handle/dump.go @@ -18,7 +18,6 @@ import ( "bytes" "compress/gzip" "encoding/json" - "fmt" "io" "time" @@ -40,15 +39,16 @@ import ( // JSONTable is used for dumping statistics. type JSONTable struct { - DatabaseName string `json:"database_name"` - TableName string `json:"table_name"` - Columns map[string]*jsonColumn `json:"columns"` - Indices map[string]*jsonColumn `json:"indices"` - ExtStats []*jsonExtendedStats `json:"ext_stats"` - Count int64 `json:"count"` - ModifyCount int64 `json:"modify_count"` - Partitions map[string]*JSONTable `json:"partitions"` - Version uint64 `json:"version"` + IsHistoricalStats bool `json:"is_historical_stats"` + DatabaseName string `json:"database_name"` + TableName string `json:"table_name"` + Columns map[string]*jsonColumn `json:"columns"` + Indices map[string]*jsonColumn `json:"indices"` + ExtStats []*jsonExtendedStats `json:"ext_stats"` + Count int64 `json:"count"` + ModifyCount int64 `json:"modify_count"` + Partitions map[string]*JSONTable `json:"partitions"` + Version uint64 `json:"version"` } type jsonExtendedStats struct { @@ -142,6 +142,14 @@ var ( // DumpHistoricalStatsBySnapshot dumped json tables from mysql.stats_meta_history and mysql.stats_history func (h *Handle) DumpHistoricalStatsBySnapshot(dbName string, tableInfo *model.TableInfo, snapshot uint64) (jt *JSONTable, err error) { + historicalStatsEnabled, err := h.CheckHistoricalStatsEnable() + if err != nil { + return nil, errors.Errorf("check %v failed: %v", variable.TiDBEnableHistoricalStats, err) + } + if !historicalStatsEnabled { + return nil, errors.Errorf("%v should be enabled", variable.TiDBEnableHistoricalStats) + } + defer func() { if err == nil { dumpHistoricalStatsSuccessCounter.Inc() @@ -149,10 +157,9 @@ func (h *Handle) DumpHistoricalStatsBySnapshot(dbName string, tableInfo *model.T dumpHistoricalStatsFailedCounter.Inc() } }() - pi := tableInfo.GetPartitionInfo() if pi == nil { - return h.tableHistoricalStatsToJSON(tableInfo.ID, snapshot) + return h.getTableHistoricalStatsToJsonWithFallback(dbName, tableInfo, tableInfo.ID, snapshot) } jsonTbl := &JSONTable{ DatabaseName: dbName, @@ -160,27 +167,19 @@ func (h *Handle) DumpHistoricalStatsBySnapshot(dbName string, tableInfo *model.T Partitions: make(map[string]*JSONTable, len(pi.Definitions)), } for _, def := range pi.Definitions { - tbl, err := h.tableHistoricalStatsToJSON(def.ID, snapshot) + tbl, err := h.getTableHistoricalStatsToJsonWithFallback(dbName, tableInfo, def.ID, snapshot) if err != nil { return nil, errors.Trace(err) } - if tbl == nil { - continue - } jsonTbl.Partitions[def.Name.L] = tbl } - h.mu.Lock() - isDynamicMode := variable.PartitionPruneMode(h.mu.ctx.GetSessionVars().PartitionPruneMode.Load()) == variable.Dynamic - h.mu.Unlock() - if isDynamicMode { - tbl, err := h.tableHistoricalStatsToJSON(tableInfo.ID, snapshot) - if err != nil { - logutil.BgLogger().Warn("dump global historical stats failed", - zap.Int64("table-id", tableInfo.ID), - zap.String("table-name", tableInfo.Name.String())) - } else if tbl != nil { - jsonTbl.Partitions["global"] = tbl - } + tbl, err := h.getTableHistoricalStatsToJsonWithFallback(dbName, tableInfo, tableInfo.ID, snapshot) + if err != nil { + return nil, err + } + // dump its global-stats if existed + if tbl != nil { + jsonTbl.Partitions["global"] = tbl } return jsonTbl, nil } @@ -250,11 +249,23 @@ func GenJSONTableFromStats(dbName string, tableInfo *model.TableInfo, tbl *stati return jsonTbl, nil } -func (h *Handle) tableHistoricalStatsToJSON(physicalID int64, snapshot uint64) (*JSONTable, error) { - reader, err := h.getGlobalStatsReader(0) +// getTableHistoricalStatsToJsonWithFallback try to get table historical stats, if not exit, directly fallback to the latest stats +func (h *Handle) getTableHistoricalStatsToJsonWithFallback(dbName string, tableInfo *model.TableInfo, physicalID int64, snapshot uint64) (*JSONTable, error) { + jt, exist, err := h.tableHistoricalStatsToJSON(physicalID, snapshot) if err != nil { return nil, err } + if !exist { + return h.tableStatsToJSON(dbName, tableInfo, physicalID, 0) + } + return jt, nil +} + +func (h *Handle) tableHistoricalStatsToJSON(physicalID int64, snapshot uint64) (*JSONTable, bool, error) { + reader, err := h.getGlobalStatsReader(0) + if err != nil { + return nil, false, err + } defer func() { err1 := h.releaseGlobalStatsReader(reader) if err == nil && err1 != nil { @@ -265,33 +276,39 @@ func (h *Handle) tableHistoricalStatsToJSON(physicalID int64, snapshot uint64) ( // get meta version rows, _, err := reader.Read("select distinct version from mysql.stats_meta_history where table_id = %? and version <= %? order by version desc limit 1", physicalID, snapshot) if err != nil { - return nil, errors.AddStack(err) + return nil, false, errors.AddStack(err) } if len(rows) < 1 { - return nil, fmt.Errorf("failed to get records of stats_meta_history for table_id = %v, snapshot = %v", physicalID, snapshot) + logutil.BgLogger().Warn("failed to get records of stats_meta_history", + zap.Int64("table-id", physicalID), + zap.Uint64("snapshotTS", snapshot)) + return nil, false, nil } statsMetaVersion := rows[0].GetInt64(0) // get stats meta rows, _, err = reader.Read("select modify_count, count from mysql.stats_meta_history where table_id = %? and version = %?", physicalID, statsMetaVersion) if err != nil { - return nil, errors.AddStack(err) + return nil, false, errors.AddStack(err) } modifyCount, count := rows[0].GetInt64(0), rows[0].GetInt64(1) // get stats version rows, _, err = reader.Read("select distinct version from mysql.stats_history where table_id = %? and version <= %? order by version desc limit 1", physicalID, snapshot) if err != nil { - return nil, errors.AddStack(err) + return nil, false, errors.AddStack(err) } if len(rows) < 1 { - return nil, fmt.Errorf("failed to get record of stats_history for table_id = %v, snapshot = %v", physicalID, snapshot) + logutil.BgLogger().Warn("failed to get record of stats_history", + zap.Int64("table-id", physicalID), + zap.Uint64("snapshotTS", snapshot)) + return nil, false, nil } statsVersion := rows[0].GetInt64(0) // get stats rows, _, err = reader.Read("select stats_data from mysql.stats_history where table_id = %? and version = %? order by seq_no", physicalID, statsVersion) if err != nil { - return nil, errors.AddStack(err) + return nil, false, errors.AddStack(err) } blocks := make([][]byte, 0) for _, row := range rows { @@ -299,11 +316,12 @@ func (h *Handle) tableHistoricalStatsToJSON(physicalID int64, snapshot uint64) ( } jsonTbl, err := BlocksToJSONTable(blocks) if err != nil { - return nil, errors.AddStack(err) + return nil, false, errors.AddStack(err) } jsonTbl.Count = count jsonTbl.ModifyCount = modifyCount - return jsonTbl, nil + jsonTbl.IsHistoricalStats = true + return jsonTbl, true, nil } func (h *Handle) tableStatsToJSON(dbName string, tableInfo *model.TableInfo, physicalID int64, snapshot uint64) (*JSONTable, error) { From ccbda88e577bec6de52076c655270db0dfc94cf7 Mon Sep 17 00:00:00 2001 From: yisaer Date: Tue, 31 Jan 2023 15:12:27 +0800 Subject: [PATCH 2/6] fix --- statistics/handle/dump.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/statistics/handle/dump.go b/statistics/handle/dump.go index c6a17e86ffd95..016fc858857ca 100644 --- a/statistics/handle/dump.go +++ b/statistics/handle/dump.go @@ -159,7 +159,7 @@ func (h *Handle) DumpHistoricalStatsBySnapshot(dbName string, tableInfo *model.T }() pi := tableInfo.GetPartitionInfo() if pi == nil { - return h.getTableHistoricalStatsToJsonWithFallback(dbName, tableInfo, tableInfo.ID, snapshot) + return h.getTableHistoricalStatsToJSONWithFallback(dbName, tableInfo, tableInfo.ID, snapshot) } jsonTbl := &JSONTable{ DatabaseName: dbName, @@ -167,13 +167,13 @@ func (h *Handle) DumpHistoricalStatsBySnapshot(dbName string, tableInfo *model.T Partitions: make(map[string]*JSONTable, len(pi.Definitions)), } for _, def := range pi.Definitions { - tbl, err := h.getTableHistoricalStatsToJsonWithFallback(dbName, tableInfo, def.ID, snapshot) + tbl, err := h.getTableHistoricalStatsToJSONWithFallback(dbName, tableInfo, def.ID, snapshot) if err != nil { return nil, errors.Trace(err) } jsonTbl.Partitions[def.Name.L] = tbl } - tbl, err := h.getTableHistoricalStatsToJsonWithFallback(dbName, tableInfo, tableInfo.ID, snapshot) + tbl, err := h.getTableHistoricalStatsToJSONWithFallback(dbName, tableInfo, tableInfo.ID, snapshot) if err != nil { return nil, err } @@ -249,8 +249,8 @@ func GenJSONTableFromStats(dbName string, tableInfo *model.TableInfo, tbl *stati return jsonTbl, nil } -// getTableHistoricalStatsToJsonWithFallback try to get table historical stats, if not exit, directly fallback to the latest stats -func (h *Handle) getTableHistoricalStatsToJsonWithFallback(dbName string, tableInfo *model.TableInfo, physicalID int64, snapshot uint64) (*JSONTable, error) { +// getTableHistoricalStatsToJSONWithFallback try to get table historical stats, if not exit, directly fallback to the latest stats +func (h *Handle) getTableHistoricalStatsToJSONWithFallback(dbName string, tableInfo *model.TableInfo, physicalID int64, snapshot uint64) (*JSONTable, error) { jt, exist, err := h.tableHistoricalStatsToJSON(physicalID, snapshot) if err != nil { return nil, err From c1c6478bc86ec2ceb4d9ad29558c83798db94ed9 Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 1 Feb 2023 13:43:05 +0800 Subject: [PATCH 3/6] add test --- server/statistics_handler_test.go | 176 ++++++++++++++++++++++++------ 1 file changed, 141 insertions(+), 35 deletions(-) diff --git a/server/statistics_handler_test.go b/server/statistics_handler_test.go index e0ecc7ba853f0..6e66368386878 100644 --- a/server/statistics_handler_test.go +++ b/server/statistics_handler_test.go @@ -16,7 +16,6 @@ package server import ( "database/sql" - "encoding/json" "fmt" "io" "os" @@ -125,33 +124,99 @@ func TestDumpStatsAPI(t *testing.T) { _, err = fp1.Write(js) require.NoError(t, err) checkData(t, path1, client) - - testDumpPartitionTableStats(t, client, statsHandler) } -func testDumpPartitionTableStats(t *testing.T, client *testServerClient, handler *StatsHandler) { - preparePartitionData(t, client, handler) - check := func(dumpStats bool) { - expectedLen := 1 - if dumpStats { - expectedLen = 2 - } - url := fmt.Sprintf("/stats/dump/test/test2?dumpPartitionStats=%v", dumpStats) - resp0, err := client.fetchStatus(url) - require.NoError(t, err) - defer func() { - resp0.Body.Close() - }() - b, err := io.ReadAll(resp0.Body) - require.NoError(t, err) - jsonTable := &handle.JSONTable{} - err = json.Unmarshal(b, jsonTable) +func TestDumpPartitionTableStats(t *testing.T) { + store := testkit.CreateMockStore(t) + + driver := NewTiDBDriver(store) + client := newTestServerClient() + cfg := newTestConfig() + cfg.Port = client.port + cfg.Status.StatusPort = client.statusPort + cfg.Status.ReportStatus = true + cfg.Socket = fmt.Sprintf("/tmp/tidb-mock-%d.sock", time.Now().UnixNano()) + + server, err := NewServer(cfg, driver) + require.NoError(t, err) + defer server.Close() + + client.port = getPortFromTCPAddr(server.listener.Addr()) + client.statusPort = getPortFromTCPAddr(server.statusListener.Addr()) + go func() { + err := server.Run() require.NoError(t, err) - require.NotNil(t, jsonTable.Partitions["global"]) - require.Len(t, jsonTable.Partitions, expectedLen) - } - check(false) - check(true) + }() + client.waitUntilServerOnline() + + dom, err := session.GetDomain(store) + require.NoError(t, err) + statsHandler := &StatsHandler{dom} + + preparePartitionData(t, client, statsHandler) + tableInfo, err := dom.InfoSchema().TableByName(model.NewCIStr("tidb"), model.NewCIStr("test2")) + require.NoError(t, err) + err = dom.GetHistoricalStatsWorker().DumpHistoricalStats(tableInfo.Meta().ID, dom.StatsHandle()) + require.NoError(t, err) + + router := mux.NewRouter() + router.Handle("/stats/dump/{db}/{table}", statsHandler) + + resp0, err := client.fetchStatus("/stats/dump/tidb/test2") + require.NoError(t, err) + defer func() { + require.NoError(t, resp0.Body.Close()) + }() + + path := "/tmp/stats2.json" + fp, err := os.Create(path) + require.NoError(t, err) + require.NotNil(t, fp) + defer func() { + require.NoError(t, fp.Close()) + require.NoError(t, os.Remove(path)) + }() + + js, err := io.ReadAll(resp0.Body) + require.NoError(t, err) + _, err = fp.Write(js) + require.NoError(t, err) + checkPartitionTableData(t, path, client) + + // sleep for 1 seconds to ensure the existence of tidb.test2 + time.Sleep(time.Second) + timeBeforeDropStats := time.Now() + snapshot := timeBeforeDropStats.Format("20060102150405") + prepare4DumpPartitionTableHistoryStats(t, client) + + // test dump history stats + resp1, err := client.fetchStatus("/stats/dump/tidb/test2") + require.NoError(t, err) + defer func() { + require.NoError(t, resp1.Body.Close()) + }() + js, err = io.ReadAll(resp1.Body) + require.NoError(t, err) + + path1 := "/tmp/stats2_history.json" + fp1, err := os.Create(path1) + require.NoError(t, err) + require.NotNil(t, fp1) + defer func() { + require.NoError(t, fp1.Close()) + require.NoError(t, os.Remove(path1)) + }() + + resp2, err := client.fetchStatus("/stats/dump/tidb/test2/" + snapshot) + require.NoError(t, err) + defer func() { + require.NoError(t, resp2.Body.Close()) + }() + js, err = io.ReadAll(resp2.Body) + require.NoError(t, err) + _, err = fp1.Write(js) + require.NoError(t, err) + checkPartitionTableData(t, path1, client) } func prepareData(t *testing.T, client *testServerClient, statHandle *StatsHandler) { @@ -172,8 +237,8 @@ func prepareData(t *testing.T, client *testServerClient, statHandle *StatsHandle tk.MustExec("create index c on test (a, b)") tk.MustExec("insert test values (1, 's')") require.NoError(t, h.DumpStatsDeltaToKV(handle.DumpAll)) - tk.MustExec("analyze table test") tk.MustExec("set global tidb_enable_historical_stats = 1") + tk.MustExec("analyze table test") tk.MustExec("insert into test(a,b) values (1, 'v'),(3, 'vvv'),(5, 'vv')") is := statHandle.do.InfoSchema() require.NoError(t, h.DumpStatsDeltaToKV(handle.DumpAll)) @@ -189,15 +254,22 @@ func preparePartitionData(t *testing.T, client *testServerClient, statHandle *St }() h := statHandle.do.StatsHandle() tk := testkit.NewDBTestKit(t, db) + tk.MustExec("create database tidb") + tk.MustExec("use tidb") tk.MustExec("create table test2(a int) PARTITION BY RANGE ( a ) (PARTITION p0 VALUES LESS THAN (6))") - tk.MustExec("insert into test2 (a) values (1)") + err = h.HandleDDLEvent(<-h.DDLEventCh()) + require.NoError(t, err) + tk.MustExec("insert into test2 (a) values (1),(2),(3),(4)") + require.NoError(t, h.DumpStatsDeltaToKV(handle.DumpAll)) + tk.MustExec("set global tidb_enable_historical_stats = 1") tk.MustExec("analyze table test2") + tk.MustExec("insert into test2 (a) values (1),(2),(3),(4)") is := statHandle.do.InfoSchema() require.NoError(t, h.DumpStatsDeltaToKV(handle.DumpAll)) require.NoError(t, h.Update(is)) } -func prepare4DumpHistoryStats(t *testing.T, client *testServerClient) { +func prepare4DumpPartitionTableHistoryStats(t *testing.T, client *testServerClient) { db, err := sql.Open("mysql", client.getDSN()) require.NoError(t, err, "Error connecting") defer func() { @@ -206,15 +278,20 @@ func prepare4DumpHistoryStats(t *testing.T, client *testServerClient) { }() tk := testkit.NewDBTestKit(t, db) + tk.MustExec("use tidb") + tk.MustExec("drop table tidb.test2") + tk.MustExec("create table test2 (a int) PARTITION BY RANGE ( a ) (PARTITION p0 VALUES LESS THAN (6))") +} - safePointName := "tikv_gc_safe_point" - safePointValue := "20060102-15:04:05 -0700" - safePointComment := "All versions after safe point can be accessed. (DO NOT EDIT)" - updateSafePoint := fmt.Sprintf(`INSERT INTO mysql.tidb VALUES ('%[1]s', '%[2]s', '%[3]s') - ON DUPLICATE KEY - UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) - tk.MustExec(updateSafePoint) +func prepare4DumpHistoryStats(t *testing.T, client *testServerClient) { + db, err := sql.Open("mysql", client.getDSN()) + require.NoError(t, err, "Error connecting") + defer func() { + err := db.Close() + require.NoError(t, err) + }() + tk := testkit.NewDBTestKit(t, db) tk.MustExec("drop table tidb.test") tk.MustExec("create table tidb.test (a int, b varchar(20))") } @@ -281,3 +358,32 @@ func checkData(t *testing.T, path string, client *testServerClient) { require.Equal(t, int64(4), count) require.NoError(t, rows.Close()) } + +func checkPartitionTableData(t *testing.T, path string, client *testServerClient) { + db, err := sql.Open("mysql", client.getDSN(func(config *mysql.Config) { + config.AllowAllFiles = true + config.Params["sql_mode"] = "''" + })) + require.NoError(t, err, "Error connecting") + tk := testkit.NewDBTestKit(t, db) + defer func() { + err := db.Close() + require.NoError(t, err) + }() + + tk.MustExec("use tidb") + tk.MustExec("drop stats test2") + tk.MustExec(fmt.Sprintf("load stats '%s'", path)) + rows := tk.MustQuery("show stats_meta") + require.True(t, rows.Next(), "unexpected data") + var dbName, tableName string + var modifyCount, count int64 + var other interface{} + err = rows.Scan(&dbName, &tableName, &other, &other, &modifyCount, &count) + require.NoError(t, err) + require.Equal(t, "tidb", dbName) + require.Equal(t, "test2", tableName) + require.Equal(t, int64(4), modifyCount) + require.Equal(t, int64(8), count) + require.NoError(t, rows.Close()) +} From 3c6973f1ddcddcada4eba854841d3129a22f2430 Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 1 Feb 2023 16:13:58 +0800 Subject: [PATCH 4/6] add test --- server/statistics_handler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/statistics_handler_test.go b/server/statistics_handler_test.go index 6e66368386878..5e2b3e1cd45ed 100644 --- a/server/statistics_handler_test.go +++ b/server/statistics_handler_test.go @@ -195,7 +195,7 @@ func TestDumpPartitionTableStats(t *testing.T) { defer func() { require.NoError(t, resp1.Body.Close()) }() - js, err = io.ReadAll(resp1.Body) + _, err = io.ReadAll(resp1.Body) require.NoError(t, err) path1 := "/tmp/stats2_history.json" From 1a079e1b878e8aa714dd14e56cab89f6b99a58b1 Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 1 Feb 2023 17:08:04 +0800 Subject: [PATCH 5/6] Revert "add test" This reverts commit 3c6973f1ddcddcada4eba854841d3129a22f2430. --- server/statistics_handler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/statistics_handler_test.go b/server/statistics_handler_test.go index 5e2b3e1cd45ed..6e66368386878 100644 --- a/server/statistics_handler_test.go +++ b/server/statistics_handler_test.go @@ -195,7 +195,7 @@ func TestDumpPartitionTableStats(t *testing.T) { defer func() { require.NoError(t, resp1.Body.Close()) }() - _, err = io.ReadAll(resp1.Body) + js, err = io.ReadAll(resp1.Body) require.NoError(t, err) path1 := "/tmp/stats2_history.json" From 7c09d9cbdb0b60215c7241ee12ad74bc4b1db279 Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 1 Feb 2023 17:08:14 +0800 Subject: [PATCH 6/6] Revert "add test" This reverts commit c1c6478bc86ec2ceb4d9ad29558c83798db94ed9. --- server/statistics_handler_test.go | 176 ++++++------------------------ 1 file changed, 35 insertions(+), 141 deletions(-) diff --git a/server/statistics_handler_test.go b/server/statistics_handler_test.go index 6e66368386878..e0ecc7ba853f0 100644 --- a/server/statistics_handler_test.go +++ b/server/statistics_handler_test.go @@ -16,6 +16,7 @@ package server import ( "database/sql" + "encoding/json" "fmt" "io" "os" @@ -124,99 +125,33 @@ func TestDumpStatsAPI(t *testing.T) { _, err = fp1.Write(js) require.NoError(t, err) checkData(t, path1, client) -} - -func TestDumpPartitionTableStats(t *testing.T) { - store := testkit.CreateMockStore(t) - - driver := NewTiDBDriver(store) - client := newTestServerClient() - cfg := newTestConfig() - cfg.Port = client.port - cfg.Status.StatusPort = client.statusPort - cfg.Status.ReportStatus = true - cfg.Socket = fmt.Sprintf("/tmp/tidb-mock-%d.sock", time.Now().UnixNano()) - server, err := NewServer(cfg, driver) - require.NoError(t, err) - defer server.Close() + testDumpPartitionTableStats(t, client, statsHandler) +} - client.port = getPortFromTCPAddr(server.listener.Addr()) - client.statusPort = getPortFromTCPAddr(server.statusListener.Addr()) - go func() { - err := server.Run() +func testDumpPartitionTableStats(t *testing.T, client *testServerClient, handler *StatsHandler) { + preparePartitionData(t, client, handler) + check := func(dumpStats bool) { + expectedLen := 1 + if dumpStats { + expectedLen = 2 + } + url := fmt.Sprintf("/stats/dump/test/test2?dumpPartitionStats=%v", dumpStats) + resp0, err := client.fetchStatus(url) require.NoError(t, err) - }() - client.waitUntilServerOnline() - - dom, err := session.GetDomain(store) - require.NoError(t, err) - statsHandler := &StatsHandler{dom} - - preparePartitionData(t, client, statsHandler) - tableInfo, err := dom.InfoSchema().TableByName(model.NewCIStr("tidb"), model.NewCIStr("test2")) - require.NoError(t, err) - err = dom.GetHistoricalStatsWorker().DumpHistoricalStats(tableInfo.Meta().ID, dom.StatsHandle()) - require.NoError(t, err) - - router := mux.NewRouter() - router.Handle("/stats/dump/{db}/{table}", statsHandler) - - resp0, err := client.fetchStatus("/stats/dump/tidb/test2") - require.NoError(t, err) - defer func() { - require.NoError(t, resp0.Body.Close()) - }() - - path := "/tmp/stats2.json" - fp, err := os.Create(path) - require.NoError(t, err) - require.NotNil(t, fp) - defer func() { - require.NoError(t, fp.Close()) - require.NoError(t, os.Remove(path)) - }() - - js, err := io.ReadAll(resp0.Body) - require.NoError(t, err) - _, err = fp.Write(js) - require.NoError(t, err) - checkPartitionTableData(t, path, client) - - // sleep for 1 seconds to ensure the existence of tidb.test2 - time.Sleep(time.Second) - timeBeforeDropStats := time.Now() - snapshot := timeBeforeDropStats.Format("20060102150405") - prepare4DumpPartitionTableHistoryStats(t, client) - - // test dump history stats - resp1, err := client.fetchStatus("/stats/dump/tidb/test2") - require.NoError(t, err) - defer func() { - require.NoError(t, resp1.Body.Close()) - }() - js, err = io.ReadAll(resp1.Body) - require.NoError(t, err) - - path1 := "/tmp/stats2_history.json" - fp1, err := os.Create(path1) - require.NoError(t, err) - require.NotNil(t, fp1) - defer func() { - require.NoError(t, fp1.Close()) - require.NoError(t, os.Remove(path1)) - }() - - resp2, err := client.fetchStatus("/stats/dump/tidb/test2/" + snapshot) - require.NoError(t, err) - defer func() { - require.NoError(t, resp2.Body.Close()) - }() - js, err = io.ReadAll(resp2.Body) - require.NoError(t, err) - _, err = fp1.Write(js) - require.NoError(t, err) - checkPartitionTableData(t, path1, client) + defer func() { + resp0.Body.Close() + }() + b, err := io.ReadAll(resp0.Body) + require.NoError(t, err) + jsonTable := &handle.JSONTable{} + err = json.Unmarshal(b, jsonTable) + require.NoError(t, err) + require.NotNil(t, jsonTable.Partitions["global"]) + require.Len(t, jsonTable.Partitions, expectedLen) + } + check(false) + check(true) } func prepareData(t *testing.T, client *testServerClient, statHandle *StatsHandler) { @@ -237,8 +172,8 @@ func prepareData(t *testing.T, client *testServerClient, statHandle *StatsHandle tk.MustExec("create index c on test (a, b)") tk.MustExec("insert test values (1, 's')") require.NoError(t, h.DumpStatsDeltaToKV(handle.DumpAll)) - tk.MustExec("set global tidb_enable_historical_stats = 1") tk.MustExec("analyze table test") + tk.MustExec("set global tidb_enable_historical_stats = 1") tk.MustExec("insert into test(a,b) values (1, 'v'),(3, 'vvv'),(5, 'vv')") is := statHandle.do.InfoSchema() require.NoError(t, h.DumpStatsDeltaToKV(handle.DumpAll)) @@ -254,22 +189,15 @@ func preparePartitionData(t *testing.T, client *testServerClient, statHandle *St }() h := statHandle.do.StatsHandle() tk := testkit.NewDBTestKit(t, db) - tk.MustExec("create database tidb") - tk.MustExec("use tidb") tk.MustExec("create table test2(a int) PARTITION BY RANGE ( a ) (PARTITION p0 VALUES LESS THAN (6))") - err = h.HandleDDLEvent(<-h.DDLEventCh()) - require.NoError(t, err) - tk.MustExec("insert into test2 (a) values (1),(2),(3),(4)") - require.NoError(t, h.DumpStatsDeltaToKV(handle.DumpAll)) - tk.MustExec("set global tidb_enable_historical_stats = 1") + tk.MustExec("insert into test2 (a) values (1)") tk.MustExec("analyze table test2") - tk.MustExec("insert into test2 (a) values (1),(2),(3),(4)") is := statHandle.do.InfoSchema() require.NoError(t, h.DumpStatsDeltaToKV(handle.DumpAll)) require.NoError(t, h.Update(is)) } -func prepare4DumpPartitionTableHistoryStats(t *testing.T, client *testServerClient) { +func prepare4DumpHistoryStats(t *testing.T, client *testServerClient) { db, err := sql.Open("mysql", client.getDSN()) require.NoError(t, err, "Error connecting") defer func() { @@ -278,20 +206,15 @@ func prepare4DumpPartitionTableHistoryStats(t *testing.T, client *testServerClie }() tk := testkit.NewDBTestKit(t, db) - tk.MustExec("use tidb") - tk.MustExec("drop table tidb.test2") - tk.MustExec("create table test2 (a int) PARTITION BY RANGE ( a ) (PARTITION p0 VALUES LESS THAN (6))") -} -func prepare4DumpHistoryStats(t *testing.T, client *testServerClient) { - db, err := sql.Open("mysql", client.getDSN()) - require.NoError(t, err, "Error connecting") - defer func() { - err := db.Close() - require.NoError(t, err) - }() + safePointName := "tikv_gc_safe_point" + safePointValue := "20060102-15:04:05 -0700" + safePointComment := "All versions after safe point can be accessed. (DO NOT EDIT)" + updateSafePoint := fmt.Sprintf(`INSERT INTO mysql.tidb VALUES ('%[1]s', '%[2]s', '%[3]s') + ON DUPLICATE KEY + UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) + tk.MustExec(updateSafePoint) - tk := testkit.NewDBTestKit(t, db) tk.MustExec("drop table tidb.test") tk.MustExec("create table tidb.test (a int, b varchar(20))") } @@ -358,32 +281,3 @@ func checkData(t *testing.T, path string, client *testServerClient) { require.Equal(t, int64(4), count) require.NoError(t, rows.Close()) } - -func checkPartitionTableData(t *testing.T, path string, client *testServerClient) { - db, err := sql.Open("mysql", client.getDSN(func(config *mysql.Config) { - config.AllowAllFiles = true - config.Params["sql_mode"] = "''" - })) - require.NoError(t, err, "Error connecting") - tk := testkit.NewDBTestKit(t, db) - defer func() { - err := db.Close() - require.NoError(t, err) - }() - - tk.MustExec("use tidb") - tk.MustExec("drop stats test2") - tk.MustExec(fmt.Sprintf("load stats '%s'", path)) - rows := tk.MustQuery("show stats_meta") - require.True(t, rows.Next(), "unexpected data") - var dbName, tableName string - var modifyCount, count int64 - var other interface{} - err = rows.Scan(&dbName, &tableName, &other, &other, &modifyCount, &count) - require.NoError(t, err) - require.Equal(t, "tidb", dbName) - require.Equal(t, "test2", tableName) - require.Equal(t, int64(4), modifyCount) - require.Equal(t, int64(8), count) - require.NoError(t, rows.Close()) -}