Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

*: add table storage info status api #22859

Merged
merged 11 commits into from
Jul 19, 2021
Merged
4 changes: 1 addition & 3 deletions domain/domain.go
Original file line number Diff line number Diff line change
Expand Up @@ -622,13 +622,11 @@ func (do *Domain) Close() {
terror.Log(errors.Trace(do.etcdClient.Close()))
}

do.sysSessionPool.Close()
do.slowQuery.Close()

do.cancel()
do.wg.Wait()

do.sysSessionPool.Close()

logutil.BgLogger().Info("domain closed", zap.Duration("take time", time.Since(startTime)))
}

Expand Down
126 changes: 126 additions & 0 deletions server/http_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import (
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/binloginfo"
"github.com/pingcap/tidb/sessionctx/stmtctx"
"github.com/pingcap/tidb/sessionctx/variable"
Expand All @@ -62,6 +63,7 @@ import (
"github.com/pingcap/tidb/util/gcutil"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/pdapi"
"github.com/pingcap/tidb/util/sqlexec"
"github.com/tikv/client-go/v2/tikv"
"go.uber.org/zap"
)
Expand Down Expand Up @@ -328,6 +330,11 @@ type schemaHandler struct {
*tikvHandlerTool
}

// schemaStorageHandler is the handler for list database or table schemas.
type schemaStorageHandler struct {
*tikvHandlerTool
}

type dbTableHandler struct {
*tikvHandlerTool
}
Expand Down Expand Up @@ -917,6 +924,125 @@ func (h flashReplicaHandler) handleStatusReport(w http.ResponseWriter, req *http
zap.Error(err))
}

type schemaTableStorage struct {
TableSchema string `json:"table_schema"`
TableName string `json:"table_name"`
TableRows int64 `json:"table_rows"`
AvgRowLength int64 `json:"avg_row_length"`
DataLength int64 `json:"data_length"`
MaxDataLength int64 `json:"max_data_length"`
IndexLength int64 `json:"index_length"`
DataFree int64 `json:"data_free"`
}

func getSchemaTablesStorageInfo(h *schemaStorageHandler, schema *model.CIStr, table *model.CIStr) (messages []*schemaTableStorage, err error) {
var s session.Session
if s, err = session.CreateSession(h.Store); err != nil {
return
}
defer s.Close()

ctx := s.(sessionctx.Context)
condition := make([]string, 0)
params := make([]interface{}, 0)

if schema != nil {
condition = append(condition, `TABLE_SCHEMA = %?`)
params = append(params, schema.O)
}
if table != nil {
condition = append(condition, `TABLE_NAME = %?`)
params = append(params, table.O)
}

sql := `select TABLE_SCHEMA,TABLE_NAME,TABLE_ROWS,AVG_ROW_LENGTH,DATA_LENGTH,MAX_DATA_LENGTH,INDEX_LENGTH,DATA_FREE from INFORMATION_SCHEMA.TABLES`
if len(condition) > 0 {
sql += ` WHERE ` + strings.Join(condition, ` AND `)
}
var results sqlexec.RecordSet
if results, err = ctx.(sqlexec.SQLExecutor).ExecuteInternal(context.TODO(), sql, params...); err != nil {
logutil.BgLogger().Error(`ExecuteInternal`, zap.Error(err))
} else if results != nil {
messages = make([]*schemaTableStorage, 0)
defer terror.Call(results.Close)
for {
req := results.NewChunk()
if err = results.Next(context.TODO(), req); err != nil {
break
}

if req.NumRows() == 0 {
break
}

for i := 0; i < req.NumRows(); i++ {
messages = append(messages, &schemaTableStorage{
TableSchema: req.GetRow(i).GetString(0),
TableName: req.GetRow(i).GetString(1),
TableRows: req.GetRow(i).GetInt64(2),
AvgRowLength: req.GetRow(i).GetInt64(3),
DataLength: req.GetRow(i).GetInt64(4),
MaxDataLength: req.GetRow(i).GetInt64(5),
IndexLength: req.GetRow(i).GetInt64(6),
DataFree: req.GetRow(i).GetInt64(7),
})
}
}
}
return
}

// ServeHTTP handles request of list a database or table's schemas.
func (h schemaStorageHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
schema, err := h.schema()
if err != nil {
writeError(w, err)
return
}

// parse params
params := mux.Vars(req)

var (
dbName *model.CIStr
tableName *model.CIStr
isSingle bool
)

if reqDbName, ok := params[pDBName]; ok {
cDBName := model.NewCIStr(reqDbName)
// all table schemas in a specified database
schemaInfo, exists := schema.SchemaByName(cDBName)
if !exists {
writeError(w, infoschema.ErrDatabaseNotExists.GenWithStackByArgs(reqDbName))
return
}
dbName = &schemaInfo.Name

if reqTableName, ok := params[pTableName]; ok {
// table schema of a specified table name
cTableName := model.NewCIStr(reqTableName)
data, e := schema.TableByName(cDBName, cTableName)
if e != nil {
writeError(w, e)
return
}
tableName = &data.Meta().Name
isSingle = true
}
}

if results, e := getSchemaTablesStorageInfo(&h, dbName, tableName); e != nil {
writeError(w, e)
} else {
if isSingle {
writeData(w, results[0])
} else {
writeData(w, results)
}
}
}

// ServeHTTP handles request of list a database or table's schemas.
func (h schemaHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
schema, err := h.schema()
Expand Down
58 changes: 55 additions & 3 deletions server/http_handler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ import (
"github.com/pingcap/tidb/util/codec"
"github.com/pingcap/tidb/util/deadlockhistory"
"github.com/pingcap/tidb/util/rowcodec"
"github.com/pingcap/tidb/util/testkit"
"github.com/pingcap/tidb/util/versioninfo"
"github.com/tikv/client-go/v2/tikv"
"go.uber.org/zap"
Expand All @@ -67,6 +68,7 @@ type basicHTTPHandlerTestSuite struct {
store kv.Storage
domain *domain.Domain
tidbdrv *TiDBDriver
sh *StatsHandler
}

type HTTPHandlerTestSuite struct {
Expand Down Expand Up @@ -490,22 +492,26 @@ func (ts *basicHTTPHandlerTestSuite) startServer(c *C) {
c.Assert(err, IsNil)
}()
ts.waitUntilServerOnline()

do, err := session.GetDomain(ts.store)
c.Assert(err, IsNil)
ts.sh = &StatsHandler{do}
}

func getPortFromTCPAddr(addr net.Addr) uint {
return uint(addr.(*net.TCPAddr).Port)
}

func (ts *basicHTTPHandlerTestSuite) stopServer(c *C) {
if ts.server != nil {
ts.server.Close()
}
if ts.domain != nil {
ts.domain.Close()
}
if ts.store != nil {
ts.store.Close()
}
if ts.server != nil {
ts.server.Close()
}
}

func (ts *basicHTTPHandlerTestSuite) prepareData(c *C) {
Expand Down Expand Up @@ -1112,6 +1118,52 @@ func (ts *HTTPHandlerTestSuite) TestGetSchema(c *C) {
c.Assert(dbtbl.TableInfo, DeepEquals, t)
}

func (ts *HTTPHandlerTestSuite) TestGetSchemaStorage(c *C) {
ts.startServer(c)
ts.prepareData(c)
defer ts.stopServer(c)

do := ts.domain
h := do.StatsHandle()
do.SetStatsUpdating(true)

tk := testkit.NewTestKitWithInit(c, ts.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t (c int, d int, e char(5), index idx(e))")
tk.MustExec(`insert into t(c, d, e) values(1, 2, "c"), (2, 3, "d"), (3, 4, "e")`)
h.FlushStats()

resp, err := ts.fetchStatus("/schema_storage/test")
c.Assert(err, IsNil)
decoder := json.NewDecoder(resp.Body)
var tables []*schemaTableStorage
err = decoder.Decode(&tables)
c.Assert(err, IsNil)
c.Assert(len(tables), Equals, 1)
expects := []string{`t`}
names := make([]string, len(tables))
for i, v := range tables {
names[i] = v.TableName
}

sort.Strings(names)
c.Assert(names, DeepEquals, expects)

c.Assert(
[]int64{
tables[0].TableRows,
tables[0].AvgRowLength,
tables[0].DataLength,
tables[0].MaxDataLength,
tables[0].IndexLength,
tables[0].DataFree,
},
DeepEquals,
[]int64{3, 18, 54, 0, 6, 0},
)
}

func (ts *HTTPHandlerTestSuite) TestAllHistory(c *C) {
ts.startServer(c)
ts.prepareData(c)
Expand Down
5 changes: 5 additions & 0 deletions server/http_status.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,11 @@ func (s *Server) startHTTPServer() {
router.Handle("/schema/{db}", schemaHandler{tikvHandlerTool})
router.Handle("/schema/{db}/{table}", schemaHandler{tikvHandlerTool})
router.Handle("/tables/{colID}/{colTp}/{colFlag}/{colLen}", valueHandler{})

router.Handle("/schema_storage", schemaStorageHandler{tikvHandlerTool}).Name("Schema Storage")
router.Handle("/schema_storage/{db}", schemaStorageHandler{tikvHandlerTool})
router.Handle("/schema_storage/{db}/{table}", schemaStorageHandler{tikvHandlerTool})

router.Handle("/ddl/history", ddlHistoryJobHandler{tikvHandlerTool}).Name("DDL_History")
router.Handle("/ddl/owner/resign", ddlResignOwnerHandler{tikvHandlerTool.Store.(kv.Storage)}).Name("DDL_Owner_Resign")

Expand Down