From 92d3108a1a41b2b2a9f60080f7ba217738629b3c Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 13 May 2020 21:55:38 +0800 Subject: [PATCH 1/6] util/admin: support admin check index on partition table Signed-off-by: crazycs520 --- executor/admin_test.go | 26 ++++++++ executor/builder.go | 23 +------ executor/executor.go | 61 ++---------------- go.sum | 1 + planner/core/common_plans.go | 10 +-- planner/core/planbuilder.go | 119 ++++++++++++++++++++++------------- 6 files changed, 111 insertions(+), 129 deletions(-) diff --git a/executor/admin_test.go b/executor/admin_test.go index c7229218e5eaf..75eeb577a138c 100644 --- a/executor/admin_test.go +++ b/executor/admin_test.go @@ -46,6 +46,32 @@ func (s *testSuite1) TestAdminCheckIndexRange(c *C) { result.Check(testkit.Rows("-1 hi 4", "2 cd 2")) } +func (s *testSuite5) TestAdminCheckIndex(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + check := func() { + tk.MustExec("insert admin_test (c1, c2) values (1, 1), (2, 2), (5, 5), (10, 10), (11, 11), (NULL, NULL)") + tk.MustExec("admin check index admin_test c1") + tk.MustExec("admin check index admin_test c2") + } + tk.MustExec("drop table if exists admin_test") + tk.MustExec("create table admin_test (c1 int, c2 int, c3 int default 1, index (c1), unique key(c2))") + check() + + // Test for hash partition table. + tk.MustExec("drop table if exists admin_test") + tk.MustExec("create table admin_test (c1 int, c2 int, c3 int default 1, index (c1), unique key(c2)) partition by hash(c2) partitions 5;") + tk.MustExec("insert admin_test (c1, c2) values (1, 1), (2, 2), (NULL, NULL)") + + // Test for range partition table. + tk.MustExec("drop table if exists admin_test") + tk.MustExec(`create table admin_test (c1 int, c2 int, c3 int default 1, index (c1), unique key(c2)) PARTITION BY RANGE ( c2 ) ( + PARTITION p0 VALUES LESS THAN (5), + PARTITION p1 VALUES LESS THAN (10), + PARTITION p2 VALUES LESS THAN (MAXVALUE))`) + tk.MustExec("insert admin_test (c1, c2) values (1, 1), (2, 2), (NULL, NULL)") +} + func (s *testSuite5) TestAdminRecoverIndex(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") diff --git a/executor/builder.go b/executor/builder.go index 207797d4e5264..7803b73eb5521 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -106,8 +106,6 @@ func (b *executorBuilder) build(p plannercore.Plan) Executor { return b.buildChange(v) case *plannercore.CheckTable: return b.buildCheckTable(v) - case *plannercore.CheckIndex: - return b.buildCheckIndex(v) case *plannercore.RecoverIndex: return b.buildRecoverIndex(v) case *plannercore.CleanupIndex: @@ -331,26 +329,6 @@ func (b *executorBuilder) buildShowSlow(v *plannercore.ShowSlow) Executor { return e } -func (b *executorBuilder) buildCheckIndex(v *plannercore.CheckIndex) Executor { - readerExec, err := buildNoRangeIndexLookUpReader(b, v.IndexLookUpReader) - if err != nil { - b.err = err - return nil - } - - buildIndexLookUpChecker(b, v.IndexLookUpReader, readerExec) - - e := &CheckIndexExec{ - baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()), - dbName: v.DBName, - tableName: readerExec.table.Meta().Name.L, - idxName: v.IdxName, - is: b.is, - src: readerExec, - } - return e -} - // buildIndexLookUpChecker builds check information to IndexLookUpReader. func buildIndexLookUpChecker(b *executorBuilder, readerPlan *plannercore.PhysicalIndexLookUpReader, readerExec *IndexLookUpExecutor) { @@ -403,6 +381,7 @@ func (b *executorBuilder) buildCheckTable(v *plannercore.CheckTable) Executor { srcs: readerExecs, exitCh: make(chan struct{}), retCh: make(chan error, len(readerExecs)), + checkIndex: v.CheckIndex, } return e } diff --git a/executor/executor.go b/executor/executor.go index 769c8e62c8c95..8aa15799ee0e1 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -62,7 +62,6 @@ import ( var ( _ Executor = &baseExecutor{} - _ Executor = &CheckIndexExec{} _ Executor = &CheckTableExec{} _ Executor = &HashAggExec{} _ Executor = &HashJoinExec{} @@ -637,6 +636,7 @@ type CheckTableExec struct { is infoschema.InfoSchema exitCh chan struct{} retCh chan error + checkIndex bool } // Open implements the Executor Open interface. @@ -724,6 +724,10 @@ func (e *CheckTableExec) Next(ctx context.Context, req *chunk.Chunk) error { } greater, idxOffset, err := admin.CheckIndicesCount(e.ctx, e.dbName, e.table.Meta().Name.O, idxNames) if err != nil { + // For admin check index statement, for speed up and compatibility, doesn't do below checks. + if e.checkIndex { + return errors.Trace(err) + } if greater == admin.IdxCntGreater { err = e.checkTableIndexHandle(ctx, e.indexInfos[idxOffset]) } else if greater == admin.TblCntGreater { @@ -790,61 +794,6 @@ func (e *CheckTableExec) checkTableRecord(idxOffset int) error { return nil } -// CheckIndexExec represents the executor of checking an index. -// It is built from the "admin check index" statement, and it checks -// the consistency of the index data with the records of the table. -type CheckIndexExec struct { - baseExecutor - - dbName string - tableName string - idxName string - src *IndexLookUpExecutor - done bool - is infoschema.InfoSchema -} - -// Open implements the Executor Open interface. -func (e *CheckIndexExec) Open(ctx context.Context) error { - if err := e.baseExecutor.Open(ctx); err != nil { - return err - } - if err := e.src.Open(ctx); err != nil { - return err - } - e.done = false - return nil -} - -// Close implements the Executor Close interface. -func (e *CheckIndexExec) Close() error { - return e.src.Close() -} - -// Next implements the Executor Next interface. -func (e *CheckIndexExec) Next(ctx context.Context, req *chunk.Chunk) error { - if e.done { - return nil - } - defer func() { e.done = true }() - - _, _, err := admin.CheckIndicesCount(e.ctx, e.dbName, e.tableName, []string{e.idxName}) - if err != nil { - return err - } - chk := newFirstChunk(e.src) - for { - err := Next(ctx, e.src, chk) - if err != nil { - return err - } - if chk.NumRows() == 0 { - break - } - } - return nil -} - // ShowSlowExec represents the executor of showing the slow queries. // It is build from the "admin show slow" statement: // admin show slow top [internal | all] N diff --git a/go.sum b/go.sum index 85d08c7650c41..4c47d4fb2dd40 100644 --- a/go.sum +++ b/go.sum @@ -408,6 +408,7 @@ github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.19.10+incompatible h1:lA4Pi29JEVIQIgATSeftHSY0rMGI9CLrl2ZvDLiahto= github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index cf45c46da260a..3c315e1a83248 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -80,6 +80,7 @@ type CheckTable struct { Table table.Table IndexInfos []*model.IndexInfo IndexLookUpReaders []*PhysicalIndexLookUpReader + CheckIndex bool } // RecoverIndex is used for backfilling corrupted index data. @@ -98,15 +99,6 @@ type CleanupIndex struct { IndexName string } -// CheckIndex is used for checking index data, built from the 'admin check index' statement. -type CheckIndex struct { - baseSchemaProducer - - IndexLookUpReader *PhysicalIndexLookUpReader - DBName string - IdxName string -} - // CheckIndexRange is used for checking index data, output the index values that handle within begin and end. type CheckIndexRange struct { baseSchemaProducer diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 4cc682e664ace..e0af2970325b7 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -884,53 +884,15 @@ func (b *PlanBuilder) buildPrepare(x *ast.PrepareStmt) Plan { return p } -func (b *PlanBuilder) buildCheckIndex(ctx context.Context, dbName model.CIStr, as *ast.AdminStmt) (Plan, error) { - tblName := as.Tables[0] - tbl, err := b.is.TableByName(dbName, tblName.Name) - if err != nil { - return nil, err - } - tblInfo := tbl.Meta() - - // get index information - var idx *model.IndexInfo - for _, index := range tblInfo.Indices { - if index.Name.L == strings.ToLower(as.Index) { - idx = index - break - } - } - if idx == nil { - return nil, errors.Errorf("index %s do not exist", as.Index) - } - if idx.State != model.StatePublic { - return nil, errors.Errorf("index %s state %s isn't public", as.Index, idx.State) - } - - return b.buildPhysicalIndexLookUpReader(ctx, dbName, tbl, idx) -} - func (b *PlanBuilder) buildAdmin(ctx context.Context, as *ast.AdminStmt) (Plan, error) { var ret Plan var err error switch as.Tp { - case ast.AdminCheckTable: + case ast.AdminCheckTable, ast.AdminCheckIndex: ret, err = b.buildAdminCheckTable(ctx, as) if err != nil { return ret, err } - case ast.AdminCheckIndex: - dbName := as.Tables[0].Schema - readerPlan, err := b.buildCheckIndex(ctx, dbName, as) - if err != nil { - return ret, err - } - - ret = &CheckIndex{ - DBName: dbName.L, - IdxName: as.Index, - IndexLookUpReader: readerPlan.(*PhysicalIndexLookUpReader), - } case ast.AdminRecoverIndex: p := &RecoverIndex{Table: as.Tables[0], IndexName: as.Index} p.setSchemaAndNames(buildRecoverIndexFields()) @@ -1167,12 +1129,12 @@ func (b *PlanBuilder) buildPhysicalIndexLookUpReader(ctx context.Context, dbName return rootT.p, nil } -func (b *PlanBuilder) buildPhysicalIndexLookUpReaders(ctx context.Context, dbName model.CIStr, tbl table.Table) ([]Plan, []*model.IndexInfo, error) { +func (b *PlanBuilder) buildPhysicalIndexLookUpReaders(ctx context.Context, dbName model.CIStr, tbl table.Table, indices []table.Index) ([]Plan, []*model.IndexInfo, error) { tblInfo := tbl.Meta() // get index information indexInfos := make([]*model.IndexInfo, 0, len(tblInfo.Indices)) indexLookUpReaders := make([]Plan, 0, len(tblInfo.Indices)) - for _, idx := range tbl.Indices() { + for _, idx := range indices { idxInfo := idx.Meta() if idxInfo.State != model.StatePublic { logutil.Logger(context.Background()).Info("build physical index lookup reader, the index isn't public", @@ -1206,6 +1168,79 @@ func (b *PlanBuilder) buildPhysicalIndexLookUpReaders(ctx context.Context, dbNam } func (b *PlanBuilder) buildAdminCheckTable(ctx context.Context, as *ast.AdminStmt) (*CheckTable, error) { + tblName := as.Tables[0] + tableInfo := as.Tables[0].TableInfo + tbl, ok := b.is.TableByID(tableInfo.ID) + if !ok { + return nil, infoschema.ErrTableNotExists.GenWithStackByArgs(tblName.DBInfo.Name.O, tableInfo.Name.O) + } + p := &CheckTable{ + DBName: tblName.Schema.O, + Table: tbl, + } + var readerPlans []Plan + var indexInfos []*model.IndexInfo + var err error + if as.Tp == ast.AdminCheckIndex { + // get index information + var idx table.Index + idxName := strings.ToLower(as.Index) + for _, index := range tbl.Indices() { + if index.Meta().Name.L == idxName { + idx = index + break + } + } + if idx == nil { + return nil, errors.Errorf("index %s do not exist", as.Index) + } + if idx.Meta().State != model.StatePublic { + return nil, errors.Errorf("index %s state %s isn't public", as.Index, idx.Meta().State) + } + p.CheckIndex = true + readerPlans, indexInfos, err = b.buildPhysicalIndexLookUpReaders(ctx, tblName.Schema, tbl, []table.Index{idx}) + } else { + readerPlans, indexInfos, err = b.buildPhysicalIndexLookUpReaders(ctx, tblName.Schema, tbl, tbl.Indices()) + } + if err != nil { + return nil, errors.Trace(err) + } + readers := make([]*PhysicalIndexLookUpReader, 0, len(readerPlans)) + for _, plan := range readerPlans { + readers = append(readers, plan.(*PhysicalIndexLookUpReader)) + } + p.IndexInfos = indexInfos + p.IndexLookUpReaders = readers + return p, nil +} + +func (b *PlanBuilder) buildCheckIndex(ctx context.Context, dbName model.CIStr, as *ast.AdminStmt) (Plan, error) { + tblName := as.Tables[0] + tbl, err := b.is.TableByName(dbName, tblName.Name) + if err != nil { + return nil, err + } + tblInfo := tbl.Meta() + + // get index information + var idx *model.IndexInfo + for _, index := range tblInfo.Indices { + if index.Name.L == strings.ToLower(as.Index) { + idx = index + break + } + } + if idx == nil { + return nil, errors.Errorf("index %s do not exist", as.Index) + } + if idx.State != model.StatePublic { + return nil, errors.Errorf("index %s state %s isn't public", as.Index, idx.State) + } + + return b.buildPhysicalIndexLookUpReader(ctx, dbName, tbl, idx) +} + +func (b *PlanBuilder) buildAdminCheckIndex(ctx context.Context, as *ast.AdminStmt) (*CheckTable, error) { tbl := as.Tables[0] tableInfo := as.Tables[0].TableInfo table, ok := b.is.TableByID(tableInfo.ID) @@ -1216,7 +1251,7 @@ func (b *PlanBuilder) buildAdminCheckTable(ctx context.Context, as *ast.AdminStm DBName: tbl.Schema.O, Table: table, } - readerPlans, indexInfos, err := b.buildPhysicalIndexLookUpReaders(ctx, tbl.Schema, table) + readerPlans, indexInfos, err := b.buildPhysicalIndexLookUpReaders(ctx, tbl.Schema, table, table.Indices()) if err != nil { return nil, errors.Trace(err) } From 5087279dd10ed5835abb0bd0e8effb838026fc80 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 13 May 2020 21:59:52 +0800 Subject: [PATCH 2/6] go.mod: tidy Signed-off-by: crazycs520 --- go.sum | 1 - 1 file changed, 1 deletion(-) diff --git a/go.sum b/go.sum index 4c47d4fb2dd40..85d08c7650c41 100644 --- a/go.sum +++ b/go.sum @@ -408,7 +408,6 @@ github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.19.10+incompatible h1:lA4Pi29JEVIQIgATSeftHSY0rMGI9CLrl2ZvDLiahto= github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= From 153526da9749b0e1e073add87e7e9c4f7808ec57 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 13 May 2020 22:02:28 +0800 Subject: [PATCH 3/6] fmt code Signed-off-by: crazycs520 --- planner/core/common_plans.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 3c315e1a83248..38f68735c389e 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -80,7 +80,7 @@ type CheckTable struct { Table table.Table IndexInfos []*model.IndexInfo IndexLookUpReaders []*PhysicalIndexLookUpReader - CheckIndex bool + CheckIndex bool } // RecoverIndex is used for backfilling corrupted index data. From e5a90bf241f0d85333ff9f55996769736bdb4856 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Wed, 13 May 2020 22:22:57 +0800 Subject: [PATCH 4/6] fix log Signed-off-by: crazycs520 --- executor/executor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/executor.go b/executor/executor.go index 8aa15799ee0e1..c719b966d4216 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -750,7 +750,7 @@ func (e *CheckTableExec) Next(ctx context.Context, req *chunk.Chunk) error { util.WithRecovery(func() { err1 := e.checkIndexHandle(ctx, e.srcs[num]) if err1 != nil { - logutil.Logger(ctx).Info("check index handle failed", zap.Error(err)) + logutil.Logger(ctx).Info("check index handle failed", zap.Error(err1)) } }, e.handlePanic) }(i) From 54c6044e3f42954f5dc4a8524f748fc0b8617343 Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Thu, 14 May 2020 12:07:22 +0800 Subject: [PATCH 5/6] update test Signed-off-by: crazycs520 --- executor/admin_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/executor/admin_test.go b/executor/admin_test.go index 75eeb577a138c..b9058001ccb9f 100644 --- a/executor/admin_test.go +++ b/executor/admin_test.go @@ -61,7 +61,7 @@ func (s *testSuite5) TestAdminCheckIndex(c *C) { // Test for hash partition table. tk.MustExec("drop table if exists admin_test") tk.MustExec("create table admin_test (c1 int, c2 int, c3 int default 1, index (c1), unique key(c2)) partition by hash(c2) partitions 5;") - tk.MustExec("insert admin_test (c1, c2) values (1, 1), (2, 2), (NULL, NULL)") + check() // Test for range partition table. tk.MustExec("drop table if exists admin_test") @@ -69,7 +69,7 @@ func (s *testSuite5) TestAdminCheckIndex(c *C) { PARTITION p0 VALUES LESS THAN (5), PARTITION p1 VALUES LESS THAN (10), PARTITION p2 VALUES LESS THAN (MAXVALUE))`) - tk.MustExec("insert admin_test (c1, c2) values (1, 1), (2, 2), (NULL, NULL)") + check() } func (s *testSuite5) TestAdminRecoverIndex(c *C) { From 5cc821e2b610429a1546ce5250e38c0b8503a28a Mon Sep 17 00:00:00 2001 From: crazycs520 Date: Thu, 14 May 2020 21:54:07 +0800 Subject: [PATCH 6/6] address comment Signed-off-by: crazycs520 --- planner/core/planbuilder.go | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index e0af2970325b7..c0be82b573636 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1240,30 +1240,6 @@ func (b *PlanBuilder) buildCheckIndex(ctx context.Context, dbName model.CIStr, a return b.buildPhysicalIndexLookUpReader(ctx, dbName, tbl, idx) } -func (b *PlanBuilder) buildAdminCheckIndex(ctx context.Context, as *ast.AdminStmt) (*CheckTable, error) { - tbl := as.Tables[0] - tableInfo := as.Tables[0].TableInfo - table, ok := b.is.TableByID(tableInfo.ID) - if !ok { - return nil, infoschema.ErrTableNotExists.GenWithStackByArgs(tbl.DBInfo.Name.O, tableInfo.Name.O) - } - p := &CheckTable{ - DBName: tbl.Schema.O, - Table: table, - } - readerPlans, indexInfos, err := b.buildPhysicalIndexLookUpReaders(ctx, tbl.Schema, table, table.Indices()) - if err != nil { - return nil, errors.Trace(err) - } - readers := make([]*PhysicalIndexLookUpReader, 0, len(readerPlans)) - for _, plan := range readerPlans { - readers = append(readers, plan.(*PhysicalIndexLookUpReader)) - } - p.IndexInfos = indexInfos - p.IndexLookUpReaders = readers - return p, nil -} - func (b *PlanBuilder) buildCheckIndexSchema(tn *ast.TableName, indexName string) (*expression.Schema, types.NameSlice, error) { schema := expression.NewSchema() var names types.NameSlice