From 13b02cc802539a5017ae31f740293208651924ba Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 19 Mar 2020 14:34:17 +0800 Subject: [PATCH 1/3] model: add schema name, table name to ddl job. (#11561) (#15450) --- ddl/ddl_api.go | 27 +++++++++++++++++++++++ executor/executor.go | 26 +++++++++++++++++++--- executor/executor_test.go | 44 +++++++++++++++++++++++++++++++++++-- planner/core/planbuilder.go | 3 ++- 4 files changed, 94 insertions(+), 6 deletions(-) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 87da2f312dd6a..45320bce44fb0 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -85,6 +85,7 @@ func (d *ddl) CreateSchema(ctx sessionctx.Context, schema model.CIStr, charsetIn job := &model.Job{ SchemaID: schemaID, + SchemaName: dbInfo.Name.L, Type: model.ActionCreateSchema, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{dbInfo}, @@ -143,6 +144,7 @@ func (d *ddl) AlterSchema(ctx sessionctx.Context, stmt *ast.AlterDatabaseStmt) ( // Do the DDL job. job := &model.Job{ SchemaID: dbInfo.ID, + SchemaName: dbInfo.Name.L, Type: model.ActionModifySchemaCharsetAndCollate, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{toCharset, toCollate}, @@ -160,6 +162,7 @@ func (d *ddl) DropSchema(ctx sessionctx.Context, schema model.CIStr) (err error) } job := &model.Job{ SchemaID: old.ID, + SchemaName: old.Name.L, Type: model.ActionDropSchema, BinlogInfo: &model.HistoryInfo{}, } @@ -1225,6 +1228,7 @@ func (d *ddl) CreateTableWithLike(ctx sessionctx.Context, ident, referIdent ast. job := &model.Job{ SchemaID: schema.ID, TableID: tblInfo.ID, + SchemaName: schema.Name.L, Type: model.ActionCreateTable, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{tblInfo}, @@ -1397,6 +1401,7 @@ func (d *ddl) CreateTable(ctx sessionctx.Context, s *ast.CreateTableStmt) (err e job := &model.Job{ SchemaID: schema.ID, TableID: tbInfo.ID, + SchemaName: schema.Name.L, Type: model.ActionCreateTable, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{tbInfo}, @@ -1463,6 +1468,7 @@ func (d *ddl) RecoverTable(ctx sessionctx.Context, tbInfo *model.TableInfo, sche job := &model.Job{ SchemaID: schemaID, TableID: tbInfo.ID, + SchemaName: schema.Name.L, Type: model.ActionRecoverTable, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{tbInfo, autoID, dropJobID, snapshotTS, recoverTableCheckFlagNone}, @@ -1529,6 +1535,7 @@ func (d *ddl) CreateView(ctx sessionctx.Context, s *ast.CreateViewStmt) (err err job := &model.Job{ SchemaID: schema.ID, TableID: tbInfo.ID, + SchemaName: schema.Name.L, Type: model.ActionCreateView, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{tbInfo, s.OrReplace, oldViewTblID}, @@ -1996,6 +2003,7 @@ func (d *ddl) RebaseAutoID(ctx sessionctx.Context, ident ast.Ident, newBase int6 job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionRebaseAutoID, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{newBase}, @@ -2026,6 +2034,7 @@ func (d *ddl) ShardRowID(ctx sessionctx.Context, tableIdent ast.Ident, uVal uint Type: model.ActionShardRowID, SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{uVal}, } @@ -2142,6 +2151,7 @@ func (d *ddl) AddColumn(ctx sessionctx.Context, ti ast.Ident, spec *ast.AlterTab job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionAddColumn, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{col, spec.Position, 0}, @@ -2187,6 +2197,7 @@ func (d *ddl) AddTablePartitions(ctx sessionctx.Context, ident ast.Ident, spec * job := &model.Job{ SchemaID: schema.ID, TableID: meta.ID, + SchemaName: schema.Name.L, Type: model.ActionAddTablePartition, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{partInfo}, @@ -2259,6 +2270,7 @@ func (d *ddl) TruncateTablePartition(ctx sessionctx.Context, ident ast.Ident, sp job := &model.Job{ SchemaID: schema.ID, TableID: meta.ID, + SchemaName: schema.Name.L, Type: model.ActionTruncateTablePartition, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{pid}, @@ -2301,6 +2313,7 @@ func (d *ddl) DropTablePartition(ctx sessionctx.Context, ident ast.Ident, spec * job := &model.Job{ SchemaID: schema.ID, TableID: meta.ID, + SchemaName: schema.Name.L, Type: model.ActionDropTablePartition, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{partName}, @@ -2339,6 +2352,7 @@ func (d *ddl) DropColumn(ctx sessionctx.Context, ti ast.Ident, colName model.CIS job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionDropColumn, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{colName}, @@ -2676,6 +2690,7 @@ func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, or job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionModifyColumn, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{&newCol, originalColName, spec.Position, modifyColumnTp}, @@ -2810,6 +2825,7 @@ func (d *ddl) AlterColumn(ctx sessionctx.Context, ident ast.Ident, spec *ast.Alt job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionSetDefaultValue, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{col}, @@ -2836,6 +2852,7 @@ func (d *ddl) AlterTableComment(ctx sessionctx.Context, ident ast.Ident, spec *a job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionModifyTableComment, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{spec.Comment}, @@ -2887,6 +2904,7 @@ func (d *ddl) AlterTableCharsetAndCollate(ctx sessionctx.Context, ident ast.Iden job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionModifyTableCharsetAndCollate, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{toCharset, toCollate}, @@ -2979,6 +2997,7 @@ func (d *ddl) RenameIndex(ctx sessionctx.Context, ident ast.Ident, spec *ast.Alt job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionRenameIndex, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{spec.FromKey, spec.ToKey}, @@ -3003,6 +3022,7 @@ func (d *ddl) DropTable(ctx sessionctx.Context, ti ast.Ident) (err error) { job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionDropTable, BinlogInfo: &model.HistoryInfo{}, } @@ -3026,6 +3046,7 @@ func (d *ddl) DropView(ctx sessionctx.Context, ti ast.Ident) (err error) { job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionDropView, BinlogInfo: &model.HistoryInfo{}, } @@ -3048,6 +3069,7 @@ func (d *ddl) TruncateTable(ctx sessionctx.Context, ti ast.Ident) error { job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionTruncateTable, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{newTableID}, @@ -3097,6 +3119,7 @@ func (d *ddl) RenameTable(ctx sessionctx.Context, oldIdent, newIdent ast.Ident, job := &model.Job{ SchemaID: newSchema.ID, TableID: oldTbl.Meta().ID, + SchemaName: newSchema.Name.L, Type: model.ActionRenameTable, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{oldSchema.ID, newIdent.Name}, @@ -3229,6 +3252,7 @@ func (d *ddl) CreateIndex(ctx sessionctx.Context, ti ast.Ident, unique bool, ind job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionAddIndex, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{unique, indexName, idxColNames, indexOption}, @@ -3285,6 +3309,7 @@ func (d *ddl) CreateForeignKey(ctx sessionctx.Context, ti ast.Ident, fkName mode job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionAddForeignKey, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{fkInfo}, @@ -3311,6 +3336,7 @@ func (d *ddl) DropForeignKey(ctx sessionctx.Context, ti ast.Ident, fkName model. job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionDropForeignKey, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{fkName}, @@ -3371,6 +3397,7 @@ func (d *ddl) DropIndex(ctx sessionctx.Context, ti ast.Ident, indexName model.CI SchemaID: schema.ID, TableID: t.Meta().ID, Type: jobTp, + SchemaName: schema.Name.L, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{indexName}, } diff --git a/executor/executor.go b/executor/executor.go index 34d9fd852f60e..3732b5df0fd2c 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -411,15 +411,35 @@ func (e *ShowDDLJobsExec) Next(ctx context.Context, req *chunk.Chunk) error { numCurBatch := mathutil.Min(req.Capacity(), len(e.jobs)-e.cursor) for i := e.cursor; i < e.cursor+numCurBatch; i++ { req.AppendInt64(0, e.jobs[i].ID) - req.AppendString(1, getSchemaName(e.is, e.jobs[i].SchemaID)) - req.AppendString(2, getTableName(e.is, e.jobs[i].TableID)) + schemaName := e.jobs[i].SchemaName + tableName := "" + finishTS := uint64(0) + if e.jobs[i].BinlogInfo != nil { + finishTS = e.jobs[i].BinlogInfo.FinishedTS + if e.jobs[i].BinlogInfo.TableInfo != nil { + tableName = e.jobs[i].BinlogInfo.TableInfo.Name.L + } + if len(schemaName) == 0 && e.jobs[i].BinlogInfo.DBInfo != nil { + schemaName = e.jobs[i].BinlogInfo.DBInfo.Name.L + } + } + // For compatibility, the old version of DDL Job wasn't store the schema name and table name. + if len(schemaName) == 0 { + schemaName = getSchemaName(e.is, e.jobs[i].SchemaID) + } + if len(tableName) == 0 { + tableName = getTableName(e.is, e.jobs[i].TableID) + } + req.AppendString(1, schemaName) + req.AppendString(2, tableName) req.AppendString(3, e.jobs[i].Type.String()) req.AppendString(4, e.jobs[i].SchemaState.String()) req.AppendInt64(5, e.jobs[i].SchemaID) req.AppendInt64(6, e.jobs[i].TableID) req.AppendInt64(7, e.jobs[i].RowCount) req.AppendString(8, model.TSConvert2Time(e.jobs[i].StartTS).String()) - req.AppendString(9, e.jobs[i].State.String()) + req.AppendString(9, model.TSConvert2Time(finishTS).String()) + req.AppendString(10, e.jobs[i].State.String()) } e.cursor += numCurBatch return nil diff --git a/executor/executor_test.go b/executor/executor_test.go index 9656df0ac490d..08a3e802eb218 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -216,7 +216,7 @@ func (s *testSuite) TestAdmin(c *C) { err = r.Next(ctx, req) c.Assert(err, IsNil) row = req.GetRow(0) - c.Assert(row.Len(), Equals, 10) + c.Assert(row.Len(), Equals, 11) txn, err = s.store.Begin() c.Assert(err, IsNil) historyJobs, err := admin.GetHistoryDDLJobs(txn, admin.DefNumHistoryJobs) @@ -232,7 +232,7 @@ func (s *testSuite) TestAdmin(c *C) { err = r.Next(ctx, req) c.Assert(err, IsNil) row = req.GetRow(0) - c.Assert(row.Len(), Equals, 10) + c.Assert(row.Len(), Equals, 11) c.Assert(row.GetInt64(0), Equals, historyJobs[0].ID) c.Assert(err, IsNil) @@ -301,6 +301,13 @@ func (s *testSuite) TestAdmin(c *C) { tk.MustExec("ALTER TABLE t1 ADD INDEX idx3 (c4);") tk.MustExec("admin check table t1;") + // Test admin show ddl jobs table name after table has been droped. + tk.MustExec("drop table if exists t1;") + re := tk.MustQuery("admin show ddl jobs 1") + rows := re.Rows() + c.Assert(len(rows), Equals, 1) + c.Assert(rows[0][2], Equals, "t1") + // Test for reverse scan get history ddl jobs when ddl history jobs queue has multiple regions. txn, err = s.store.Begin() c.Assert(err, IsNil) @@ -318,6 +325,39 @@ func (s *testSuite) TestAdmin(c *C) { c.Assert(historyJobs, DeepEquals, historyJobs2) } +func (s *testSuite) TestAdminShowDDLJobs(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database if not exists test_admin_show_ddl_jobs") + tk.MustExec("use test_admin_show_ddl_jobs") + tk.MustExec("create table t (a int);") + + re := tk.MustQuery("admin show ddl jobs 1") + row := re.Rows()[0] + c.Assert(row[1], Equals, "test_admin_show_ddl_jobs") + jobID, err := strconv.Atoi(row[0].(string)) + c.Assert(err, IsNil) + + c.Assert(tk.Se.NewTxn(context.Background()), IsNil) + txn, err := tk.Se.Txn(true) + c.Assert(err, IsNil) + t := meta.NewMeta(txn) + job, err := t.GetHistoryDDLJob(int64(jobID)) + c.Assert(err, IsNil) + c.Assert(job, NotNil) + // Test for compatibility. Old TiDB version doesn't have SchemaName field, and the BinlogInfo maybe nil. + // See PR: 11561. + job.BinlogInfo = nil + job.SchemaName = "" + err = t.AddHistoryDDLJob(job, true) + c.Assert(err, IsNil) + err = tk.Se.CommitTxn(context.Background()) + c.Assert(err, IsNil) + + re = tk.MustQuery("admin show ddl jobs 1") + row = re.Rows()[0] + c.Assert(row[1], Equals, "test_admin_show_ddl_jobs") +} + func (s *testSuite) TestAdminChecksumOfPartitionedTable(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("USE test;") diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 62351c1d8bbe5..d65f121be0bce 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1169,7 +1169,7 @@ func buildCleanupIndexFields() *expression.Schema { } func buildShowDDLJobsFields() *expression.Schema { - schema := expression.NewSchema(make([]*expression.Column, 0, 10)...) + schema := expression.NewSchema(make([]*expression.Column, 0, 11)...) schema.Append(buildColumn("", "JOB_ID", mysql.TypeLonglong, 4)) schema.Append(buildColumn("", "DB_NAME", mysql.TypeVarchar, 64)) schema.Append(buildColumn("", "TABLE_NAME", mysql.TypeVarchar, 64)) @@ -1179,6 +1179,7 @@ func buildShowDDLJobsFields() *expression.Schema { schema.Append(buildColumn("", "TABLE_ID", mysql.TypeLonglong, 4)) schema.Append(buildColumn("", "ROW_COUNT", mysql.TypeLonglong, 4)) schema.Append(buildColumn("", "START_TIME", mysql.TypeVarchar, 64)) + schema.Append(buildColumn("", "END_TIME", mysql.TypeVarchar, 64)) schema.Append(buildColumn("", "STATE", mysql.TypeVarchar, 64)) return schema } From 5c2d9e0e7732d32afcfe6f697edb4ce4a11885c6 Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Wed, 15 Apr 2020 20:40:21 +0800 Subject: [PATCH 2/3] update parser --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3de5f3b514397..1ec9d5d289f73 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20200311073257-e53d835099b0 github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd - github.com/pingcap/parser v3.0.12-0.20200317072324-41ea4b21f5aa+incompatible + github.com/pingcap/parser v3.0.13-0.20200415122949-7873549f01a5+incompatible github.com/pingcap/pd v1.1.0-beta.0.20191223090411-ea2b748f6ee2 github.com/pingcap/tidb-tools v3.0.6-0.20191119150227-ff0a3c6e5763+incompatible github.com/pingcap/tipb v0.0.0-20200401051346-bec3080a5428 diff --git a/go.sum b/go.sum index 4c3474346a76a..6c0aa1e210407 100644 --- a/go.sum +++ b/go.sum @@ -159,6 +159,8 @@ github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd h1:hWDol43WY5PGhsh3+87 github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= github.com/pingcap/parser v3.0.12-0.20200317072324-41ea4b21f5aa+incompatible h1:i8348dPpUM748ZtMPHvjCgagg/By7OJlzXHKgkc1tyY= github.com/pingcap/parser v3.0.12-0.20200317072324-41ea4b21f5aa+incompatible/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v3.0.13-0.20200415122949-7873549f01a5+incompatible h1:YEfvpsi69NTq6icPNU+WLdqowCXJWLhM00UD2qyRQkc= +github.com/pingcap/parser v3.0.13-0.20200415122949-7873549f01a5+incompatible/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v1.1.0-beta.0.20191223090411-ea2b748f6ee2 h1:NL23b8tsg6M1QpSQedK14/Jx++QeyKL2rGiBvXAQVfA= github.com/pingcap/pd v1.1.0-beta.0.20191223090411-ea2b748f6ee2/go.mod h1:b4gaAPSxaVVtaB+EHamV4Nsv8JmTdjlw0cTKmp4+dRQ= github.com/pingcap/tidb-tools v3.0.6-0.20191119150227-ff0a3c6e5763+incompatible h1:I8HirWsu1MZp6t9G/g8yKCEjJJxtHooKakEgccvdJ4M= From 134a996597e7b8f830071dc6dcc63a67e64f0fdc Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Wed, 15 Apr 2020 20:42:11 +0800 Subject: [PATCH 3/3] go mod tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 6c0aa1e210407..4621e13ae386e 100644 --- a/go.sum +++ b/go.sum @@ -157,8 +157,6 @@ github.com/pingcap/kvproto v0.0.0-20200311073257-e53d835099b0 h1:dXXNHvDwAEN1YNg github.com/pingcap/kvproto v0.0.0-20200311073257-e53d835099b0/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd h1:hWDol43WY5PGhsh3+8794bFHY1bPrmu6bTalpssCrGg= github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v3.0.12-0.20200317072324-41ea4b21f5aa+incompatible h1:i8348dPpUM748ZtMPHvjCgagg/By7OJlzXHKgkc1tyY= -github.com/pingcap/parser v3.0.12-0.20200317072324-41ea4b21f5aa+incompatible/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/parser v3.0.13-0.20200415122949-7873549f01a5+incompatible h1:YEfvpsi69NTq6icPNU+WLdqowCXJWLhM00UD2qyRQkc= github.com/pingcap/parser v3.0.13-0.20200415122949-7873549f01a5+incompatible/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v1.1.0-beta.0.20191223090411-ea2b748f6ee2 h1:NL23b8tsg6M1QpSQedK14/Jx++QeyKL2rGiBvXAQVfA=