From 4a4acbe5fdf999eed3af72e03160e005a3c9fd51 Mon Sep 17 00:00:00 2001 From: rebelice Date: Tue, 5 Jan 2021 15:02:19 +0800 Subject: [PATCH 01/16] statistics: GC index usage information (#21014) --- domain/domain.go | 14 +++++++++++--- statistics/handle/handle_test.go | 33 ++++++++++++++++++++++++++++++-- statistics/handle/update.go | 10 ++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/domain/domain.go b/domain/domain.go index dd60effb3cbaa..f4692f71814cf 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -1104,14 +1104,14 @@ func (do *Domain) UpdateTableStatsLoop(ctx sessionctx.Context) error { do.wg.Add(1) go do.loadStatsWorker() } + owner := do.newOwnerManager(handle.StatsPrompt, handle.StatsOwnerKey) if do.indexUsageSyncLease > 0 { do.wg.Add(1) - go do.syncIndexUsageWorker() + go do.syncIndexUsageWorker(owner) } if do.statsLease <= 0 { return nil } - owner := do.newOwnerManager(handle.StatsPrompt, handle.StatsOwnerKey) do.wg.Add(1) do.SetStatsUpdating(true) go do.updateStatsWorker(ctx, owner) @@ -1179,9 +1179,10 @@ func (do *Domain) loadStatsWorker() { } } -func (do *Domain) syncIndexUsageWorker() { +func (do *Domain) syncIndexUsageWorker(owner owner.Manager) { defer util.Recover(metrics.LabelDomain, "syncIndexUsageWorker", nil, false) idxUsageSyncTicker := time.NewTicker(do.indexUsageSyncLease) + gcStatsTicker := time.NewTicker(100 * do.indexUsageSyncLease) handle := do.StatsHandle() defer func() { idxUsageSyncTicker.Stop() @@ -1197,6 +1198,13 @@ func (do *Domain) syncIndexUsageWorker() { if err := handle.DumpIndexUsageToKV(); err != nil { logutil.BgLogger().Debug("dump index usage failed", zap.Error(err)) } + case <-gcStatsTicker.C: + if !owner.IsOwner() { + continue + } + if err := handle.GCIndexUsage(); err != nil { + logutil.BgLogger().Error("[stats] gc index usage failed", zap.Error(err)) + } } } } diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index a62d8524482f4..16fc00896c344 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -809,11 +809,16 @@ func (s *testStatsSuite) TestCorrelationStatsCompute(c *C) { c.Assert(foundS1 && foundS2, IsTrue) } -func (s *testStatsSuite) TestIndexUsageInformation(c *C) { +var _ = SerialSuites(&statsSerialSuite{&testStatsSuite{}}) + +type statsSerialSuite struct { + *testStatsSuite +} + +func (s *statsSerialSuite) TestIndexUsageInformation(c *C) { defer cleanEnv(c, s.store, s.do) tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - tk.MustExec("drop table if exists t_idx") tk.MustExec("create table t_idx(a int, b int)") tk.MustExec("create unique index idx_a on t_idx(a)") tk.MustExec("create unique index idx_b on t_idx(b)") @@ -848,3 +853,27 @@ func (s *testStatsSuite) TestIndexUsageInformation(c *C) { "test t_idx idx_b 2 2", )) } + +func (s *statsSerialSuite) TestGCIndexUsageInformation(c *C) { + defer cleanEnv(c, s.store, s.do) + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("create table t_idx(a int, b int)") + tk.MustExec("create unique index idx_a on t_idx(a)") + tk.MustQuery("select a from t_idx where a=1") + do := s.do + err := do.StatsHandle().DumpIndexUsageToKV() + c.Assert(err, IsNil) + querySQL := `select count(distinct idx.table_schema, idx.table_name, idx.key_name, stats.query_count, stats.rows_selected) + from mysql.schema_index_usage as stats, information_schema.tidb_indexes as idx, information_schema.tables as tables + where tables.table_schema = idx.table_schema + AND tables.table_name = idx.table_name + AND tables.tidb_table_id = stats.table_id + AND idx.index_id = stats.index_id + AND idx.table_name = "t_idx"` + tk.MustQuery(querySQL).Check(testkit.Rows("1")) + tk.MustExec("drop index `idx_a` on t_idx") + err = do.StatsHandle().GCIndexUsage() + c.Assert(err, IsNil) + tk.MustQuery(querySQL).Check(testkit.Rows("0")) +} diff --git a/statistics/handle/update.go b/statistics/handle/update.go index 7ac34a55ef260..dbc06d0595a26 100644 --- a/statistics/handle/update.go +++ b/statistics/handle/update.go @@ -320,6 +320,16 @@ func (h *Handle) DumpIndexUsageToKV() error { return nil } +// GCIndexUsage will delete the usage information of those indexes that do not exist. +func (h *Handle) GCIndexUsage() error { + // For performance and implementation reasons, mysql.schema_index_usage doesn't handle DDL. + // We periodically delete the usage information of non-existent indexes through information_schema.tidb_indexes. + // This sql will delete the usage information of those indexes that not in information_schema.tidb_indexes. + sql := `delete from mysql.SCHEMA_INDEX_USAGE as stats where stats.index_id not in (select idx.index_id from information_schema.tidb_indexes as idx)` + _, _, err := h.restrictedExec.ExecRestrictedSQL(sql) + return err +} + var ( // DumpStatsDeltaRatio is the lower bound of `Modify Count / Table Count` for stats delta to be dumped. DumpStatsDeltaRatio = 1 / 10000.0 From 029eef87ed2e11968f3c96f65020670506ba36f3 Mon Sep 17 00:00:00 2001 From: Zhou Kunqin <25057648+time-and-fate@users.noreply.github.com> Date: Tue, 5 Jan 2021 16:30:33 +0800 Subject: [PATCH 02/16] planner, expression: fix error when using IN combined with subquery (#22080) --- expression/builtin_other.go | 123 ++++++++++++---------- expression/builtin_other_vec_generated.go | 58 ++++++---- expression/generator/other_vec.go | 9 +- expression/scalar_function.go | 23 ---- planner/core/integration_test.go | 9 ++ 5 files changed, 120 insertions(+), 102 deletions(-) diff --git a/expression/builtin_other.go b/expression/builtin_other.go index 513dab9f206ad..faa41cd0a256a 100644 --- a/expression/builtin_other.go +++ b/expression/builtin_other.go @@ -155,8 +155,10 @@ func (c *inFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) type baseInSig struct { baseBuiltinFunc - nonConstArgs []Expression - hasNull bool + // nonConstArgsIdx stores the indices of non-constant args in the baseBuiltinFunc.args (the first arg is not included). + // It works with builtinInXXXSig.hashset to accelerate 'eval'. + nonConstArgsIdx []int + hasNull bool } // builtinInIntSig see https://dev.mysql.com/doc/refman/5.7/en/comparison-operators.html#function_in @@ -167,7 +169,7 @@ type builtinInIntSig struct { } func (b *builtinInIntSig) buildHashMapForConstArgs(ctx sessionctx.Context) error { - b.nonConstArgs = []Expression{b.args[0]} + b.nonConstArgsIdx = make([]int, 0) b.hashSet = make(map[int64]bool, len(b.args)-1) for i := 1; i < len(b.args); i++ { if b.args[i].ConstItem(b.ctx.GetSessionVars().StmtCtx) { @@ -181,7 +183,7 @@ func (b *builtinInIntSig) buildHashMapForConstArgs(ctx sessionctx.Context) error } b.hashSet[val] = mysql.HasUnsignedFlag(b.args[i].GetType().Flag) } else { - b.nonConstArgs = append(b.nonConstArgs, b.args[i]) + b.nonConstArgsIdx = append(b.nonConstArgsIdx, i) } } return nil @@ -190,10 +192,8 @@ func (b *builtinInIntSig) buildHashMapForConstArgs(ctx sessionctx.Context) error func (b *builtinInIntSig) Clone() builtinFunc { newSig := &builtinInIntSig{} newSig.cloneFrom(&b.baseBuiltinFunc) - newSig.nonConstArgs = make([]Expression, 0, len(b.nonConstArgs)) - for _, arg := range b.nonConstArgs { - newSig.nonConstArgs = append(newSig.nonConstArgs, arg.Clone()) - } + newSig.nonConstArgsIdx = make([]int, len(b.nonConstArgsIdx)) + copy(newSig.nonConstArgsIdx, b.nonConstArgsIdx) newSig.hashSet = b.hashSet newSig.hasNull = b.hasNull return newSig @@ -206,9 +206,8 @@ func (b *builtinInIntSig) evalInt(row chunk.Row) (int64, bool, error) { } isUnsigned0 := mysql.HasUnsignedFlag(b.args[0].GetType().Flag) - args := b.args + args := b.args[1:] if len(b.hashSet) != 0 { - args = b.nonConstArgs if isUnsigned, ok := b.hashSet[arg0]; ok { if (isUnsigned0 && isUnsigned) || (!isUnsigned0 && !isUnsigned) { return 1, false, nil @@ -217,10 +216,14 @@ func (b *builtinInIntSig) evalInt(row chunk.Row) (int64, bool, error) { return 1, false, nil } } + args = args[:0] + for _, i := range b.nonConstArgsIdx { + args = append(args, b.args[i]) + } } hasNull := b.hasNull - for _, arg := range args[1:] { + for _, arg := range args { evaledArg, isNull, err := arg.EvalInt(b.ctx, row) if err != nil { return 0, true, err @@ -258,7 +261,7 @@ type builtinInStringSig struct { } func (b *builtinInStringSig) buildHashMapForConstArgs(ctx sessionctx.Context) error { - b.nonConstArgs = []Expression{b.args[0]} + b.nonConstArgsIdx = make([]int, 0) b.hashSet = set.NewStringSet() collator := collate.GetCollator(b.collation) for i := 1; i < len(b.args); i++ { @@ -273,7 +276,7 @@ func (b *builtinInStringSig) buildHashMapForConstArgs(ctx sessionctx.Context) er } b.hashSet.Insert(string(collator.Key(val))) // should do memory copy here } else { - b.nonConstArgs = append(b.nonConstArgs, b.args[i]) + b.nonConstArgsIdx = append(b.nonConstArgsIdx, i) } } @@ -283,10 +286,8 @@ func (b *builtinInStringSig) buildHashMapForConstArgs(ctx sessionctx.Context) er func (b *builtinInStringSig) Clone() builtinFunc { newSig := &builtinInStringSig{} newSig.cloneFrom(&b.baseBuiltinFunc) - newSig.nonConstArgs = make([]Expression, 0, len(b.nonConstArgs)) - for _, arg := range b.nonConstArgs { - newSig.nonConstArgs = append(newSig.nonConstArgs, arg.Clone()) - } + newSig.nonConstArgsIdx = make([]int, len(b.nonConstArgsIdx)) + copy(newSig.nonConstArgsIdx, b.nonConstArgsIdx) newSig.hashSet = b.hashSet newSig.hasNull = b.hasNull return newSig @@ -298,17 +299,20 @@ func (b *builtinInStringSig) evalInt(row chunk.Row) (int64, bool, error) { return 0, isNull0, err } - args := b.args + args := b.args[1:] collator := collate.GetCollator(b.collation) if len(b.hashSet) != 0 { - args = b.nonConstArgs if b.hashSet.Exist(string(collator.Key(arg0))) { return 1, false, nil } + args = args[:0] + for _, i := range b.nonConstArgsIdx { + args = append(args, b.args[i]) + } } hasNull := b.hasNull - for _, arg := range args[1:] { + for _, arg := range args { evaledArg, isNull, err := arg.EvalString(b.ctx, row) if err != nil { return 0, true, err @@ -331,7 +335,7 @@ type builtinInRealSig struct { } func (b *builtinInRealSig) buildHashMapForConstArgs(ctx sessionctx.Context) error { - b.nonConstArgs = []Expression{b.args[0]} + b.nonConstArgsIdx = make([]int, 0) b.hashSet = set.NewFloat64Set() for i := 1; i < len(b.args); i++ { if b.args[i].ConstItem(b.ctx.GetSessionVars().StmtCtx) { @@ -345,7 +349,7 @@ func (b *builtinInRealSig) buildHashMapForConstArgs(ctx sessionctx.Context) erro } b.hashSet.Insert(val) } else { - b.nonConstArgs = append(b.nonConstArgs, b.args[i]) + b.nonConstArgsIdx = append(b.nonConstArgsIdx, i) } } @@ -355,10 +359,8 @@ func (b *builtinInRealSig) buildHashMapForConstArgs(ctx sessionctx.Context) erro func (b *builtinInRealSig) Clone() builtinFunc { newSig := &builtinInRealSig{} newSig.cloneFrom(&b.baseBuiltinFunc) - newSig.nonConstArgs = make([]Expression, 0, len(b.nonConstArgs)) - for _, arg := range b.nonConstArgs { - newSig.nonConstArgs = append(newSig.nonConstArgs, arg.Clone()) - } + newSig.nonConstArgsIdx = make([]int, len(b.nonConstArgsIdx)) + copy(newSig.nonConstArgsIdx, b.nonConstArgsIdx) newSig.hashSet = b.hashSet newSig.hasNull = b.hasNull return newSig @@ -369,15 +371,19 @@ func (b *builtinInRealSig) evalInt(row chunk.Row) (int64, bool, error) { if isNull0 || err != nil { return 0, isNull0, err } - args := b.args + args := b.args[1:] if len(b.hashSet) != 0 { - args = b.nonConstArgs if b.hashSet.Exist(arg0) { return 1, false, nil } + args = args[:0] + for _, i := range b.nonConstArgsIdx { + args = append(args, b.args[i]) + } } + hasNull := b.hasNull - for _, arg := range args[1:] { + for _, arg := range args { evaledArg, isNull, err := arg.EvalReal(b.ctx, row) if err != nil { return 0, true, err @@ -400,7 +406,7 @@ type builtinInDecimalSig struct { } func (b *builtinInDecimalSig) buildHashMapForConstArgs(ctx sessionctx.Context) error { - b.nonConstArgs = []Expression{b.args[0]} + b.nonConstArgsIdx = make([]int, 0) b.hashSet = set.NewStringSet() for i := 1; i < len(b.args); i++ { if b.args[i].ConstItem(b.ctx.GetSessionVars().StmtCtx) { @@ -418,7 +424,7 @@ func (b *builtinInDecimalSig) buildHashMapForConstArgs(ctx sessionctx.Context) e } b.hashSet.Insert(string(key)) } else { - b.nonConstArgs = append(b.nonConstArgs, b.args[i]) + b.nonConstArgsIdx = append(b.nonConstArgsIdx, i) } } @@ -428,10 +434,8 @@ func (b *builtinInDecimalSig) buildHashMapForConstArgs(ctx sessionctx.Context) e func (b *builtinInDecimalSig) Clone() builtinFunc { newSig := &builtinInDecimalSig{} newSig.cloneFrom(&b.baseBuiltinFunc) - newSig.nonConstArgs = make([]Expression, 0, len(b.nonConstArgs)) - for _, arg := range b.nonConstArgs { - newSig.nonConstArgs = append(newSig.nonConstArgs, arg.Clone()) - } + newSig.nonConstArgsIdx = make([]int, len(b.nonConstArgsIdx)) + copy(newSig.nonConstArgsIdx, b.nonConstArgsIdx) newSig.hashSet = b.hashSet newSig.hasNull = b.hasNull return newSig @@ -443,20 +447,23 @@ func (b *builtinInDecimalSig) evalInt(row chunk.Row) (int64, bool, error) { return 0, isNull0, err } - args := b.args + args := b.args[1:] key, err := arg0.ToHashKey() if err != nil { return 0, true, err } if len(b.hashSet) != 0 { - args = b.nonConstArgs if b.hashSet.Exist(string(key)) { return 1, false, nil } + args = args[:0] + for _, i := range b.nonConstArgsIdx { + args = append(args, b.args[i]) + } } hasNull := b.hasNull - for _, arg := range args[1:] { + for _, arg := range args { evaledArg, isNull, err := arg.EvalDecimal(b.ctx, row) if err != nil { return 0, true, err @@ -479,7 +486,7 @@ type builtinInTimeSig struct { } func (b *builtinInTimeSig) buildHashMapForConstArgs(ctx sessionctx.Context) error { - b.nonConstArgs = []Expression{b.args[0]} + b.nonConstArgsIdx = make([]int, 0) b.hashSet = make(map[types.CoreTime]struct{}, len(b.args)-1) for i := 1; i < len(b.args); i++ { if b.args[i].ConstItem(b.ctx.GetSessionVars().StmtCtx) { @@ -493,7 +500,7 @@ func (b *builtinInTimeSig) buildHashMapForConstArgs(ctx sessionctx.Context) erro } b.hashSet[val.CoreTime()] = struct{}{} } else { - b.nonConstArgs = append(b.nonConstArgs, b.args[i]) + b.nonConstArgsIdx = append(b.nonConstArgsIdx, i) } } @@ -503,10 +510,8 @@ func (b *builtinInTimeSig) buildHashMapForConstArgs(ctx sessionctx.Context) erro func (b *builtinInTimeSig) Clone() builtinFunc { newSig := &builtinInTimeSig{} newSig.cloneFrom(&b.baseBuiltinFunc) - newSig.nonConstArgs = make([]Expression, 0, len(b.nonConstArgs)) - for _, arg := range b.nonConstArgs { - newSig.nonConstArgs = append(newSig.nonConstArgs, arg.Clone()) - } + newSig.nonConstArgsIdx = make([]int, len(b.nonConstArgsIdx)) + copy(newSig.nonConstArgsIdx, b.nonConstArgsIdx) newSig.hashSet = b.hashSet newSig.hasNull = b.hasNull return newSig @@ -517,15 +522,19 @@ func (b *builtinInTimeSig) evalInt(row chunk.Row) (int64, bool, error) { if isNull0 || err != nil { return 0, isNull0, err } - args := b.args + args := b.args[1:] if len(b.hashSet) != 0 { - args = b.nonConstArgs if _, ok := b.hashSet[arg0.CoreTime()]; ok { return 1, false, nil } + args = args[:0] + for _, i := range b.nonConstArgsIdx { + args = append(args, b.args[i]) + } } + hasNull := b.hasNull - for _, arg := range args[1:] { + for _, arg := range args { evaledArg, isNull, err := arg.EvalTime(b.ctx, row) if err != nil { return 0, true, err @@ -548,7 +557,7 @@ type builtinInDurationSig struct { } func (b *builtinInDurationSig) buildHashMapForConstArgs(ctx sessionctx.Context) error { - b.nonConstArgs = []Expression{b.args[0]} + b.nonConstArgsIdx = make([]int, 0) b.hashSet = make(map[time.Duration]struct{}, len(b.args)-1) for i := 1; i < len(b.args); i++ { if b.args[i].ConstItem(b.ctx.GetSessionVars().StmtCtx) { @@ -562,7 +571,7 @@ func (b *builtinInDurationSig) buildHashMapForConstArgs(ctx sessionctx.Context) } b.hashSet[val.Duration] = struct{}{} } else { - b.nonConstArgs = append(b.nonConstArgs, b.args[i]) + b.nonConstArgsIdx = append(b.nonConstArgsIdx, i) } } @@ -572,10 +581,8 @@ func (b *builtinInDurationSig) buildHashMapForConstArgs(ctx sessionctx.Context) func (b *builtinInDurationSig) Clone() builtinFunc { newSig := &builtinInDurationSig{} newSig.cloneFrom(&b.baseBuiltinFunc) - newSig.nonConstArgs = make([]Expression, 0, len(b.nonConstArgs)) - for _, arg := range b.nonConstArgs { - newSig.nonConstArgs = append(newSig.nonConstArgs, arg.Clone()) - } + newSig.nonConstArgsIdx = make([]int, len(b.nonConstArgsIdx)) + copy(newSig.nonConstArgsIdx, b.nonConstArgsIdx) newSig.hashSet = b.hashSet newSig.hasNull = b.hasNull return newSig @@ -586,15 +593,19 @@ func (b *builtinInDurationSig) evalInt(row chunk.Row) (int64, bool, error) { if isNull0 || err != nil { return 0, isNull0, err } - args := b.args + args := b.args[1:] if len(b.hashSet) != 0 { - args = b.nonConstArgs if _, ok := b.hashSet[arg0.Duration]; ok { return 1, false, nil } + args = args[:0] + for _, i := range b.nonConstArgsIdx { + args = append(args, b.args[i]) + } } + hasNull := b.hasNull - for _, arg := range args[1:] { + for _, arg := range args { evaledArg, isNull, err := arg.EvalDuration(b.ctx, row) if err != nil { return 0, true, err diff --git a/expression/builtin_other_vec_generated.go b/expression/builtin_other_vec_generated.go index e44f2f6759ca0..0cdf900f0793d 100644 --- a/expression/builtin_other_vec_generated.go +++ b/expression/builtin_other_vec_generated.go @@ -53,9 +53,8 @@ func (b *builtinInIntSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) e } isUnsigned0 := mysql.HasUnsignedFlag(b.args[0].GetType().Flag) var compareResult int - args := b.args + args := b.args[1:] if len(b.hashSet) != 0 { - args = b.nonConstArgs for i := 0; i < n; i++ { if buf0.IsNull(i) { hasNull[i] = true @@ -73,9 +72,13 @@ func (b *builtinInIntSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) e } } } + args = args[:0] + for _, i := range b.nonConstArgsIdx { + args = append(args, b.args[i]) + } } - for j := 1; j < len(args); j++ { + for j := 0; j < len(args); j++ { if err := args[j].VecEvalInt(b.ctx, input, buf1); err != nil { return err } @@ -153,10 +156,9 @@ func (b *builtinInStringSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column } } var compareResult int - args := b.args + args := b.args[1:] if len(b.hashSet) != 0 { collator := collate.GetCollator(b.collation) - args = b.nonConstArgs for i := 0; i < n; i++ { if buf0.IsNull(i) { hasNull[i] = true @@ -168,9 +170,13 @@ func (b *builtinInStringSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column result.SetNull(i, false) } } + args = args[:0] + for _, i := range b.nonConstArgsIdx { + args = append(args, b.args[i]) + } } - for j := 1; j < len(args); j++ { + for j := 0; j < len(args); j++ { if err := args[j].VecEvalString(b.ctx, input, buf1); err != nil { return err } @@ -232,9 +238,8 @@ func (b *builtinInDecimalSig) vecEvalInt(input *chunk.Chunk, result *chunk.Colum } } var compareResult int - args := b.args + args := b.args[1:] if len(b.hashSet) != 0 { - args = b.nonConstArgs for i := 0; i < n; i++ { if buf0.IsNull(i) { hasNull[i] = true @@ -250,9 +255,13 @@ func (b *builtinInDecimalSig) vecEvalInt(input *chunk.Chunk, result *chunk.Colum result.SetNull(i, false) } } + args = args[:0] + for _, i := range b.nonConstArgsIdx { + args = append(args, b.args[i]) + } } - for j := 1; j < len(args); j++ { + for j := 0; j < len(args); j++ { if err := args[j].VecEvalDecimal(b.ctx, input, buf1); err != nil { return err } @@ -319,9 +328,8 @@ func (b *builtinInRealSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) } } var compareResult int - args := b.args + args := b.args[1:] if len(b.hashSet) != 0 { - args = b.nonConstArgs for i := 0; i < n; i++ { if buf0.IsNull(i) { hasNull[i] = true @@ -333,9 +341,13 @@ func (b *builtinInRealSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) result.SetNull(i, false) } } + args = args[:0] + for _, i := range b.nonConstArgsIdx { + args = append(args, b.args[i]) + } } - for j := 1; j < len(args); j++ { + for j := 0; j < len(args); j++ { if err := args[j].VecEvalReal(b.ctx, input, buf1); err != nil { return err } @@ -399,9 +411,8 @@ func (b *builtinInTimeSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) } } var compareResult int - args := b.args + args := b.args[1:] if len(b.hashSet) != 0 { - args = b.nonConstArgs for i := 0; i < n; i++ { if buf0.IsNull(i) { hasNull[i] = true @@ -413,9 +424,13 @@ func (b *builtinInTimeSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) result.SetNull(i, false) } } + args = args[:0] + for _, i := range b.nonConstArgsIdx { + args = append(args, b.args[i]) + } } - for j := 1; j < len(args); j++ { + for j := 0; j < len(args); j++ { if err := args[j].VecEvalTime(b.ctx, input, buf1); err != nil { return err } @@ -479,9 +494,8 @@ func (b *builtinInDurationSig) vecEvalInt(input *chunk.Chunk, result *chunk.Colu } } var compareResult int - args := b.args + args := b.args[1:] if len(b.hashSet) != 0 { - args = b.nonConstArgs for i := 0; i < n; i++ { if buf0.IsNull(i) { hasNull[i] = true @@ -493,9 +507,13 @@ func (b *builtinInDurationSig) vecEvalInt(input *chunk.Chunk, result *chunk.Colu result.SetNull(i, false) } } + args = args[:0] + for _, i := range b.nonConstArgsIdx { + args = append(args, b.args[i]) + } } - for j := 1; j < len(args); j++ { + for j := 0; j < len(args); j++ { if err := args[j].VecEvalDuration(b.ctx, input, buf1); err != nil { return err } @@ -553,9 +571,9 @@ func (b *builtinInJSONSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) } hasNull := make([]bool, n) var compareResult int - args := b.args + args := b.args[1:] - for j := 1; j < len(args); j++ { + for j := 0; j < len(args); j++ { if err := args[j].VecEvalJSON(b.ctx, input, buf1); err != nil { return err } diff --git a/expression/generator/other_vec.go b/expression/generator/other_vec.go index 5096f1d8e5940..8016ba7619766 100644 --- a/expression/generator/other_vec.go +++ b/expression/generator/other_vec.go @@ -144,13 +144,12 @@ func (b *{{.SigName}}) vecEvalInt(input *chunk.Chunk, result *chunk.Column) erro isUnsigned0 := mysql.HasUnsignedFlag(b.args[0].GetType().Flag) {{- end }} var compareResult int - args := b.args + args := b.args[1:] {{- if not $InputJSON}} if len(b.hashSet) != 0 { {{- if $InputString }} collator := collate.GetCollator(b.collation) {{- end }} - args = b.nonConstArgs for i := 0; i < n; i++ { if buf0.IsNull(i) { hasNull[i] = true @@ -202,10 +201,14 @@ func (b *{{.SigName}}) vecEvalInt(input *chunk.Chunk, result *chunk.Column) erro {{- end }} {{- end }} } + args = args[:0] + for _, i := range b.nonConstArgsIdx { + args = append(args, b.args[i]) + } } {{- end }} - for j := 1; j < len(args); j++ { + for j := 0; j < len(args); j++ { if err := args[j].VecEval{{ .Input.TypeName }}(b.ctx, input, buf1); err != nil { return err } diff --git a/expression/scalar_function.go b/expression/scalar_function.go index 16aa6d948b7b5..57e0419c26a63 100644 --- a/expression/scalar_function.go +++ b/expression/scalar_function.go @@ -428,29 +428,6 @@ func (sf *ScalarFunction) ResolveIndices(schema *Schema) (Expression, error) { } func (sf *ScalarFunction) resolveIndices(schema *Schema) error { - if sf.FuncName.L == ast.In { - args := []Expression{} - switch inFunc := sf.Function.(type) { - case *builtinInIntSig: - args = inFunc.nonConstArgs - case *builtinInStringSig: - args = inFunc.nonConstArgs - case *builtinInTimeSig: - args = inFunc.nonConstArgs - case *builtinInDurationSig: - args = inFunc.nonConstArgs - case *builtinInRealSig: - args = inFunc.nonConstArgs - case *builtinInDecimalSig: - args = inFunc.nonConstArgs - } - for _, arg := range args { - err := arg.resolveIndices(schema) - if err != nil { - return err - } - } - } for _, arg := range sf.GetArgs() { err := arg.resolveIndices(schema) if err != nil { diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 255edbe26d76f..86d71826f4d89 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -2392,3 +2392,12 @@ func (s *testIntegrationSuite) TestIssue22105(c *C) { tk.MustQuery(tt).Check(testkit.Rows(output[i].Plan...)) } } + +func (s *testIntegrationSuite) TestIssue22071(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("create table t (a int);") + tk.MustExec("insert into t values(1),(2),(5)") + tk.MustQuery("select n in (1,2) from (select a in (1,2) as n from t) g;").Sort().Check(testkit.Rows("0", "1", "1")) + tk.MustQuery("select n in (1,n) from (select a in (1,2) as n from t) g;").Check(testkit.Rows("1", "1", "1")) +} From 6742ed201d16ea488f82f9d05cd3ea82fa60cc27 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Tue, 5 Jan 2021 17:01:39 +0800 Subject: [PATCH 03/16] planner/core: fix a bug of adding enforcer. (#22086) --- planner/cascades/implementation_rules.go | 2 +- planner/core/exhaust_physical_plans.go | 20 +++++----- planner/core/find_best_task.go | 31 ++++++++------- planner/core/find_best_task_test.go | 26 ++++++------ planner/core/physical_plan_test.go | 42 ++++++++++++++++++++ planner/core/plan.go | 2 +- planner/core/rule_inject_extra_projection.go | 6 +-- planner/core/testdata/plan_suite_in.json | 11 +++++ planner/core/testdata/plan_suite_out.json | 23 +++++++++++ planner/property/physical_property.go | 23 +++++------ 10 files changed, 130 insertions(+), 56 deletions(-) diff --git a/planner/cascades/implementation_rules.go b/planner/cascades/implementation_rules.go index 0327474cab431..d7a08b4fabaab 100644 --- a/planner/cascades/implementation_rules.go +++ b/planner/cascades/implementation_rules.go @@ -272,7 +272,7 @@ func (r *ImplSelection) OnImplement(expr *memo.GroupExpr, reqProp *property.Phys logicalSel := expr.ExprNode.(*plannercore.LogicalSelection) physicalSel := plannercore.PhysicalSelection{ Conditions: logicalSel.Conditions, - }.Init(logicalSel.SCtx(), expr.Group.Prop.Stats.ScaleByExpectCnt(reqProp.ExpectedCnt), logicalSel.SelectBlockOffset(), reqProp.Clone()) + }.Init(logicalSel.SCtx(), expr.Group.Prop.Stats.ScaleByExpectCnt(reqProp.ExpectedCnt), logicalSel.SelectBlockOffset(), reqProp.CloneEssentialFields()) switch expr.Group.EngineType { case memo.EngineTiDB: return []memo.Implementation{impl.NewTiDBSelectionImpl(physicalSel)}, nil diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 434b578038800..d00b64f73a2af 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -44,7 +44,7 @@ func (p *LogicalUnionScan) exhaustPhysicalPlans(prop *property.PhysicalProperty) if prop.IsFlashProp() { return nil, true } - childProp := prop.Clone() + childProp := prop.CloneEssentialFields() us := PhysicalUnionScan{ Conditions: p.conditions, HandleCols: p.handleCols, @@ -1736,7 +1736,7 @@ func (p *LogicalJoin) tryToGetMppHashJoin(prop *property.PhysicalProperty, useBC baseJoin.InnerChildIdx = preferredBuildIndex childrenProps := make([]*property.PhysicalProperty, 2) if useBCJ { - childrenProps[preferredBuildIndex] = &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64, PartitionTp: property.BroadcastType, Enforced: true} + childrenProps[preferredBuildIndex] = &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64, PartitionTp: property.BroadcastType, CanAddEnforcer: true} expCnt := math.MaxFloat64 if prop.ExpectedCnt < p.stats.RowCount { expCntScale := prop.ExpectedCnt / p.stats.RowCount @@ -1767,8 +1767,8 @@ func (p *LogicalJoin) tryToGetMppHashJoin(prop *property.PhysicalProperty, useBC lkeys = chooseSubsetOfJoinKeys(lkeys, matches) rkeys = chooseSubsetOfJoinKeys(rkeys, matches) } - childrenProps[0] = &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64, PartitionTp: property.HashType, PartitionCols: lkeys, Enforced: true} - childrenProps[1] = &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64, PartitionTp: property.HashType, PartitionCols: rkeys, Enforced: true} + childrenProps[0] = &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64, PartitionTp: property.HashType, PartitionCols: lkeys, CanAddEnforcer: true} + childrenProps[1] = &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64, PartitionTp: property.HashType, PartitionCols: rkeys, CanAddEnforcer: true} } join := PhysicalHashJoin{ basePhysicalJoin: baseJoin, @@ -2074,7 +2074,7 @@ func (p *LogicalWindow) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([ var byItems []property.SortItem byItems = append(byItems, p.PartitionBy...) byItems = append(byItems, p.OrderBy...) - childProperty := &property.PhysicalProperty{ExpectedCnt: math.MaxFloat64, SortItems: byItems, Enforced: true} + childProperty := &property.PhysicalProperty{ExpectedCnt: math.MaxFloat64, SortItems: byItems, CanAddEnforcer: true} if !prop.IsPrefix(childProperty) { return nil, true } @@ -2111,9 +2111,9 @@ func (la *LogicalAggregation) getEnforcedStreamAggs(prop *property.PhysicalPrope allTaskTypes := prop.GetAllPossibleChildTaskTypes() enforcedAggs := make([]PhysicalPlan, 0, len(allTaskTypes)) childProp := &property.PhysicalProperty{ - ExpectedCnt: math.Max(prop.ExpectedCnt*la.inputCount/la.stats.RowCount, prop.ExpectedCnt), - Enforced: true, - SortItems: property.SortItemsFromCols(la.GetGroupByCols(), desc), + ExpectedCnt: math.Max(prop.ExpectedCnt*la.inputCount/la.stats.RowCount, prop.ExpectedCnt), + CanAddEnforcer: true, + SortItems: property.SortItemsFromCols(la.GetGroupByCols(), desc), } if !prop.IsPrefix(childProp) { return enforcedAggs @@ -2314,7 +2314,7 @@ func (la *LogicalAggregation) exhaustPhysicalPlans(prop *property.PhysicalProper } func (p *LogicalSelection) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { - childProp := prop.Clone() + childProp := prop.CloneEssentialFields() sel := PhysicalSelection{ Conditions: p.Conditions, }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, childProp) @@ -2366,7 +2366,7 @@ func (p *LogicalLock) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]P if prop.IsFlashProp() { return nil, true } - childProp := prop.Clone() + childProp := prop.CloneEssentialFields() lock := PhysicalLock{ Lock: p.Lock, TblID2Handle: p.tblID2Handle, diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index da1480236914e..58ecf757d55d6 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -200,7 +200,7 @@ func (p *baseLogicalPlan) rebuildChildTasks(childTasks *[]task, pp PhysicalPlan, return nil } -func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPlan, prop *property.PhysicalProperty, planCounter *PlanCounterTp) (task, int64, error) { +func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPlan, prop *property.PhysicalProperty, addEnforcer bool, planCounter *PlanCounterTp) (task, int64, error) { var bestTask task = invalidTask var curCntPlan, cntPlan int64 childTasks := make([]task, 0, len(p.children)) @@ -249,7 +249,7 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl } // Enforce curTask property - if prop.Enforced { + if addEnforcer { curTask = enforceProperty(prop, curTask, p.basePlan.ctx) } @@ -290,6 +290,8 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun return bestTask, 1, nil } + canAddEnforcer := prop.CanAddEnforcer + if prop.TaskTp != property.RootTaskType && !prop.IsFlashProp() { // Currently all plan cannot totally push down to TiKV. p.storeTask(prop, invalidTask) @@ -300,7 +302,7 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun cntPlan = 0 // prop should be read only because its cached hashcode might be not consistent // when it is changed. So we clone a new one for the temporary changes. - newProp := prop.Clone() + newProp := prop.CloneEssentialFields() var plansFitsProp, plansNeedEnforce []PhysicalPlan var hintWorksWithProp bool // Maybe the plan can satisfy the required property, @@ -310,10 +312,10 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun // If there is a hint in the plan and the hint cannot satisfy the property, // we enforce this property and try to generate the PhysicalPlan again to // make sure the hint can work. - newProp.Enforced = true + canAddEnforcer = true } - if newProp.Enforced { + if canAddEnforcer { // Then, we use the empty property to get physicalPlans and // try to get the task with an enforced sort. newProp.SortItems = []property.SortItem{} @@ -328,7 +330,7 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun // can work. plansFitsProp = nil } - if !hintCanWork && !hintWorksWithProp && !prop.Enforced { + if !hintCanWork && !hintWorksWithProp && !prop.CanAddEnforcer { // If the original property is not enforced and hint cannot // work anyway, we give up `plansNeedEnforce` for efficiency, plansNeedEnforce = nil @@ -336,10 +338,9 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun newProp = prop } - newProp.Enforced = false var cnt int64 var curTask task - if bestTask, cnt, err = p.enumeratePhysicalPlans4Task(plansFitsProp, newProp, planCounter); err != nil { + if bestTask, cnt, err = p.enumeratePhysicalPlans4Task(plansFitsProp, newProp, false, planCounter); err != nil { return nil, 0, err } cntPlan += cnt @@ -347,8 +348,7 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun goto END } - newProp.Enforced = true - curTask, cnt, err = p.enumeratePhysicalPlans4Task(plansNeedEnforce, newProp, planCounter) + curTask, cnt, err = p.enumeratePhysicalPlans4Task(plansNeedEnforce, newProp, true, planCounter) if err != nil { return nil, 0, err } @@ -617,15 +617,15 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter var cnt int64 // If prop.enforced is true, the prop.cols need to be set nil for ds.findBestTask. // Before function return, reset it for enforcing task prop and storing map. - oldProp := prop.Clone() - if prop.Enforced { + oldProp := prop.CloneEssentialFields() + if prop.CanAddEnforcer { // First, get the bestTask without enforced prop - prop.Enforced = false + prop.CanAddEnforcer = false t, cnt, err = ds.findBestTask(prop, planCounter) if err != nil { return nil, 0, err } - prop.Enforced = true + prop.CanAddEnforcer = true if t != invalidTask { ds.storeTask(prop, t) cntPlan = cnt @@ -641,9 +641,10 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter if err != nil { return } - if prop.Enforced { + if prop.CanAddEnforcer { prop = oldProp t = enforceProperty(prop, t, ds.basePlan.ctx) + prop.CanAddEnforcer = true } ds.storeTask(prop, t) if ds.SampleInfo != nil && !t.invalid() { diff --git a/planner/core/find_best_task_test.go b/planner/core/find_best_task_test.go index 6ea25bbbe0b04..dc83476d1cfac 100644 --- a/planner/core/find_best_task_test.go +++ b/planner/core/find_best_task_test.go @@ -84,7 +84,7 @@ func (p *mockLogicalPlan4Test) getPhysicalPlan1(prop *property.PhysicalProperty) physicalPlan1 := mockPhysicalPlan4Test{planType: 1, costOverflow: p.costOverflow}.Init(p.ctx) physicalPlan1.stats = &property.StatsInfo{RowCount: 1} physicalPlan1.childrenReqProps = make([]*property.PhysicalProperty, 1) - physicalPlan1.childrenReqProps[0] = prop.Clone() + physicalPlan1.childrenReqProps[0] = prop.CloneEssentialFields() return physicalPlan1 } @@ -175,8 +175,8 @@ func (s *testFindBestTaskSuite) TestEnforcedProperty(c *C) { items := []property.SortItem{item0, item1} prop0 := &property.PhysicalProperty{ - SortItems: items, - Enforced: false, + SortItems: items, + CanAddEnforcer: false, } // should return invalid task because no physical plan can match this property. task, _, err := mockPlan.findBestTask(prop0, &PlanCounterDisabled) @@ -184,8 +184,8 @@ func (s *testFindBestTaskSuite) TestEnforcedProperty(c *C) { c.Assert(task.invalid(), IsTrue) prop1 := &property.PhysicalProperty{ - SortItems: items, - Enforced: true, + SortItems: items, + CanAddEnforcer: true, } // should return the valid task when the property is enforced. task, _, err = mockPlan.findBestTask(prop1, &PlanCounterDisabled) @@ -208,8 +208,8 @@ func (s *testFindBestTaskSuite) TestHintCannotFitProperty(c *C) { items := []property.SortItem{item0} // case 1, The property is not empty and enforced, should enforce a sort. prop0 := &property.PhysicalProperty{ - SortItems: items, - Enforced: true, + SortItems: items, + CanAddEnforcer: true, } task, _, err := mockPlan0.findBestTask(prop0, &PlanCounterDisabled) c.Assert(err, IsNil) @@ -224,8 +224,8 @@ func (s *testFindBestTaskSuite) TestHintCannotFitProperty(c *C) { // case 2, The property is not empty but not enforced, still need to enforce a sort // to ensure the hint can work prop1 := &property.PhysicalProperty{ - SortItems: items, - Enforced: false, + SortItems: items, + CanAddEnforcer: false, } task, _, err = mockPlan0.findBestTask(prop1, &PlanCounterDisabled) c.Assert(err, IsNil) @@ -240,8 +240,8 @@ func (s *testFindBestTaskSuite) TestHintCannotFitProperty(c *C) { // case 3, The hint cannot work even if the property is empty, should return a warning // and generate physicalPlan1. prop2 := &property.PhysicalProperty{ - SortItems: items, - Enforced: false, + SortItems: items, + CanAddEnforcer: false, } mockPlan1 := mockLogicalPlan4Test{ hasHintForPlan2: true, @@ -261,8 +261,8 @@ func (s *testFindBestTaskSuite) TestHintCannotFitProperty(c *C) { // the same with case 3. ctx.GetSessionVars().StmtCtx.SetWarnings(nil) prop3 := &property.PhysicalProperty{ - SortItems: items, - Enforced: true, + SortItems: items, + CanAddEnforcer: true, } task, _, err = mockPlan1.findBestTask(prop3, &PlanCounterDisabled) c.Assert(err, IsNil) diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index e9a61aefb33b1..38bf9910e9b9e 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -559,6 +559,48 @@ func (s *testPlanSuite) TestIndexJoinUnionScan(c *C) { } } +func (s *testPlanSuite) TestMergeJoinUnionScan(c *C) { + defer testleak.AfterTest(c)() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + tk := testkit.NewTestKit(c, store) + defer func() { + dom.Close() + store.Close() + }() + + tk.MustExec("use test") + var input [][]string + var output []struct { + SQL []string + Plan []string + } + tk.MustExec("create table t1 (c_int int, c_str varchar(40), primary key (c_int))") + tk.MustExec("create table t2 (c_int int, c_str varchar(40), primary key (c_int))") + tk.MustExec("insert into t1 (`c_int`, `c_str`) values (11, 'keen williamson'), (10, 'gracious hermann')") + tk.MustExec("insert into t2 (`c_int`, `c_str`) values (10, 'gracious hermann')") + + s.testData.GetTestCases(c, &input, &output) + for i, ts := range input { + tk.MustExec("begin") + for j, tt := range ts { + if j != len(ts)-1 { + tk.MustExec(tt) + } + s.testData.OnRecord(func() { + output[i].SQL = ts + if j == len(ts)-1 { + output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + } + }) + if j == len(ts)-1 { + tk.MustQuery(tt).Check(testkit.Rows(output[i].Plan...)) + } + } + tk.MustExec("rollback") + } +} + func (s *testPlanSuite) TestDoSubquery(c *C) { defer testleak.AfterTest(c)() store, dom, err := newStoreWithBootstrap() diff --git a/planner/core/plan.go b/planner/core/plan.go index 08cf8fcd873c9..e5cd253ebb1a0 100644 --- a/planner/core/plan.go +++ b/planner/core/plan.go @@ -388,7 +388,7 @@ func (p *basePhysicalPlan) cloneWithSelf(newSelf PhysicalPlan) (*basePhysicalPla base.children = append(base.children, cloned) } for _, prop := range p.childrenReqProps { - base.childrenReqProps = append(base.childrenReqProps, prop.Clone()) + base.childrenReqProps = append(base.childrenReqProps, prop.CloneEssentialFields()) } return base, nil } diff --git a/planner/core/rule_inject_extra_projection.go b/planner/core/rule_inject_extra_projection.go index 62d6b17cbcd28..f10bc52dc44bc 100644 --- a/planner/core/rule_inject_extra_projection.go +++ b/planner/core/rule_inject_extra_projection.go @@ -149,7 +149,7 @@ func InjectProjBelowAgg(aggPlan PhysicalPlan, aggFuncs []*aggregation.AggFuncDes } child := aggPlan.Children()[0] - prop := aggPlan.GetChildReqProps(0).Clone() + prop := aggPlan.GetChildReqProps(0).CloneEssentialFields() proj := PhysicalProjection{ Exprs: projExprs, AvoidColumnEvaluator: false, @@ -216,7 +216,7 @@ func InjectProjBelowSort(p PhysicalPlan, orderByItems []*util.ByItems) PhysicalP item.Expr = newArg } - childProp := p.GetChildReqProps(0).Clone() + childProp := p.GetChildReqProps(0).CloneEssentialFields() bottomProj := PhysicalProjection{ Exprs: bottomProjExprs, AvoidColumnEvaluator: false, @@ -265,7 +265,7 @@ func TurnNominalSortIntoProj(p PhysicalPlan, onlyColumn bool, orderByItems []*ut bottomProjSchemaCols = append(bottomProjSchemaCols, newArg) } - childProp := p.GetChildReqProps(0).Clone() + childProp := p.GetChildReqProps(0).CloneEssentialFields() bottomProj := PhysicalProjection{ Exprs: bottomProjExprs, AvoidColumnEvaluator: false, diff --git a/planner/core/testdata/plan_suite_in.json b/planner/core/testdata/plan_suite_in.json index 1faae0dea1076..aa3a5b86b0e02 100644 --- a/planner/core/testdata/plan_suite_in.json +++ b/planner/core/testdata/plan_suite_in.json @@ -513,6 +513,17 @@ ] ] }, + { + "name": "TestMergeJoinUnionScan", + "cases": [ + [ + "insert into t2 values (11, 'amazing merkle')", + "insert into t2 values (12, 'amazing merkle')", + // Test Merge Join + UnionScan + TableScan. + "explain select /*+ MERGE_JOIN(t1,t2) */ * from t1, t2 where t1.c_int = t2.c_int and t1.c_int = t2.c_int order by t1.c_int, t2.c_str;" + ] + ] + }, { "name": "TestSemiJoinToInner", "cases": [ diff --git a/planner/core/testdata/plan_suite_out.json b/planner/core/testdata/plan_suite_out.json index adca4a5dc32c4..620d83d1a852f 100644 --- a/planner/core/testdata/plan_suite_out.json +++ b/planner/core/testdata/plan_suite_out.json @@ -1393,6 +1393,29 @@ } ] }, + { + "Name": "TestMergeJoinUnionScan", + "Cases": [ + { + "SQL": [ + "insert into t2 values (11, 'amazing merkle')", + "insert into t2 values (12, 'amazing merkle')", + "explain select /*+ MERGE_JOIN(t1,t2) */ * from t1, t2 where t1.c_int = t2.c_int and t1.c_int = t2.c_int order by t1.c_int, t2.c_str;" + ], + "Plan": [ + "Sort_8 12500.00 root test.t1.c_int, test.t2.c_str", + "└─MergeJoin_11 12500.00 root inner join, left key:test.t1.c_int, test.t1.c_int, right key:test.t2.c_int, test.t2.c_int", + " ├─Sort_19(Build) 10000.00 root test.t2.c_int, test.t2.c_int", + " │ └─UnionScan_16 10000.00 root ", + " │ └─TableReader_18 10000.00 root data:TableFullScan_17", + " │ └─TableFullScan_17 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", + " └─Sort_14(Probe) 10000.00 root test.t1.c_int, test.t1.c_int", + " └─TableReader_13 10000.00 root data:TableFullScan_12", + " └─TableFullScan_12 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo" + ] + } + ] + }, { "Name": "TestSemiJoinToInner", "Cases": [ diff --git a/planner/property/physical_property.go b/planner/property/physical_property.go index c5c47e452f353..8ddb1a6212437 100644 --- a/planner/property/physical_property.go +++ b/planner/property/physical_property.go @@ -66,8 +66,8 @@ type PhysicalProperty struct { // calculated when function "HashCode()" being called. hashcode []byte - // whether need to enforce property. - Enforced bool + // indicates that whether we are allowed to add an enforcer. + CanAddEnforcer bool // If the partition type is hash, the data should be reshuffled by partition cols. PartitionCols []*expression.Column @@ -79,10 +79,10 @@ type PhysicalProperty struct { // NewPhysicalProperty builds property from columns. func NewPhysicalProperty(taskTp TaskType, cols []*expression.Column, desc bool, expectCnt float64, enforced bool) *PhysicalProperty { return &PhysicalProperty{ - SortItems: SortItemsFromCols(cols, desc), - TaskTp: taskTp, - ExpectedCnt: expectCnt, - Enforced: enforced, + SortItems: SortItemsFromCols(cols, desc), + TaskTp: taskTp, + ExpectedCnt: expectCnt, + CanAddEnforcer: enforced, } } @@ -167,7 +167,7 @@ func (p *PhysicalProperty) HashCode() []byte { } hashcodeSize := 8 + 8 + 8 + (16+8)*len(p.SortItems) + 8 p.hashcode = make([]byte, 0, hashcodeSize) - if p.Enforced { + if p.CanAddEnforcer { p.hashcode = codec.EncodeInt(p.hashcode, 1) } else { p.hashcode = codec.EncodeInt(p.hashcode, 0) @@ -196,16 +196,13 @@ func (p *PhysicalProperty) String() string { return fmt.Sprintf("Prop{cols: %v, TaskTp: %s, expectedCount: %v}", p.SortItems, p.TaskTp, p.ExpectedCnt) } -// Clone returns a copy of PhysicalProperty. Currently, this function is only used to build new -// required property for children plan in `exhaustPhysicalPlans`, so we don't copy `Enforced` field -// because if `Enforced` is true, the `SortItems` must be empty now, this makes `Enforced` meaningless -// for children nodes. -func (p *PhysicalProperty) Clone() *PhysicalProperty { +// CloneEssentialFields returns a copy of PhysicalProperty. We only copy the essential fields that really indicate the +// property, specifically, `CanAddEnforcer` should not be included. +func (p *PhysicalProperty) CloneEssentialFields() *PhysicalProperty { prop := &PhysicalProperty{ SortItems: p.SortItems, TaskTp: p.TaskTp, ExpectedCnt: p.ExpectedCnt, - Enforced: p.Enforced, PartitionTp: p.PartitionTp, PartitionCols: p.PartitionCols, } From 85d8d0605f251dab065aed4e8e05b177bdb4dcc0 Mon Sep 17 00:00:00 2001 From: xiongjiwei Date: Tue, 5 Jan 2021 17:51:11 +0800 Subject: [PATCH 04/16] executor: fix select into outfile with year type column has no data (#22175) --- executor/select_into.go | 2 +- executor/select_into_test.go | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/executor/select_into.go b/executor/select_into.go index 2c65712b216e4..6afe7c66dde10 100644 --- a/executor/select_into.go +++ b/executor/select_into.go @@ -162,7 +162,7 @@ func (s *SelectIntoExec) dumpToOutfile() error { } s.fieldBuf = s.fieldBuf[:0] switch col.GetType().Tp { - case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong: + case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeYear: s.fieldBuf = strconv.AppendInt(s.fieldBuf, row.GetInt64(j), 10) case mysql.TypeLonglong: if mysql.HasUnsignedFlag(col.GetType().Flag) { diff --git a/executor/select_into_test.go b/executor/select_into_test.go index 4d9e4861a0496..c499db9bdd322 100644 --- a/executor/select_into_test.go +++ b/executor/select_into_test.go @@ -266,3 +266,16 @@ func (s *testSuite1) TestEscapeType(c *C) { cmpAndRm(`1,1,11,11,{"key": 11},11,11 `, outfile, c) } + +func (s *testSuite1) TestYearType(c *C) { + outfile := randomSelectFilePath("TestYearType") + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(time1 year(4) default '2030');") + tk.MustExec("insert into t values (2010), (2011), (2012);") + tk.MustExec("insert into t values ();") + + tk.MustExec(fmt.Sprintf("select * from t into outfile '%v' fields terminated by ',' optionally enclosed by '\"' lines terminated by '\\n';", outfile)) + cmpAndRm("2010\n2011\n2012\n2030\n", outfile, c) +} From f55e8f2bf8354b68f19863c94ed78e5fd9074b35 Mon Sep 17 00:00:00 2001 From: tangenta Date: Tue, 5 Jan 2021 18:18:19 +0800 Subject: [PATCH 05/16] table: fix insert into _tidb_rowid panic and rebase it if needed (#22062) --- ddl/column.go | 2 +- ddl/ddl_api.go | 4 +-- executor/insert_common.go | 75 ++++++++++++++++++++++++++++++++++++--- executor/rowid_test.go | 49 +++++++++++++++++++++++++ executor/write.go | 2 +- meta/autoid/autoid.go | 28 +++++++-------- 6 files changed, 136 insertions(+), 24 deletions(-) diff --git a/ddl/column.go b/ddl/column.go index adfd3521ded36..2dc842d0578bd 100644 --- a/ddl/column.go +++ b/ddl/column.go @@ -1481,7 +1481,7 @@ func adjustColumnInfoInModifyColumn( func checkAndApplyNewAutoRandomBits(job *model.Job, t *meta.Meta, tblInfo *model.TableInfo, newCol *model.ColumnInfo, oldName *model.CIStr, newAutoRandBits uint64) error { schemaID := job.SchemaID - newLayout := autoid.NewAutoRandomIDLayout(&newCol.FieldType, newAutoRandBits) + newLayout := autoid.NewShardIDLayout(&newCol.FieldType, newAutoRandBits) // GenAutoRandomID first to prevent concurrent update. _, err := t.GenAutoRandomID(schemaID, tblInfo.ID, 1) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 37a4de5eaeac2..0f4929b052f98 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -1337,7 +1337,7 @@ func setTableAutoRandomBits(ctx sessionctx.Context, tbInfo *model.TableInfo, col return errors.Trace(err) } - layout := autoid.NewAutoRandomIDLayout(col.Tp, autoRandBits) + layout := autoid.NewShardIDLayout(col.Tp, autoRandBits) if autoRandBits == 0 { return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomNonPositive) } else if autoRandBits > autoid.MaxAutoRandomBits { @@ -2526,7 +2526,7 @@ func (d *ddl) RebaseAutoID(ctx sessionctx.Context, ident ast.Ident, newBase int6 break } } - layout := autoid.NewAutoRandomIDLayout(&autoRandColTp, tbInfo.AutoRandomBits) + layout := autoid.NewShardIDLayout(&autoRandColTp, tbInfo.AutoRandomBits) if layout.IncrementalMask()&newBase != newBase { errMsg := fmt.Sprintf(autoid.AutoRandomRebaseOverflow, newBase, layout.IncrementalBitsCapacity()) return errors.Trace(ErrInvalidAutoRandom.GenWithStackByArgs(errMsg)) diff --git a/executor/insert_common.go b/executor/insert_common.go index a59171280d7dd..9270a3f89b400 100644 --- a/executor/insert_common.go +++ b/executor/insert_common.go @@ -151,7 +151,6 @@ func (e *InsertValues) initInsertColumns() error { return errors.Errorf("insert, update and replace statements for _tidb_rowid are not supported.") } e.hasExtraHandle = true - break } } @@ -554,6 +553,13 @@ func (e *InsertValues) fillColValue(ctx context.Context, datum types.Datum, idx } return d, nil } + if column.ID == model.ExtraHandleID && hasValue { + d, err := e.adjustImplicitRowID(ctx, datum, hasValue, column) + if err != nil { + return types.Datum{}, err + } + return d, nil + } if !hasValue { d, err := e.getColDefaultValue(idx, column) if e.handleErr(column, &datum, 0, err) != nil { @@ -572,7 +578,14 @@ func (e *InsertValues) fillColValue(ctx context.Context, datum types.Datum, idx // https://dev.mysql.com/doc/refman/8.0/en/innodb-auto-increment-handling.html func (e *InsertValues) fillRow(ctx context.Context, row []types.Datum, hasValue []bool) ([]types.Datum, error) { gCols := make([]*table.Column, 0) - for i, c := range e.Table.Cols() { + tCols := e.Table.Cols() + if e.hasExtraHandle { + col := &table.Column{} + col.ColumnInfo = model.NewExtraHandleColInfo() + col.ColumnInfo.Offset = len(tCols) + tCols = append(tCols, col) + } + for i, c := range tCols { var err error // Evaluate the generated columns later after real columns set if c.IsGenerated() { @@ -884,7 +897,7 @@ func (e *InsertValues) allocAutoRandomID(ctx context.Context, fieldType *types.F if err != nil { return 0, err } - layout := autoid.NewAutoRandomIDLayout(fieldType, tableInfo.AutoRandomBits) + layout := autoid.NewShardIDLayout(fieldType, tableInfo.AutoRandomBits) if tables.OverflowShardBits(autoRandomID, tableInfo.AutoRandomBits, layout.TypeBitsLength, layout.HasSignBit) { return 0, autoid.ErrAutoRandReadFailed } @@ -900,12 +913,66 @@ func (e *InsertValues) rebaseAutoRandomID(recordID int64, fieldType *types.Field alloc := e.Table.Allocators(e.ctx).Get(autoid.AutoRandomType) tableInfo := e.Table.Meta() - layout := autoid.NewAutoRandomIDLayout(fieldType, tableInfo.AutoRandomBits) + layout := autoid.NewShardIDLayout(fieldType, tableInfo.AutoRandomBits) autoRandomID := layout.IncrementalMask() & recordID return alloc.Rebase(tableInfo.ID, autoRandomID, true) } +func (e *InsertValues) adjustImplicitRowID(ctx context.Context, d types.Datum, hasValue bool, c *table.Column) (types.Datum, error) { + var err error + var recordID int64 + if !hasValue { + d.SetNull() + } + if !d.IsNull() { + recordID = d.GetInt64() + } + // Use the value if it's not null and not 0. + if recordID != 0 { + if !e.ctx.GetSessionVars().AllowWriteRowID { + return types.Datum{}, errors.Errorf("insert, update and replace statements for _tidb_rowid are not supported.") + } + err = e.rebaseImplicitRowID(recordID) + if err != nil { + return types.Datum{}, err + } + d.SetInt64(recordID) + return d, nil + } + // Change NULL to auto id. + // Change value 0 to auto id, if NoAutoValueOnZero SQL mode is not set. + if d.IsNull() || e.ctx.GetSessionVars().SQLMode&mysql.ModeNoAutoValueOnZero == 0 { + _, err := e.ctx.Txn(true) + if err != nil { + return types.Datum{}, errors.Trace(err) + } + intHandle, err := tables.AllocHandle(ctx, e.ctx, e.Table) + if err != nil { + return types.Datum{}, err + } + recordID = intHandle.IntValue() + } + err = setDatumAutoIDAndCast(e.ctx, &d, recordID, c) + if err != nil { + return types.Datum{}, err + } + return d, nil +} + +func (e *InsertValues) rebaseImplicitRowID(recordID int64) error { + if recordID < 0 { + return nil + } + alloc := e.Table.Allocators(e.ctx).Get(autoid.RowIDAllocType) + tableInfo := e.Table.Meta() + + layout := autoid.NewShardIDLayout(types.NewFieldType(mysql.TypeLonglong), tableInfo.ShardRowIDBits) + newTiDBRowIDBase := layout.IncrementalMask() & recordID + + return alloc.Rebase(tableInfo.ID, newTiDBRowIDBase, true) +} + func (e *InsertValues) handleWarning(err error) { sc := e.ctx.GetSessionVars().StmtCtx sc.AppendWarning(err) diff --git a/executor/rowid_test.go b/executor/rowid_test.go index 00b5bed0c3441..62a0c82be54b3 100644 --- a/executor/rowid_test.go +++ b/executor/rowid_test.go @@ -87,3 +87,52 @@ func (s *testSuite1) TestNotAllowWriteRowID(c *C) { // tk.MustExec("insert into tt (id, c, _tidb_rowid) values(30000,10,1);") // tk.MustExec("admin check table tt;") } + +// Test for https://github.com/pingcap/tidb/issues/22029. +func (s *testSuite3) TestExplicitInsertRowID(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("drop database if exists test_explicit_insert_rowid;") + tk.MustExec("create database test_explicit_insert_rowid;") + tk.MustExec("use test_explicit_insert_rowid;") + tk.MustExec("create table t (a int);") + tk.MustExec("set @@tidb_opt_write_row_id = true;") + + // Check that _tidb_rowid insertion success. + tk.MustExec("insert into t (_tidb_rowid, a) values (1, 1), (2, 2);") + tk.MustQuery("select *, _tidb_rowid from t;").Check(testkit.Rows("1 1", "2 2")) + // Test that explicitly insert _tidb_rowid will trigger rebase. + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(id varchar(10), c int);") + tk.MustExec("insert t values('one', 101), ('two', 102);") + tk.MustQuery("select *, _tidb_rowid from t;"). + Check(testkit.Rows("one 101 1", "two 102 2")) + // check that _tidb_rowid takes the values from the insert statement + tk.MustExec("insert t (id, c, _tidb_rowid) values ('three', 103, 9), ('four', 104, 16), ('five', 105, 5);") + tk.MustQuery("select *, _tidb_rowid from t where c > 102;"). + Check(testkit.Rows("five 105 5", "three 103 9", "four 104 16")) + // check that the new _tidb_rowid is rebased + tk.MustExec("insert t values ('six', 106), ('seven', 107);") + tk.MustQuery("select *, _tidb_rowid from t where c > 105;"). + Check(testkit.Rows("six 106 17", "seven 107 18")) + + // Check that shard_row_id_bits are ignored during rebase. + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int) shard_row_id_bits = 5;") + tk.MustExec("insert into t values (1);") + tk.MustQuery("select *, ((1 << (64-5-1)) - 1) & _tidb_rowid from t order by a;").Check(testkit.Rows("1 1")) + tk.MustExec("insert into t (a, _tidb_rowid) values (2, (1<<62)+5);") + tk.MustExec("insert into t values (3);") + tk.MustQuery("select *, ((1 << (64-5-1)) - 1) & _tidb_rowid from t order by a;").Check(testkit.Rows("1 1", "2 5", "3 6")) + tk.MustExec("insert into t (a, _tidb_rowid) values (4, null);") + tk.MustQuery("select *, ((1 << (64-5-1)) - 1) & _tidb_rowid from t order by a;").Check(testkit.Rows("1 1", "2 5", "3 6", "4 7")) + + tk.MustExec("delete from t;") + tk.MustExec("SET sql_mode=(SELECT CONCAT(@@sql_mode,',NO_AUTO_VALUE_ON_ZERO'));") + tk.MustExec("insert into t (a, _tidb_rowid) values (5, 0);") + tk.MustQuery("select *, ((1 << (64-5-1)) - 1) & _tidb_rowid from t order by a;").Check(testkit.Rows("5 0")) + tk.MustExec("SET sql_mode=(SELECT REPLACE(@@sql_mode,'NO_AUTO_VALUE_ON_ZERO',''));") + tk.MustExec("insert into t (a, _tidb_rowid) values (6, 0);") + tk.MustQuery("select *, ((1 << (64-5-1)) - 1) & _tidb_rowid from t order by a;").Check(testkit.Rows("5 0", "6 8")) + tk.MustExec("insert into t (_tidb_rowid, a) values (0, 7);") + tk.MustQuery("select *, ((1 << (64-5-1)) - 1) & _tidb_rowid from t order by a;").Check(testkit.Rows("5 0", "6 8", "7 9")) +} diff --git a/executor/write.go b/executor/write.go index 08ea3d454613f..5b34ef6149c0c 100644 --- a/executor/write.go +++ b/executor/write.go @@ -250,7 +250,7 @@ func rebaseAutoRandomValue(sctx sessionctx.Context, t table.Table, newData *type if recordID < 0 { return nil } - layout := autoid.NewAutoRandomIDLayout(&col.FieldType, tableInfo.AutoRandomBits) + layout := autoid.NewShardIDLayout(&col.FieldType, tableInfo.AutoRandomBits) // Set bits except incremental_bits to zero. recordID = recordID & (1< Date: Tue, 5 Jan 2021 20:44:41 +0800 Subject: [PATCH 06/16] planner: do not use indexMerge when the path only use a single index (#22168) --- planner/core/integration_test.go | 25 +++++++++++++++ planner/core/stats.go | 11 +++++++ .../core/testdata/integration_suite_in.json | 7 +++++ .../core/testdata/integration_suite_out.json | 31 ++++++++++++++++--- 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 86d71826f4d89..b7ede55ed52fc 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -2285,6 +2285,31 @@ func (s *testIntegrationSuite) TestConditionColPruneInPhysicalUnionScan(c *C) { Check(testkit.Rows("0")) } +func (s *testIntegrationSuite) TestInvalidHint(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("use test") + tk.MustExec("drop table if exists tt") + tk.MustExec("create table tt(a int, key(a));") + + var input []string + var output []struct { + SQL string + Plan []string + Warnings []string + } + s.testData.GetTestCases(c, &input, &output) + warning := "show warnings;" + for i, tt := range input { + s.testData.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + output[i].Warnings = s.testData.ConvertRowsToStrings(tk.MustQuery(warning).Rows()) + }) + tk.MustQuery(tt).Check(testkit.Rows(output[i].Plan...)) + } +} + // Test for issue https://github.com/pingcap/tidb/issues/18320 func (s *testIntegrationSuite) TestNonaggregateColumnWithSingleValueInOnlyFullGroupByMode(c *C) { tk := testkit.NewTestKit(c, s.store) diff --git a/planner/core/stats.go b/planner/core/stats.go index ebe7e4f046417..b1e156b186a43 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -424,6 +424,17 @@ func (ds *DataSource) generateIndexMergeOrPaths() { } partialPaths = append(partialPaths, partialPath) } + // If all of the partialPaths use the same index, we will not use the indexMerge. + singlePath := true + for i := len(partialPaths) - 1; i >= 1; i-- { + if partialPaths[i].Index != partialPaths[i-1].Index { + singlePath = false + break + } + } + if singlePath { + continue + } if len(partialPaths) > 1 { possiblePath := ds.buildIndexMergeOrPath(partialPaths, i) if possiblePath == nil { diff --git a/planner/core/testdata/integration_suite_in.json b/planner/core/testdata/integration_suite_in.json index c01d844bc2a0c..9b8720e3c3ce8 100644 --- a/planner/core/testdata/integration_suite_in.json +++ b/planner/core/testdata/integration_suite_in.json @@ -224,6 +224,13 @@ "select * from pt where id > 4 or c = 7" ] }, + { + "name": "TestInvalidHint", + "cases": [ + "explain select /*+ use_index_merge(tt) */ * from tt where a=10 or a=20;", + "explain select /*+ use_index_merge(tt) */ * from tt where a=15 or (a < 10 or a > 20);" + ] + }, { "name": "TestApproxPercentile", "cases": [ diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index b7ce27422bf5a..3ddf100bb663e 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -275,10 +275,8 @@ { "SQL": "explain select /*+ USE_INDEX_MERGE(t, primary) */ * from t where 1 or t.c", "Plan": [ - "IndexMerge_8 10000.00 root ", - "├─TableFullScan_5(Build) 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", - "├─TableRangeScan_6(Build) 6666.67 cop[tikv] table:t range:[-inf,0), (0,+inf], keep order:false, stats:pseudo", - "└─TableRowIDScan_7(Probe) 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + "TableReader_6 10000.00 root data:TableFullScan_5", + "└─TableFullScan_5 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" ] }, { @@ -1249,6 +1247,31 @@ } ] }, + { + "Name": "TestInvalidHint", + "Cases": [ + { + "SQL": "explain select /*+ use_index_merge(tt) */ * from tt where a=10 or a=20;", + "Plan": [ + "IndexReader_6 20.00 root index:IndexRangeScan_5", + "└─IndexRangeScan_5 20.00 cop[tikv] table:tt, index:a(a) range:[10,10], [20,20], keep order:false, stats:pseudo" + ], + "Warnings": [ + "Warning 1105 IndexMerge is inapplicable or disabled" + ] + }, + { + "SQL": "explain select /*+ use_index_merge(tt) */ * from tt where a=15 or (a < 10 or a > 20);", + "Plan": [ + "IndexReader_6 6666.67 root index:IndexRangeScan_5", + "└─IndexRangeScan_5 6666.67 cop[tikv] table:tt, index:a(a) range:[-inf,10), [15,15], (20,+inf], keep order:false, stats:pseudo" + ], + "Warnings": [ + "Warning 1105 IndexMerge is inapplicable or disabled" + ] + } + ] + }, { "Name": "TestApproxPercentile", "Cases": [ From 27a9b3e8f6182e231b5b389b7acb921018fe8c62 Mon Sep 17 00:00:00 2001 From: Zhou Kunqin <25057648+time-and-fate@users.noreply.github.com> Date: Wed, 6 Jan 2021 12:08:10 +0800 Subject: [PATCH 07/16] statistics: fix stack overflow when use DNF expr immediately after add column (#22169) --- statistics/selectivity.go | 13 +++++++++++++ statistics/selectivity_test.go | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/statistics/selectivity.go b/statistics/selectivity.go index 8f46ce1bb17d1..881a09a0ea6fb 100644 --- a/statistics/selectivity.go +++ b/statistics/selectivity.go @@ -289,6 +289,7 @@ func (coll *HistColl) Selectivity(ctx sessionctx.Context, exprs []expression.Exp // Now we try to cover those still not covered DNF conditions using independence assumption, // i.e., sel(condA or condB) = sel(condA) + sel(condB) - sel(condA) * sel(condB) if mask > 0 { + OUTER: for i, expr := range remainedExprs { if mask&(1< 7") + + // Test issue 22134 + // Information about column n will not be in stats immediately after this SQL executed. + // If we don't have a check against this, DNF condition could lead to infinite recursion in Selectivity(). + testKit.MustExec("alter table t add column n timestamp;") + testKit.MustExec("select * from t where n = '2000-01-01' or n = '2000-01-02';") } func (s *testStatsSuite) TestIndexEstimationCrossValidate(c *C) { From 2386702697c420527dffb00abd44a2fc2c00092a Mon Sep 17 00:00:00 2001 From: Arenatlx <314806019@qq.com> Date: Wed, 6 Jan 2021 13:34:35 +0800 Subject: [PATCH 08/16] executor: fix data too long in union statement with select null and select varchar (#22183) --- expression/collation.go | 5 +++++ expression/integration_test.go | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/expression/collation.go b/expression/collation.go index a7b1983a844d2..3d187143fe000 100644 --- a/expression/collation.go +++ b/expression/collation.go @@ -16,6 +16,7 @@ package expression import ( "github.com/pingcap/parser/ast" "github.com/pingcap/parser/charset" + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/logutil" @@ -177,6 +178,10 @@ func deriveCoercibilityForConstant(c *Constant) Coercibility { } func deriveCoercibilityForColumn(c *Column) Coercibility { + // For specified type null, it should return CoercibilityIgnorable, which means it got the lowest priority in DeriveCollationFromExprs. + if c.RetType.Tp == mysql.TypeNull { + return CoercibilityIgnorable + } if c.RetType.EvalType() != types.ETString { return CoercibilityNumeric } diff --git a/expression/integration_test.go b/expression/integration_test.go index 0d2e1fa86694a..fdc7875a61739 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -8648,3 +8648,23 @@ func (s *testIntegrationSerialSuite) TestCollationUnion(c *C) { res = tk.MustQuery("select cast('2010-09-09' as date) a union select '2010-09-09 ';") c.Check(len(res.Rows()), Equals, 1) } + +func (s *testIntegrationSerialSuite) TestCollationUnion2(c *C) { + // For issue 22179 + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a varchar(10))") + tk.MustExec("insert into t values('aaaaaaaaa'),('天王盖地虎宝塔镇河妖')") + tk.MustQuery("select * from t").Check(testkit.Rows("aaaaaaaaa", "天王盖地虎宝塔镇河妖")) + + // check the collation of sub query of union statement. + tk.MustQuery("select collation(a) from (select null as a) aaa").Check(testkit.Rows("binary")) + tk.MustQuery("select collation(a) from (select a from t limit 1) aaa").Check(testkit.Rows("utf8mb4_bin")) + + // Reverse sub query of union statement. + tk.MustQuery("select * from (select null as a union all select a from t) aaa order by a").Check(testkit.Rows("", "aaaaaaaaa", "天王盖地虎宝塔镇河妖")) + tk.MustQuery("select * from (select a from t) aaa union all select null as a order by a").Check(testkit.Rows("", "aaaaaaaaa", "天王盖地虎宝塔镇河妖")) + tk.MustExec("drop table if exists t") +} From 67da69b122933d8a8aebe8163042a2d0b513cce6 Mon Sep 17 00:00:00 2001 From: cfzjywxk Date: Wed, 6 Jan 2021 14:01:11 +0800 Subject: [PATCH 09/16] txn: change check logic for column type change for amend (#22150) --- session/pessimistic_test.go | 21 +++++++++++++++++++++ session/schema_amender.go | 11 +++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/session/pessimistic_test.go b/session/pessimistic_test.go index 36f5dc1693708..5109c78e3603a 100644 --- a/session/pessimistic_test.go +++ b/session/pessimistic_test.go @@ -2260,3 +2260,24 @@ func (s *testPessimisticSuite) TestAmendForUniqueIndex(c *C) { err = <-errCh c.Assert(err, Equals, nil) } + +func (s *testPessimisticSuite) TestAmendWithColumnTypeChange(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("set global tidb_enable_change_column_type = 1;") + defer func() { + tk.MustExec("set global tidb_enable_change_column_type = 0;") + }() + tk2 := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop database if exists test_db") + tk.MustExec("create database test_db") + tk.MustExec("use test_db") + tk2.MustExec("use test_db") + tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;") + + tk2.MustExec("drop table if exists t") + tk2.MustExec("create table t (id int primary key, v varchar(10));") + tk.MustExec("begin pessimistic") + tk.MustExec("insert into t values (1, \"123456789\")") + tk2.MustExec("alter table t modify column v varchar(5);") + c.Assert(tk.ExecToErr("commit"), NotNil) +} diff --git a/session/schema_amender.go b/session/schema_amender.go index 05fb6d6d77c6e..64b883dc15967 100644 --- a/session/schema_amender.go +++ b/session/schema_amender.go @@ -190,8 +190,8 @@ func colChangeAmendable(colAtStart *model.ColumnInfo, colAtCommit *model.ColumnI return nil } -// collectModifyColAmendOps is used to check if there is column change from nullable to not null by now. -// TODO allow column change from nullable to not null, and generate keys check operation. +// collectModifyColAmendOps is used to check if there is only column size increasing change.Other column type changes +// such as column change from nullable to not null or column type change are not supported by now. func (a *amendCollector) collectModifyColAmendOps(tblAtStart, tblAtCommit table.Table) ([]amendOp, error) { for _, colAtCommit := range tblAtCommit.Cols() { colAtStart := findColByID(tblAtStart, colAtCommit.ID) @@ -200,6 +200,13 @@ func (a *amendCollector) collectModifyColAmendOps(tblAtStart, tblAtCommit table. if err != nil { return nil, err } + } else { + // If the column could not be found in the original schema, it could not be decided if this column + // is newly added or modified from an original column.Report error to solve the issue + // https://github.com/pingcap/tidb/issues/21470. This change will make amend fail for adding column + // and modifying columns at the same time. + return nil, errors.Errorf("column=%v id=%v is not found for table=%v checking column modify", + colAtCommit.Name, colAtCommit.ID, tblAtCommit.Meta().Name.String()) } } return nil, nil From 8ef1031623f6e5cd92062fc820726832b92f1cf2 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 6 Jan 2021 14:17:34 +0800 Subject: [PATCH 10/16] *: remove tidb-lightning pkg to fix unexpected config change (#22164) --- executor/brie.go | 1 - go.mod | 8 +--- go.sum | 107 +++++------------------------------------- server/server_test.go | 8 ++++ server/tidb_test.go | 7 +++ 5 files changed, 28 insertions(+), 103 deletions(-) diff --git a/executor/brie.go b/executor/brie.go index 1852a7d23421d..4dd86aa9fdb1b 100644 --- a/executor/brie.go +++ b/executor/brie.go @@ -29,7 +29,6 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" - _ "github.com/pingcap/tidb-lightning/lightning" // stub that update go.mod filter "github.com/pingcap/tidb-tools/pkg/table-filter" pd "github.com/tikv/pd/client" diff --git a/go.mod b/go.mod index 4d0d38b2033ae..502864a3464d3 100644 --- a/go.mod +++ b/go.mod @@ -2,13 +2,10 @@ module github.com/pingcap/tidb require ( github.com/BurntSushi/toml v0.3.1 - github.com/DATA-DOG/go-sqlmock v1.5.0 // indirect github.com/HdrHistogram/hdrhistogram-go v0.9.0 // indirect github.com/Jeffail/gabs/v2 v2.5.1 github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d - github.com/carlmjohnson/flagext v0.20.2 // indirect github.com/cheggaaa/pb/v3 v3.0.4 // indirect - github.com/cockroachdb/pebble v0.0.0-20200617141519-3b241b76ed3b // indirect github.com/codahale/hdrhistogram v0.9.0 // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 @@ -28,7 +25,7 @@ require ( github.com/gorilla/mux v1.7.4 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 - github.com/joho/sqltocsv v0.0.0-20190824231449-5650f27fd5b6 // indirect + github.com/klauspost/compress v1.10.5 // indirect github.com/klauspost/cpuid v1.2.1 github.com/kr/text v0.2.0 // indirect github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 @@ -50,7 +47,6 @@ require ( github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8 github.com/pingcap/parser v0.0.0-20201222091346-02c8ff27d0bc github.com/pingcap/sysutil v0.0.0-20201130064824-f0c8aa6a6966 - github.com/pingcap/tidb-lightning v4.0.9-0.20201106041742-a1ac97827a27+incompatible github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible github.com/pingcap/tipb v0.0.0-20201209065231-aa39b1b86217 github.com/prometheus/client_golang v1.5.1 @@ -66,7 +62,6 @@ require ( github.com/uber-go/atomic v1.4.0 github.com/uber/jaeger-client-go v2.22.1+incompatible github.com/uber/jaeger-lib v2.4.0+incompatible // indirect - github.com/xitongsys/parquet-go v1.5.4 // indirect github.com/zhangjinpeng1987/raft v0.0.0-20200819064223-df31bb68a018 // indirect go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b go.uber.org/atomic v1.7.0 @@ -84,7 +79,6 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v2 v2.3.0 // indirect honnef.co/go/tools v0.1.0 // indirect - modernc.org/mathutil v1.1.1 // indirect sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) diff --git a/go.sum b/go.sum index 1afc4f1be381d..f5045709b0b9a 100644 --- a/go.sum +++ b/go.sum @@ -8,27 +8,21 @@ cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTj cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.49.0/go.mod h1:hGvAdzcWNbyuxS3nWhD7H2cIJxjRRTRLQVB0bdputVY= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.51.0 h1:PvKAVQWCtlGUSlZkGW3QLelKaWq7KYv/MW1EboG8bfM= cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0 h1:MZQCQQaRwOrAcuKjiHWHrgKykt4fZyuwF2dtiG3fGW8= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0 h1:sAbMqjY1PEQKZBWfbu6Y6bsupJ9c4QdHnzg/VvYTLcE= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= +cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0 h1:9/vpR43S4aJaROxqQHQ3nH9lfyKKV0dC3vOmnw8ebQQ= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/spanner v1.1.0/go.mod h1:TzTaF9l2ZY2CIetNvVpUu6ZQy8YEOtzB6ICa5EwYjL0= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0 h1:RPUcBvDeYgQFMfQu1eBMq6piD1SXmLH+vK3qjewZPus= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= @@ -40,8 +34,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/ClickHouse/clickhouse-go v1.3.13/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/HdrHistogram/hdrhistogram-go v0.9.0 h1:dpujRju0R4M/QZzcnR1LH1qm+TVG3UzkWdp5tH1WMcg= @@ -92,16 +84,12 @@ github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQY github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/apache/calcite-avatica-go/v4 v4.0.0/go.mod h1:fg6MgnbY4Ta6JI0KuNaL9o/LrOMZdprQSMHWflcNs1c= github.com/apache/thrift v0.0.0-20171203172758-327ebb6c2b6d/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 h1:Jz3KVLYY5+JO7rDiX0sAuRGtuv2vG01r17Y9nLMWNUw= -github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apple/foundationdb/bindings/go v0.0.0-20200112054404-407dc0907f4f/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U= github.com/appleboy/easyssh-proxy v1.3.2/go.mod h1:Kk57I3w7OCafOjp5kgZFvxk2fO8Tca5CriBTOsbSbjY= github.com/appleboy/gin-jwt/v2 v2.6.3/go.mod h1:MfPYA4ogzvOcVkRwAxT7quHOtQmVKDpTwxyUrC2DNw0= github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/EventBus v0.0.0-20180315140547-d46933a94f05/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= -github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54= github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= @@ -129,13 +117,9 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= -github.com/carlmjohnson/flagext v0.20.2 h1:qvpMM+TytSrlh3+EIVn/pzOwwq9y13hXZab6Y4Gvqpo= -github.com/carlmjohnson/flagext v0.20.2/go.mod h1:Eenv0epIUAr4NuedNmkzI8WmBmjIxZC239XcKxYS2ac= github.com/cavaliercoder/grab v2.0.1-0.20200331080741-9f014744ee41+incompatible/go.mod h1:tTBkfNqSBfuMmMBFaO2phgyhdYhiZQ/+iXCZDzcDsMI= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA= -github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= @@ -155,16 +139,9 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/pebble v0.0.0-20200617141519-3b241b76ed3b h1:YHjo2xnqFCeFa0CdxEccHfUY1/DnXPAZdZt0+s/Mvdg= -github.com/cockroachdb/pebble v0.0.0-20200617141519-3b241b76ed3b/go.mod h1:crLnbSFbwAcQNs9FPfI1avHb5BqVgqZcr4r+IzpJ5FM= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codahale/hdrhistogram v0.9.0 h1:9GjrtRI+mLEFPtTfR/AZhcxp+Ii8NZYWq5104FbZQY0= github.com/codahale/hdrhistogram v0.9.0/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 h1:W1SHiII3e0jVwvaQFglwu3kS9NLxOeTpvik7MbKCyuQ= github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64/go.mod h1:F86k/6c7aDUdwSUevnLpHS/3Q9hzYCE99jGk2xsHnt0= github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 h1:NnLfQ77q0G4k2Of2c1ceQ0ec6MkLQyDp+IGdVM0D8XM= @@ -266,9 +243,6 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsouza/fake-gcs-server v1.17.0 h1:OeH75kBZcZa3ZE+zz/mFdJ2btt9FgqfjI7gIh9+5fvk= github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw= -github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= github.com/gibson042/canonicaljson-go v1.0.3/go.mod h1:DsLpJTThXyGNO+KZlI85C1/KDcImpP67k/RKVjcaEqo= @@ -288,7 +262,6 @@ github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxm github.com/go-echarts/go-echarts v1.0.0/go.mod h1:qbmyAb/Rl1f2w7wKba1D4LoNq4U164yO4/wedFbcWyo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.49.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -347,21 +320,16 @@ github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -386,7 +354,6 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190930153522-6ce02741cba3/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9 h1:K+lX49/3eURCE1IjlaZN//u6c+9nfDAMnyQ9E2dsJbY= github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -429,7 +396,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtg github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -453,7 +419,6 @@ github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mq github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= github.com/jackc/pgx v3.6.1+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= -github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jeremywohl/flatten v1.0.1/go.mod h1:4AmD/VxjWcI5SRB0n6szE2A6s2fsNHDLO0nAlMHgfLQ= @@ -461,7 +426,6 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -469,8 +433,6 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/joho/sqltocsv v0.0.0-20190824231449-5650f27fd5b6 h1:3Jr6Mtili6DsXSF0RwRlAqpOUWXcSVUxdOm5kFPb3xY= -github.com/joho/sqltocsv v0.0.0-20190824231449-5650f27fd5b6/go.mod h1:mAVCUAYtW9NG31eB30umMSLKcDt6mCUWSjoSn5qBh0k= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/joomcode/errorx v1.0.1/go.mod h1:kgco15ekB6cs+4Xjzo7SPeXzx38PbJzBwbnu9qfVNHQ= @@ -501,7 +463,6 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.5 h1:7q6vHIqubShURwQz8cQK6yIe/xC3IF0Vm7TGfqjewrc= github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -623,7 +584,6 @@ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= -github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= @@ -704,8 +664,6 @@ github.com/pingcap/sysutil v0.0.0-20201130064824-f0c8aa6a6966 h1:JI0wOAb8aQML0vA github.com/pingcap/sysutil v0.0.0-20201130064824-f0c8aa6a6966/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/tidb v1.1.0-beta.0.20200423094549-0ad7ce6cdae6/go.mod h1:KJXj2xHYfl1x4zcusC2JEANzVci+ietFOMh/CAmrYdw= github.com/pingcap/tidb-insight v0.3.1/go.mod h1:HZ6cGBYxViftTKLbl/V4moRol79bifZzI9ImGavJms0= -github.com/pingcap/tidb-lightning v4.0.9-0.20201106041742-a1ac97827a27+incompatible h1:y6cwuJJo8tLeHjgjODpwCSqNAv1g+9WUtcsFOHu/7ag= -github.com/pingcap/tidb-lightning v4.0.9-0.20201106041742-a1ac97827a27+incompatible/go.mod h1:UrN2l06neiEfkass87Tjvv5JFIMsOMb2DT7P7s+fwMc= github.com/pingcap/tidb-tools v3.0.13+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= @@ -766,9 +724,8 @@ github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqn github.com/relex/aini v1.2.0/go.mod h1:oFQyhvkzwi8GChiLukpBHkV2v142ls2L1MTeOSD2vic= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -822,7 +779,6 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= @@ -924,12 +880,6 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xinsnake/go-http-digest-auth-client v0.4.0/go.mod h1:QK1t1v7ylyGb363vGWu+6Irh7gyFj+N7+UZzM0L6g8I= -github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= -github.com/xitongsys/parquet-go v1.5.4 h1:zsdMNZcCv9t3YnlOfysMI78vBw+cN65jQznQlizVtqE= -github.com/xitongsys/parquet-go v1.5.4/go.mod h1:pheqtXeHQFzxJk45lRQ0UIGIivKnLXvialZSFWs81A8= -github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= -github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k= -github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xo/dburl v0.0.0-20191219122722-3cca8608d645/go.mod h1:A47W3pdWONaZmXuLZgfKLAVgUY0qvfTRM5vVDKS40S4= github.com/xo/tblfmt v0.0.0-20190609041254-28c54ec42ce8/go.mod h1:3U5kKQdIhwACye7ml3acccHmjGExY9WmUGU7rnDWgv0= @@ -959,9 +909,8 @@ go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b/go.mod h1:yVHk9ub3C go.mongodb.org/mongo-driver v1.0.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -991,7 +940,6 @@ go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -1020,12 +968,8 @@ golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxT golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299 h1:zQpM52jfKHG6II1ISZY1ZcpygvuSFZpLwfluuF89XOg= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= -golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1038,7 +982,6 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= @@ -1046,7 +989,6 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -1073,9 +1015,7 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -1125,7 +1065,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1135,19 +1074,15 @@ golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200819171115-d785dc25833f h1:KJuwZVtZBVzDmEDtB2zro9CXkD9O0dpCv4o2LHbQIAw= golang.org/x/sys v0.0.0-20200819171115-d785dc25833f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -1198,13 +1133,7 @@ golang.org/x/tools v0.0.0-20191210221141-98df12377212/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200325203130-f53864d0dba1/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= @@ -1229,10 +1158,8 @@ google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0 h1:yzlyyDW/J0w8yNFJIhiAJy4kq74S+1DOLdawELNxFMA= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0 h1:TgDr+1inK2XVUKZx3BYAqQg/GwucGdBkzZjWaTg/I+A= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1258,13 +1185,8 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191206224255-0243a4be9c8f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f h1:2wh8dWY8959cBGQvk1RD+/eQBgRYYDaZ+hT0/zsARoA= google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63 h1:YzfoEYWbODU5Fbt37+h7X16BWQbad7Q4S6gclTKFXM8= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1275,7 +1197,6 @@ google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/VividCortex/ewma.v1 v1.1.1/go.mod h1:TekXuFipeiHWiAlO1+wSS23vTcyFau5u3rxXUSXj710= @@ -1347,16 +1268,12 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/internal v1.0.0/go.mod h1:VUD/+JAkhCpvkUitlEOnhpVxCgsBI90oTzSCRcqQVSM= modernc.org/lldb v1.0.0/go.mod h1:jcRvJGWfCGodDZz8BPwiKMJxGJngQ/5DrRapkQnLob8= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= -modernc.org/mathutil v1.1.1 h1:FeylZSVX8S+58VsyJlkEj2bcpdytmp9MmDKZkKx8OIE= -modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/ql v1.0.0/go.mod h1:xGVyrLIatPcO2C1JvI/Co8c0sr6y91HKFNy4pt9JXEY= modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/server/server_test.go b/server/server_test.go index 8b35b110dfbbd..4d402c108cbf9 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -25,6 +25,7 @@ import ( "net/url" "os" "path/filepath" + "reflect" "regexp" "strconv" "strings" @@ -37,6 +38,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/log" tmysql "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/util/logutil" @@ -49,6 +51,12 @@ var ( ) func TestT(t *testing.T) { + defaultConfig := config.NewConfig() + globalConfig := config.GetGlobalConfig() + // Test for issue 22162. the global config shouldn't be changed by other pkg init function. + if !reflect.DeepEqual(defaultConfig, globalConfig) { + t.Fatalf("%#v != %#v\n", defaultConfig, globalConfig) + } CustomVerboseFlag = true logLevel := os.Getenv("log_level") logutil.InitZapLogger(logutil.NewLogConfig(logLevel, logutil.DefaultLogFormat, "", logutil.EmptyFileLogConfig, false)) diff --git a/server/tidb_test.go b/server/tidb_test.go index a3e361e9a7a7e..1a6dbe27b488f 100644 --- a/server/tidb_test.go +++ b/server/tidb_test.go @@ -140,6 +140,13 @@ func (ts *tidbTestSuite) TestPreparedTimestamp(c *C) { ts.runTestPreparedTimestamp(c) } +func (ts *tidbTestSerialSuite) TestConfigDefaultValue(c *C) { + ts.runTestsOnNewDB(c, nil, "config", func(dbt *DBTest) { + rows := dbt.mustQuery("select @@tidb_slow_log_threshold;") + ts.checkRows(c, rows, "300") + }) +} + // this test will change `kv.TxnTotalSizeLimit` which may affect other test suites, // so we must make it running in serial. func (ts *tidbTestSerialSuite) TestLoadData(c *C) { From 51794e9d304896a315d80ccb4603f59dc75f26d9 Mon Sep 17 00:00:00 2001 From: rebelice Date: Wed, 6 Jan 2021 16:28:29 +0800 Subject: [PATCH 11/16] *: rewrite origin SQL with default DB for SQL bindings (#21275) --- bindinfo/bind_test.go | 312 +++++++++++++++++++----------- bindinfo/cache.go | 2 +- bindinfo/handle.go | 23 ++- bindinfo/session_handle.go | 2 +- planner/core/planbuilder.go | 37 +++- planner/core/preprocess.go | 11 +- planner/optimize.go | 54 ++++-- session/bootstrap.go | 77 +++++++- session/bootstrap_test.go | 30 +++ types/parser_driver/value_expr.go | 5 +- util/parser/ast.go | 18 ++ 11 files changed, 419 insertions(+), 152 deletions(-) diff --git a/bindinfo/bind_test.go b/bindinfo/bind_test.go index e37ac4c111edc..88bc4ab52fc31 100644 --- a/bindinfo/bind_test.go +++ b/bindinfo/bind_test.go @@ -39,6 +39,7 @@ import ( "github.com/pingcap/tidb/store/mockstore/cluster" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/logutil" + utilparser "github.com/pingcap/tidb/util/parser" "github.com/pingcap/tidb/util/stmtsummary" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" @@ -143,6 +144,13 @@ func (s *testSuite) cleanBindingEnv(tk *testkit.TestKit) { s.domain.BindHandle().Clear() } +func normalizeWithDefaultDB(c *C, sql, db string) (string, string) { + testParser := parser.New() + stmt, err := testParser.ParseOneStmt(sql, "", "") + c.Assert(err, IsNil) + return parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(stmt, "test")) +} + func (s *testSuite) TestBindParse(c *C) { tk := testkit.NewTestKit(c, s.store) s.cleanBindingEnv(tk) @@ -150,8 +158,8 @@ func (s *testSuite) TestBindParse(c *C) { tk.MustExec("create table t(i int)") tk.MustExec("create index index_t on t(i)") - originSQL := "select * from t" - bindSQL := "select * from t use index(index_t)" + originSQL := "select * from test . t" + bindSQL := "select * from test . t use index(index_t)" defaultDb := "test" status := "using" charset := "utf8mb4" @@ -165,12 +173,12 @@ func (s *testSuite) TestBindParse(c *C) { c.Check(err, IsNil) c.Check(bindHandle.Size(), Equals, 1) - sql, hash := parser.NormalizeDigest("select * from t") + sql, hash := parser.NormalizeDigest("select * from test . t") bindData := bindHandle.GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select * from t") + c.Check(bindData.OriginalSQL, Equals, "select * from test . t") bind := bindData.Bindings[0] - c.Check(bind.BindSQL, Equals, "select * from t use index(index_t)") + c.Check(bind.BindSQL, Equals, "select * from test . t use index(index_t)") c.Check(bindData.Db, Equals, "test") c.Check(bind.Status, Equals, "using") c.Check(bind.Charset, Equals, "utf8mb4") @@ -237,100 +245,109 @@ var testSQLs = []struct { createSQL: "binding for select * from t where i>100 using select * from t use index(index_t) where i>100", overlaySQL: "binding for select * from t where i>99 using select * from t use index(index_t) where i>99", querySQL: "select * from t where i > 30.0", - originSQL: "select * from t where i > ?", - bindSQL: "select * from t use index(index_t) where i>99", + originSQL: "select * from test . t where i > ?", + bindSQL: "SELECT * FROM test.t USE INDEX (index_t) WHERE i > 99", dropSQL: "binding for select * from t where i>100", - memoryUsage: float64(97), + memoryUsage: float64(112), }, { createSQL: "binding for select * from t union all select * from t using select * from t use index(index_t) union all select * from t use index()", overlaySQL: "", querySQL: "select * from t union all select * from t", - originSQL: "select * from t union all select * from t", - bindSQL: "select * from t use index(index_t) union all select * from t use index()", + originSQL: "select * from test . t union all select * from test . t", + bindSQL: "SELECT * FROM test.t USE INDEX (index_t) UNION ALL SELECT * FROM test.t USE INDEX ()", dropSQL: "binding for select * from t union all select * from t", - memoryUsage: float64(138), + memoryUsage: float64(164), }, { createSQL: "binding for (select * from t) union all (select * from t) using (select * from t use index(index_t)) union all (select * from t use index())", overlaySQL: "", querySQL: "(select * from t) union all (select * from t)", - originSQL: "( select * from t ) union all ( select * from t )", - bindSQL: "(select * from t use index(index_t)) union all (select * from t use index())", + originSQL: "( select * from test . t ) union all ( select * from test . t )", + bindSQL: "(SELECT * FROM test.t USE INDEX (index_t)) UNION ALL (SELECT * FROM test.t USE INDEX ())", dropSQL: "binding for (select * from t) union all (select * from t)", - memoryUsage: float64(150), + memoryUsage: float64(176), }, { createSQL: "binding for select * from t intersect select * from t using select * from t use index(index_t) intersect select * from t use index()", overlaySQL: "", querySQL: "select * from t intersect select * from t", - originSQL: "select * from t intersect select * from t", - bindSQL: "select * from t use index(index_t) intersect select * from t use index()", + originSQL: "select * from test . t intersect select * from test . t", + bindSQL: "SELECT * FROM test.t USE INDEX (index_t) INTERSECT SELECT * FROM test.t USE INDEX ()", dropSQL: "binding for select * from t intersect select * from t", - memoryUsage: float64(138), + memoryUsage: float64(164), }, { createSQL: "binding for select * from t except select * from t using select * from t use index(index_t) except select * from t use index()", overlaySQL: "", querySQL: "select * from t except select * from t", - originSQL: "select * from t except select * from t", - bindSQL: "select * from t use index(index_t) except select * from t use index()", + originSQL: "select * from test . t except select * from test . t", + bindSQL: "SELECT * FROM test.t USE INDEX (index_t) EXCEPT SELECT * FROM test.t USE INDEX ()", dropSQL: "binding for select * from t except select * from t", - memoryUsage: float64(132), + memoryUsage: float64(158), + }, + { + createSQL: "binding for select * from t using select /*+ use_index(t,index_t)*/ * from t", + overlaySQL: "", + querySQL: "select * from t ", + originSQL: "select * from test . t", + bindSQL: "SELECT /*+ use_index(t index_t)*/ * FROM test.t", + dropSQL: "binding for select * from t", + memoryUsage: float64(94), }, { createSQL: "binding for delete from t where i = 1 using delete /*+ use_index(t,index_t) */ from t where i = 1", overlaySQL: "", querySQL: "delete from t where i = 2", - originSQL: "delete from t where i = ?", - bindSQL: "delete /*+ use_index(t,index_t) */ from t where i = 1", + originSQL: "delete from test . t where i = ?", + bindSQL: "DELETE /*+ use_index(t index_t)*/ FROM test.t WHERE i = 1", dropSQL: "binding for delete from t where i = 1", - memoryUsage: float64(103), + memoryUsage: float64(114), }, { createSQL: "binding for delete t, t1 from t inner join t1 on t.s = t1.s where t.i = 1 using delete /*+ use_index(t,index_t), hash_join(t,t1) */ t, t1 from t inner join t1 on t.s = t1.s where t.i = 1", overlaySQL: "", querySQL: "delete t, t1 from t inner join t1 on t.s = t1.s where t.i = 2", - originSQL: "delete t , t1 from t inner join t1 on t . s = t1 . s where t . i = ?", - bindSQL: "delete /*+ use_index(t,index_t), hash_join(t,t1) */ t, t1 from t inner join t1 on t.s = t1.s where t.i = 1", + originSQL: "delete test . t , test . t1 from test . t join test . t1 on t . s = t1 . s where t . i = ?", + bindSQL: "DELETE /*+ use_index(t index_t) hash_join(t, t1)*/ test.t,test.t1 FROM test.t JOIN test.t1 ON t.s = t1.s WHERE t.i = 1", dropSQL: "binding for delete t, t1 from t inner join t1 on t.s = t1.s where t.i = 1", - memoryUsage: float64(199), + memoryUsage: float64(233), }, { createSQL: "binding for update t set s = 'a' where i = 1 using update /*+ use_index(t,index_t) */ t set s = 'a' where i = 1", overlaySQL: "", querySQL: "update t set s='b' where i=2", - originSQL: "update t set s = ? where i = ?", - bindSQL: "update /*+ use_index(t,index_t) */ t set s = 'a' where i = 1", + originSQL: "update test . t set s = ? where i = ?", + bindSQL: "UPDATE /*+ use_index(t index_t)*/ test.t SET s='a' WHERE i = 1", dropSQL: "binding for update t set s = 'a' where i = 1", - memoryUsage: float64(115), + memoryUsage: float64(124), }, { createSQL: "binding for update t, t1 set t.s = 'a' where t.i = t1.i using update /*+ inl_join(t1) */ t, t1 set t.s = 'a' where t.i = t1.i", overlaySQL: "", querySQL: "update t , t1 set t.s='b' where t.i=t1.i", - originSQL: "update t , t1 set t . s = ? where t . i = t1 . i", - bindSQL: "update /*+ inl_join(t1) */ t, t1 set t.s = 'a' where t.i = t1.i", + originSQL: "update ( test . t ) join test . t1 set t . s = ? where t . i = t1 . i", + bindSQL: "UPDATE /*+ inl_join(t1)*/ (test.t) JOIN test.t1 SET t.s='a' WHERE t.i = t1.i", dropSQL: "binding for update t, t1 set t.s = 'a' where t.i = t1.i", - memoryUsage: float64(136), + memoryUsage: float64(170), }, { createSQL: "binding for insert into t1 select * from t where t.i = 1 using insert into t1 select /*+ use_index(t,index_t) */ * from t where t.i = 1", overlaySQL: "", querySQL: "insert into t1 select * from t where t.i = 2", - originSQL: "insert into t1 select * from t where t . i = ?", - bindSQL: "insert into t1 select /*+ use_index(t,index_t) */ * from t where t.i = 1", + originSQL: "insert into test . t1 select * from test . t where t . i = ?", + bindSQL: "INSERT INTO test.t1 SELECT /*+ use_index(t index_t)*/ * FROM test.t WHERE t.i = 1", dropSQL: "binding for insert into t1 select * from t where t.i = 1", - memoryUsage: float64(143), + memoryUsage: float64(166), }, { createSQL: "binding for replace into t1 select * from t where t.i = 1 using replace into t1 select /*+ use_index(t,index_t) */ * from t where t.i = 1", overlaySQL: "", querySQL: "replace into t1 select * from t where t.i = 2", - originSQL: "replace into t1 select * from t where t . i = ?", - bindSQL: "replace into t1 select /*+ use_index(t,index_t) */ * from t where t.i = 1", + originSQL: "replace into test . t1 select * from test . t where t . i = ?", + bindSQL: "REPLACE INTO test.t1 SELECT /*+ use_index(t index_t)*/ * FROM test.t WHERE t.i = 1", dropSQL: "binding for replace into t1 select * from t where t.i = 1", - memoryUsage: float64(145), + memoryUsage: float64(168), }, } @@ -363,7 +380,7 @@ func (s *testSuite) TestGlobalBinding(c *C) { metrics.BindMemoryUsage.WithLabelValues(metrics.ScopeGlobal, bindinfo.Using).Write(pb) c.Assert(pb.GetGauge().GetValue(), Equals, testSQL.memoryUsage) - sql, hash := parser.NormalizeDigest(testSQL.querySQL) + sql, hash := normalizeWithDefaultDB(c, testSQL.querySQL, "test") bindData := s.domain.BindHandle().GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) @@ -620,13 +637,13 @@ func (s *testSuite) TestBindingSymbolList(c *C) { c.Assert(tk.MustUseIndex("select a, b from t where a = 3 limit 1, 100", "ib(b)"), IsTrue) // Normalize - sql, hash := parser.NormalizeDigest("select a, b from t where a = 1 limit 0, 1") + sql, hash := parser.NormalizeDigest("select a, b from test . t where a = 1 limit 0, 1") bindData := s.domain.BindHandle().GetBindRecord(hash, sql, "test") c.Assert(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select a , b from t where a = ? limit ...") + c.Check(bindData.OriginalSQL, Equals, "select a , b from test . t where a = ? limit ...") bind := bindData.Bindings[0] - c.Check(bind.BindSQL, Equals, "select a, b from t use index (ib) where a = 1 limit 0, 1") + c.Check(bind.BindSQL, Equals, "SELECT a,b FROM test.t USE INDEX (ib) WHERE a = 1 LIMIT 0,1") c.Check(bindData.Db, Equals, "test") c.Check(bind.Status, Equals, "using") c.Check(bind.Charset, NotNil) @@ -706,15 +723,15 @@ func (s *testSuite) TestBestPlanInBaselines(c *C) { c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:ib") c.Assert(tk.MustUseIndex("select a, b from t where b = 3 limit 1, 100", "ib(b)"), IsTrue) - tk.MustExec(`create global binding for select a, b from t where a = 1 limit 0, 1 using select /*+ use_index(@sel_1 test.t, ia) */ a, b from t where a = 1 limit 0, 1`) - tk.MustExec(`create global binding for select a, b from t where b = 1 limit 0, 1 using select /*+ use_index(@sel_1 test.t, ib) */ a, b from t where b = 1 limit 0, 1`) + tk.MustExec(`create global binding for select a, b from t where a = 1 limit 0, 1 using select /*+ use_index(@sel_1 test.t ia) */ a, b from t where a = 1 limit 0, 1`) + tk.MustExec(`create global binding for select a, b from t where b = 1 limit 0, 1 using select /*+ use_index(@sel_1 test.t ib) */ a, b from t where b = 1 limit 0, 1`) - sql, hash := parser.NormalizeDigest("select a, b from t where a = 1 limit 0, 1") + sql, hash := normalizeWithDefaultDB(c, "select a, b from t where a = 1 limit 0, 1", "test") bindData := s.domain.BindHandle().GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select a , b from t where a = ? limit ...") + c.Check(bindData.OriginalSQL, Equals, "select a , b from test . t where a = ? limit ...") bind := bindData.Bindings[0] - c.Check(bind.BindSQL, Equals, "select /*+ use_index(@sel_1 test.t, ia) */ a, b from t where a = 1 limit 0, 1") + c.Check(bind.BindSQL, Equals, "SELECT /*+ use_index(@sel_1 test.t ia)*/ a,b FROM test.t WHERE a = 1 LIMIT 0,1") c.Check(bindData.Db, Equals, "test") c.Check(bind.Status, Equals, "using") @@ -741,12 +758,12 @@ func (s *testSuite) TestErrorBind(c *C) { _, err := tk.Exec("create global binding for select * from t where i>100 using select * from t use index(index_t) where i>100") c.Assert(err, IsNil, Commentf("err %v", err)) - sql, hash := parser.NormalizeDigest("select * from t where i > ?") + sql, hash := parser.NormalizeDigest("select * from test . t where i > ?") bindData := s.domain.BindHandle().GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select * from t where i > ?") + c.Check(bindData.OriginalSQL, Equals, "select * from test . t where i > ?") bind := bindData.Bindings[0] - c.Check(bind.BindSQL, Equals, "select * from t use index(index_t) where i>100") + c.Check(bind.BindSQL, Equals, "SELECT * FROM test.t USE INDEX (index_t) WHERE i > 100") c.Check(bindData.Db, Equals, "test") c.Check(bind.Status, Equals, "using") c.Check(bind.Charset, NotNil) @@ -883,14 +900,14 @@ func (s *testSuite) TestDMLCapturePlanBaseline(c *C) { tk.MustExec("admin capture bindings") rows = tk.MustQuery("show global bindings").Sort().Rows() c.Assert(len(rows), Equals, 4) - c.Assert(rows[0][0], Equals, "delete from t where b = ? and c > ?") - c.Assert(rows[0][1], Equals, "DELETE /*+ use_index(@`del_1` `test`.`t` `idx_b`)*/ FROM `t` WHERE `b`=1 AND `c`>1") - c.Assert(rows[1][0], Equals, "insert into t1 select * from t where t . b = ? and t . c > ?") - c.Assert(rows[1][1], Equals, "INSERT INTO `t1` SELECT /*+ use_index(@`sel_1` `test`.`t` `idx_b`)*/ * FROM `t` WHERE `t`.`b`=1 AND `t`.`c`>1") - c.Assert(rows[2][0], Equals, "replace into t1 select * from t where t . b = ? and t . c > ?") - c.Assert(rows[2][1], Equals, "REPLACE INTO `t1` SELECT /*+ use_index(@`sel_1` `test`.`t` `idx_b`)*/ * FROM `t` WHERE `t`.`b`=1 AND `t`.`c`>1") - c.Assert(rows[3][0], Equals, "update t set a = ? where b = ? and c > ?") - c.Assert(rows[3][1], Equals, "UPDATE /*+ use_index(@`upd_1` `test`.`t` `idx_b`)*/ `t` SET `a`=1 WHERE `b`=1 AND `c`>1") + c.Assert(rows[0][0], Equals, "delete from test . t where b = ? and c > ?") + c.Assert(rows[0][1], Equals, "DELETE /*+ use_index(@`del_1` `test`.`t` `idx_b`)*/ FROM `test`.`t` WHERE `b`=1 AND `c`>1") + c.Assert(rows[1][0], Equals, "insert into test . t1 select * from test . t where t . b = ? and t . c > ?") + c.Assert(rows[1][1], Equals, "INSERT INTO `test`.`t1` SELECT /*+ use_index(@`sel_1` `test`.`t` `idx_b`)*/ * FROM `test`.`t` WHERE `t`.`b`=1 AND `t`.`c`>1") + c.Assert(rows[2][0], Equals, "replace into test . t1 select * from test . t where t . b = ? and t . c > ?") + c.Assert(rows[2][1], Equals, "REPLACE INTO `test`.`t1` SELECT /*+ use_index(@`sel_1` `test`.`t` `idx_b`)*/ * FROM `test`.`t` WHERE `t`.`b`=1 AND `t`.`c`>1") + c.Assert(rows[3][0], Equals, "update test . t set a = ? where b = ? and c > ?") + c.Assert(rows[3][1], Equals, "UPDATE /*+ use_index(@`upd_1` `test`.`t` `idx_b`)*/ `test`.`t` SET `a`=1 WHERE `b`=1 AND `c`>1") } func (s *testSuite) TestCapturePlanBaseline(c *C) { @@ -918,8 +935,8 @@ func (s *testSuite) TestCapturePlanBaseline(c *C) { tk.MustExec("admin capture bindings") rows = tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][0], Equals, "select * from t where a > ?") - c.Assert(rows[0][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` )*/ * FROM `t` WHERE `a`>10") + c.Assert(rows[0][0], Equals, "select * from test . t where a > ?") + c.Assert(rows[0][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` )*/ * FROM `test`.`t` WHERE `a`>10") } func (s *testSuite) TestCaptureDBCaseSensitivity(c *C) { @@ -939,7 +956,7 @@ func (s *testSuite) TestCaptureDBCaseSensitivity(c *C) { // so there would be no new binding captured. rows := tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][1], Equals, "select /*+ use_index(t) */ * from t") + c.Assert(rows[0][1], Equals, "SELECT /*+ use_index(t )*/ * FROM SPM.t") c.Assert(rows[0][8], Equals, "manual") } @@ -957,7 +974,7 @@ func (s *testSuite) TestBaselineDBLowerCase(c *C) { tk.MustExec("admin capture bindings") rows := tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][0], Equals, "update t set a = a + ?") + c.Assert(rows[0][0], Equals, "update spm . t set a = a + ?") // default_db should have lower case. c.Assert(rows[0][2], Equals, "spm") tk.MustExec("drop global binding for update t set a = a + 1") @@ -968,7 +985,7 @@ func (s *testSuite) TestBaselineDBLowerCase(c *C) { tk.MustExec("create global binding for select * from t using select * from t") rows = tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][0], Equals, "select * from t") + c.Assert(rows[0][0], Equals, "select * from spm . t") // default_db should have lower case. c.Assert(rows[0][2], Equals, "spm") tk.MustExec("drop global binding for select * from t") @@ -979,7 +996,7 @@ func (s *testSuite) TestBaselineDBLowerCase(c *C) { tk.MustExec("create session binding for select * from t using select * from t") rows = tk.MustQuery("show session bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][0], Equals, "select * from t") + c.Assert(rows[0][0], Equals, "select * from spm . t") // default_db should have lower case. c.Assert(rows[0][2], Equals, "spm") tk.MustExec("drop session binding for select * from t") @@ -989,15 +1006,15 @@ func (s *testSuite) TestBaselineDBLowerCase(c *C) { s.cleanBindingEnv(tk) // Simulate existing bindings with upper case default_db. - tk.MustExec("insert into mysql.bind_info values('select * from t', 'select * from t', 'SPM', 'using', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + + tk.MustExec("insert into mysql.bind_info values('select * from spm . t', 'select * from spm . t', 'SPM', 'using', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "')") - tk.MustQuery("select original_sql, default_db from mysql.bind_info where original_sql = 'select * from t'").Check(testkit.Rows( - "select * from t SPM", + tk.MustQuery("select original_sql, default_db from mysql.bind_info where original_sql = 'select * from spm . t'").Check(testkit.Rows( + "select * from spm . t SPM", )) tk.MustExec("admin reload bindings") rows = tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][0], Equals, "select * from t") + c.Assert(rows[0][0], Equals, "select * from spm . t") // default_db should have lower case. c.Assert(rows[0][2], Equals, "spm") tk.MustExec("drop global binding for select * from t") @@ -1007,25 +1024,25 @@ func (s *testSuite) TestBaselineDBLowerCase(c *C) { s.cleanBindingEnv(tk) // Simulate existing bindings with upper case default_db. - tk.MustExec("insert into mysql.bind_info values('select * from t', 'select * from t', 'SPM', 'using', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + + tk.MustExec("insert into mysql.bind_info values('select * from spm . t', 'select * from spm . t', 'SPM', 'using', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "')") - tk.MustQuery("select original_sql, default_db from mysql.bind_info where original_sql = 'select * from t'").Check(testkit.Rows( - "select * from t SPM", + tk.MustQuery("select original_sql, default_db from mysql.bind_info where original_sql = 'select * from spm . t'").Check(testkit.Rows( + "select * from spm . t SPM", )) tk.MustExec("admin reload bindings") rows = tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][0], Equals, "select * from t") + c.Assert(rows[0][0], Equals, "select * from spm . t") // default_db should have lower case. c.Assert(rows[0][2], Equals, "spm") tk.MustExec("create global binding for select * from t using select * from t") rows = tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][0], Equals, "select * from t") + c.Assert(rows[0][0], Equals, "select * from spm . t") // default_db should have lower case. c.Assert(rows[0][2], Equals, "spm") - tk.MustQuery("select original_sql, default_db, status from mysql.bind_info where original_sql = 'select * from t'").Check(testkit.Rows( - "select * from t spm using", + tk.MustQuery("select original_sql, default_db, status from mysql.bind_info where original_sql = 'select * from spm . t'").Check(testkit.Rows( + "select * from spm . t spm using", )) } @@ -1074,16 +1091,16 @@ func (s *testSuite) TestCapturePreparedStmt(c *C) { tk.MustExec("admin capture bindings") rows := tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][0], Equals, "select * from t where b = ? and c > ?") - c.Assert(rows[0][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` `idx_c`)*/ * FROM `t` WHERE `b`=? AND `c`>?") + c.Assert(rows[0][0], Equals, "select * from test . t where b = ? and c > ?") + c.Assert(rows[0][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` `idx_c`)*/ * FROM `test`.`t` WHERE `b`=? AND `c`>?") c.Assert(tk.MustUseIndex("select /*+ use_index(t,idx_b) */ * from t where b = 1 and c > 1", "idx_c(c)"), IsTrue) tk.MustExec("admin flush bindings") tk.MustExec("admin evolve bindings") rows = tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][0], Equals, "select * from t where b = ? and c > ?") - c.Assert(rows[0][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` `idx_c`)*/ * FROM `t` WHERE `b`=? AND `c`>?") + c.Assert(rows[0][0], Equals, "select * from test . t where b = ? and c > ?") + c.Assert(rows[0][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` `idx_c`)*/ * FROM `test`.`t` WHERE `b`=? AND `c`>?") } func (s *testSuite) TestDropSingleBindings(c *C) { @@ -1100,11 +1117,11 @@ func (s *testSuite) TestDropSingleBindings(c *C) { // The size of bindings is equal to one. Because for one normalized sql, // the `create binding` clears all the origin bindings. c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][1], Equals, "select * from t use index(idx_b)") + c.Assert(rows[0][1], Equals, "SELECT * FROM test.t USE INDEX (idx_b)") tk.MustExec("drop binding for select * from t using select * from t use index(idx_a)") rows = tk.MustQuery("show bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][1], Equals, "select * from t use index(idx_b)") + c.Assert(rows[0][1], Equals, "SELECT * FROM test.t USE INDEX (idx_b)") tk.MustExec("drop table t") tk.MustExec("drop binding for select * from t using select * from t use index(idx_b)") rows = tk.MustQuery("show bindings").Rows() @@ -1118,11 +1135,11 @@ func (s *testSuite) TestDropSingleBindings(c *C) { // The size of bindings is equal to one. Because for one normalized sql, // the `create binding` clears all the origin bindings. c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][1], Equals, "select * from t use index(idx_b)") + c.Assert(rows[0][1], Equals, "SELECT * FROM test.t USE INDEX (idx_b)") tk.MustExec("drop global binding for select * from t using select * from t use index(idx_a)") rows = tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][1], Equals, "select * from t use index(idx_b)") + c.Assert(rows[0][1], Equals, "SELECT * FROM test.t USE INDEX (idx_b)") tk.MustExec("drop table t") tk.MustExec("drop global binding for select * from t using select * from t use index(idx_b)") rows = tk.MustQuery("show global bindings").Rows() @@ -1418,11 +1435,11 @@ func (s *testSuite) TestEvolveInvalidBindings(c *C) { tk.MustExec("create table t(a int, b int, index idx_a(a))") tk.MustExec("create global binding for select * from t where a > 10 using select /*+ USE_INDEX(t) */ * from t where a > 10") // Manufacture a rejected binding by hacking mysql.bind_info. - tk.MustExec("insert into mysql.bind_info values('select * from t where a > ?', 'select /*+ USE_INDEX(t,idx_a) */ * from t where a > 10', 'test', 'rejected', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + + tk.MustExec("insert into mysql.bind_info values('select * from test . t where a > ?', 'SELECT /*+ USE_INDEX(t,idx_a) */ * FROM test.t WHERE a > 10', 'test', 'rejected', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "')") tk.MustQuery("select bind_sql, status from mysql.bind_info where source != 'builtin'").Sort().Check(testkit.Rows( - "select /*+ USE_INDEX(t) */ * from t where a > 10 using", - "select /*+ USE_INDEX(t,idx_a) */ * from t where a > 10 rejected", + "SELECT /*+ USE_INDEX(t )*/ * FROM test.t WHERE a > 10 using", + "SELECT /*+ USE_INDEX(t,idx_a) */ * FROM test.t WHERE a > 10 rejected", )) // Reload cache from mysql.bind_info. s.domain.BindHandle().Clear() @@ -1434,10 +1451,10 @@ func (s *testSuite) TestEvolveInvalidBindings(c *C) { rows := tk.MustQuery("show global bindings").Sort().Rows() c.Assert(len(rows), Equals, 2) // Make sure this "using" binding is not overrided. - c.Assert(rows[0][1], Equals, "select /*+ USE_INDEX(t) */ * from t where a > 10") + c.Assert(rows[0][1], Equals, "SELECT /*+ USE_INDEX(t )*/ * FROM test.t WHERE a > 10") status := rows[0][3].(string) c.Assert(status == "using", IsTrue) - c.Assert(rows[1][1], Equals, "select /*+ USE_INDEX(t,idx_a) */ * from t where a > 10") + c.Assert(rows[1][1], Equals, "SELECT /*+ USE_INDEX(t,idx_a) */ * FROM test.t WHERE a > 10") status = rows[1][3].(string) c.Assert(status == "using" || status == "rejected", IsTrue) } @@ -1482,10 +1499,10 @@ func (s *testSuite) TestHintsSetEvolveTask(c *C) { bindHandle := s.domain.BindHandle() bindHandle.SaveEvolveTasksToStore() // Verify the added Binding for evolution contains valid ID and Hint, otherwise, panic may happen. - sql, hash := parser.NormalizeDigest("select * from t where a > ?") + sql, hash := normalizeWithDefaultDB(c, "select * from t where a > ?", "test") bindData := bindHandle.GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select * from t where a > ?") + c.Check(bindData.OriginalSQL, Equals, "select * from test . t where a > ?") c.Assert(len(bindData.Bindings), Equals, 2) bind := bindData.Bindings[1] c.Assert(bind.Status, Equals, bindinfo.PendingVerify) @@ -1502,10 +1519,10 @@ func (s *testSuite) TestHintsSetID(c *C) { tk.MustExec("create global binding for select * from t where a > 10 using select /*+ use_index(test.t, idx_a) */ * from t where a > 10") bindHandle := s.domain.BindHandle() // Verify the added Binding contains ID with restored query block. - sql, hash := parser.NormalizeDigest("select * from t where a > ?") + sql, hash := normalizeWithDefaultDB(c, "select * from t where a > ?", "test") bindData := bindHandle.GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select * from t where a > ?") + c.Check(bindData.OriginalSQL, Equals, "select * from test . t where a > ?") c.Assert(len(bindData.Bindings), Equals, 1) bind := bindData.Bindings[0] c.Assert(bind.ID, Equals, "use_index(@`sel_1` `test`.`t` `idx_a`)") @@ -1514,7 +1531,7 @@ func (s *testSuite) TestHintsSetID(c *C) { tk.MustExec("create global binding for select * from t where a > 10 using select /*+ use_index(t, idx_a) */ * from t where a > 10") bindData = bindHandle.GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select * from t where a > ?") + c.Check(bindData.OriginalSQL, Equals, "select * from test . t where a > ?") c.Assert(len(bindData.Bindings), Equals, 1) bind = bindData.Bindings[0] c.Assert(bind.ID, Equals, "use_index(@`sel_1` `test`.`t` `idx_a`)") @@ -1523,7 +1540,7 @@ func (s *testSuite) TestHintsSetID(c *C) { tk.MustExec("create global binding for select * from t where a > 10 using select /*+ use_index(@sel_1 t, idx_a) */ * from t where a > 10") bindData = bindHandle.GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select * from t where a > ?") + c.Check(bindData.OriginalSQL, Equals, "select * from test . t where a > ?") c.Assert(len(bindData.Bindings), Equals, 1) bind = bindData.Bindings[0] c.Assert(bind.ID, Equals, "use_index(@`sel_1` `test`.`t` `idx_a`)") @@ -1532,7 +1549,7 @@ func (s *testSuite) TestHintsSetID(c *C) { tk.MustExec("create global binding for select * from t where a > 10 using select /*+ use_index(@qb1 t, idx_a) qb_name(qb1) */ * from t where a > 10") bindData = bindHandle.GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select * from t where a > ?") + c.Check(bindData.OriginalSQL, Equals, "select * from test . t where a > ?") c.Assert(len(bindData.Bindings), Equals, 1) bind = bindData.Bindings[0] c.Assert(bind.ID, Equals, "use_index(@`sel_1` `test`.`t` `idx_a`)") @@ -1541,7 +1558,7 @@ func (s *testSuite) TestHintsSetID(c *C) { tk.MustExec("create global binding for select * from t where a > 10 using select /*+ use_index(T, IDX_A) */ * from t where a > 10") bindData = bindHandle.GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select * from t where a > ?") + c.Check(bindData.OriginalSQL, Equals, "select * from test . t where a > ?") c.Assert(len(bindData.Bindings), Equals, 1) bind = bindData.Bindings[0] c.Assert(bind.ID, Equals, "use_index(@`sel_1` `test`.`t` `idx_a`)") @@ -1552,7 +1569,7 @@ func (s *testSuite) TestHintsSetID(c *C) { tk.MustExec("create global binding for select * from t where a > 10 using select * from t where a > 10") bindData = bindHandle.GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select * from t where a > ?") + c.Check(bindData.OriginalSQL, Equals, "select * from test . t where a > ?") c.Assert(len(bindData.Bindings), Equals, 1) bind = bindData.Bindings[0] c.Assert(bind.ID, Equals, "") @@ -1590,8 +1607,8 @@ func (s *testSuite) TestCapturePlanBaselineIgnoreTiFlash(c *C) { // Don't have the TiFlash plan even we have TiFlash replica. rows = tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][0], Equals, "select * from t") - c.Assert(rows[0][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` )*/ * FROM `t`") + c.Assert(rows[0][0], Equals, "select * from test . t") + c.Assert(rows[0][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` )*/ * FROM `test`.`t`") } func (s *testSuite) TestNotEvolvePlanForReadStorageHint(c *C) { @@ -1632,7 +1649,7 @@ func (s *testSuite) TestNotEvolvePlanForReadStorageHint(c *C) { rows = tk.MustQuery("show global bindings").Rows() // None evolve task, because of the origin binding is a read_from_storage binding. c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][1], Equals, "select /*+ read_from_storage(tiflash[t]) */ * from t where a >= 1 and b >= 1") + c.Assert(rows[0][1], Equals, "SELECT /*+ read_from_storage(tiflash[t])*/ * FROM test.t WHERE a >= 1 AND b >= 1") c.Assert(rows[0][3], Equals, "using") } @@ -1738,10 +1755,10 @@ func (s *testSuite) TestBindingSource(c *C) { // Test Source for SQL created sql tk.MustExec("create global binding for select * from t where a > 10 using select * from t ignore index(idx_a) where a > 10") bindHandle := s.domain.BindHandle() - sql, hash := parser.NormalizeDigest("select * from t where a > ?") + sql, hash := normalizeWithDefaultDB(c, "select * from t where a > ?", "test") bindData := bindHandle.GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select * from t where a > ?") + c.Check(bindData.OriginalSQL, Equals, "select * from test . t where a > ?") c.Assert(len(bindData.Bindings), Equals, 1) bind := bindData.Bindings[0] c.Assert(bind.Source, Equals, bindinfo.Manual) @@ -1750,10 +1767,10 @@ func (s *testSuite) TestBindingSource(c *C) { tk.MustExec("set @@tidb_evolve_plan_baselines=1") tk.MustQuery("select * from t where a > 10") bindHandle.SaveEvolveTasksToStore() - sql, hash = parser.NormalizeDigest("select * from t where a > ?") + sql, hash = normalizeWithDefaultDB(c, "select * from t where a > ?", "test") bindData = bindHandle.GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select * from t where a > ?") + c.Check(bindData.OriginalSQL, Equals, "select * from test . t where a > ?") c.Assert(len(bindData.Bindings), Equals, 2) bind = bindData.Bindings[1] c.Assert(bind.Source, Equals, bindinfo.Evolve) @@ -1771,10 +1788,10 @@ func (s *testSuite) TestBindingSource(c *C) { tk.MustExec("select * from t ignore index(idx_a) where a < 10") tk.MustExec("admin capture bindings") bindHandle.CaptureBaselines() - sql, hash = parser.NormalizeDigest("select * from t where a < ?") + sql, hash = normalizeWithDefaultDB(c, "select * from t where a < ?", "test") bindData = bindHandle.GetBindRecord(hash, sql, "test") c.Check(bindData, NotNil) - c.Check(bindData.OriginalSQL, Equals, "select * from t where a < ?") + c.Check(bindData.OriginalSQL, Equals, "select * from test . t where a < ?") c.Assert(len(bindData.Bindings), Equals, 1) bind = bindData.Bindings[0] c.Assert(bind.Source, Equals, bindinfo.Capture) @@ -1838,20 +1855,20 @@ func (s *testSuite) TestReCreateBind(c *C) { tk.MustExec("create global binding for select * from t using select * from t") tk.MustQuery("select original_sql, status from mysql.bind_info").Check(testkit.Rows( - "select * from t using", + "select * from test . t using", )) rows := tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][0], Equals, "select * from t") + c.Assert(rows[0][0], Equals, "select * from test . t") c.Assert(rows[0][3], Equals, "using") tk.MustExec("create global binding for select * from t using select * from t") tk.MustQuery("select original_sql, status from mysql.bind_info").Check(testkit.Rows( - "select * from t using", + "select * from test . t using", )) rows = tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][0], Equals, "select * from t") + c.Assert(rows[0][0], Equals, "select * from test . t") c.Assert(rows[0][3], Equals, "using") } @@ -1885,8 +1902,8 @@ func (s *testSuite) TestCapturedBindingCharset(c *C) { tk.MustExec("admin capture bindings") rows := tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - c.Assert(rows[0][0], Equals, "update t set name = ? where name <= ?") - c.Assert(rows[0][1], Equals, "UPDATE /*+ use_index(@`upd_1` `test`.`t` `idx`)*/ `t` SET `name`=_ASCII'hello' WHERE `name`<=_ASCII'abc'") + c.Assert(rows[0][0], Equals, "update test . t set name = _ascii ? where name <= _ascii ?") + c.Assert(rows[0][1], Equals, "UPDATE /*+ use_index(@`upd_1` `test`.`t` `idx`)*/ `test`.`t` SET `name`=_ASCII'hello' WHERE `name`<=_ASCII'abc'") // Charset and Collation are empty now, they are not used currently. c.Assert(rows[0][6], Equals, "") c.Assert(rows[0][7], Equals, "") @@ -1906,8 +1923,77 @@ func (s *testSuite) TestUpdateSubqueryCapture(c *C) { tk.MustExec("admin capture bindings") rows := tk.MustQuery("show global bindings").Rows() c.Assert(len(rows), Equals, 1) - bindSQL := "UPDATE /*+ use_index(@`upd_1` `test`.`t1` `idx_b`), use_index(@`sel_1` `test`.`t2` ), hash_join(@`upd_1` `test`.`t1`), use_index(@`sel_2` `test`.`t2` )*/ `t1` SET `b`=1 WHERE `b`=2 AND (`a` IN (SELECT `a` FROM `t2` WHERE `b`=1) OR `c` IN (SELECT `a` FROM `t2` WHERE `b`=1))" + bindSQL := "UPDATE /*+ use_index(@`upd_1` `test`.`t1` `idx_b`), use_index(@`sel_1` `test`.`t2` ), hash_join(@`upd_1` `test`.`t1`), use_index(@`sel_2` `test`.`t2` )*/ `test`.`t1` SET `b`=1 WHERE `b`=2 AND (`a` IN (SELECT `a` FROM `test`.`t2` WHERE `b`=1) OR `c` IN (SELECT `a` FROM `test`.`t2` WHERE `b`=1))" c.Assert(rows[0][1], Equals, bindSQL) tk.MustExec(bindSQL) c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 0) } + +func (s *testSuite) TestIssue20417(c *C) { + tk := testkit.NewTestKit(c, s.store) + s.cleanBindingEnv(tk) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec(`CREATE TABLE t ( + pk VARBINARY(36) NOT NULL PRIMARY KEY, + b BIGINT NOT NULL, + c BIGINT NOT NULL, + pad VARBINARY(2048), + INDEX idxb(b), + INDEX idxc(c) + )`) + + // Test for create binding + s.cleanBindingEnv(tk) + tk.MustExec("create global binding for select * from t using select /*+ use_index(t, idxb) */ * from t") + rows := tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 1) + c.Assert(rows[0][0], Equals, "select * from test . t") + c.Assert(rows[0][1], Equals, "SELECT /*+ use_index(t idxb)*/ * FROM test.t") + c.Assert(tk.MustUseIndex("select * from t", "idxb(b)"), IsTrue) + c.Assert(tk.MustUseIndex("select * from test.t", "idxb(b)"), IsTrue) + + tk.MustExec("create global binding for select * from t WHERE b=2 AND c=3924541 using select /*+ use_index(@sel_1 test.t idxb) */ * from t WHERE b=2 AND c=3924541") + c.Assert(tk.MustUseIndex("SELECT /*+ use_index(@`sel_1` `test`.`t` `idxc`)*/ * FROM `test`.`t` WHERE `b`=2 AND `c`=3924541", "idxb(b)"), IsTrue) + c.Assert(tk.MustUseIndex("SELECT /*+ use_index(@`sel_1` `test`.`t` `idxc`)*/ * FROM `t` WHERE `b`=2 AND `c`=3924541", "idxb(b)"), IsTrue) + + // Test for capture baseline + s.cleanBindingEnv(tk) + stmtsummary.StmtSummaryByDigestMap.Clear() + tk.MustExec("set @@tidb_capture_plan_baselines = on") + s.domain.BindHandle().CaptureBaselines() + c.Assert(tk.Se.Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil), IsTrue) + tk.MustExec("select * from t where b=2 and c=213124") + tk.MustExec("select * from t where b=2 and c=213124") + tk.MustExec("admin capture bindings") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 1) + c.Assert(rows[0][0], Equals, "select * from test . t where b = ? and c = ?") + c.Assert(rows[0][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` `idxb`)*/ * FROM `test`.`t` WHERE `b`=2 AND `c`=213124") + tk.MustExec("set @@tidb_capture_plan_baselines = off") + + // Test for evolve baseline + s.cleanBindingEnv(tk) + tk.MustExec("set @@tidb_evolve_plan_baselines=1") + tk.MustExec("create global binding for select * from t WHERE c=3924541 using select /*+ use_index(@sel_1 test.t idxb) */ * from t WHERE c=3924541") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 1) + c.Assert(rows[0][0], Equals, "select * from test . t where c = ?") + c.Assert(rows[0][1], Equals, "SELECT /*+ use_index(@sel_1 test.t idxb)*/ * FROM test.t WHERE c = 3924541") + tk.MustExec("select /*+ use_index(t idxc)*/ * from t where c=3924541") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idxb") + tk.MustExec("admin flush bindings") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 2) + c.Assert(rows[1][0], Equals, "select * from test . t where c = ?") + c.Assert(rows[1][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` `idxc`), use_index(`t` `idxc`)*/ * FROM `test`.`t` WHERE `c`=3924541") + c.Assert(rows[1][3], Equals, "pending verify") + tk.MustExec("admin evolve bindings") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 2) + c.Assert(rows[1][0], Equals, "select * from test . t where c = ?") + c.Assert(rows[1][1], Equals, "SELECT /*+ use_index(@`sel_1` `test`.`t` `idxc`), use_index(`t` `idxc`)*/ * FROM `test`.`t` WHERE `c`=3924541") + status := rows[1][3].(string) + c.Assert(status == "using" || status == "rejected", IsTrue) + tk.MustExec("set @@tidb_evolve_plan_baselines=0") +} diff --git a/bindinfo/cache.go b/bindinfo/cache.go index c6a02cf101d81..46364affbd74d 100644 --- a/bindinfo/cache.go +++ b/bindinfo/cache.go @@ -216,7 +216,7 @@ func (br *BindRecord) shallowCopy() *BindRecord { } func (br *BindRecord) isSame(other *BindRecord) bool { - return br.OriginalSQL == other.OriginalSQL && br.Db == other.Db + return br.OriginalSQL == other.OriginalSQL } var statusIndex = map[string]int{ diff --git a/bindinfo/handle.go b/bindinfo/handle.go index 93d226367e1e7..21349685db15a 100644 --- a/bindinfo/handle.go +++ b/bindinfo/handle.go @@ -373,7 +373,7 @@ func (h *BindHandle) DropBindRecord(originalSQL, db string, binding *Binding) (e func (h *BindHandle) lockBindInfoTable() error { // h.sctx already locked. exec, _ := h.sctx.Context.(sqlexec.SQLExecutor) - _, err := exec.ExecuteInternal(context.TODO(), h.lockBindInfoSQL()) + _, err := exec.ExecuteInternal(context.TODO(), h.LockBindInfoSQL()) return err } @@ -539,7 +539,7 @@ func (c cache) removeDeletedBindRecord(hash string, meta *BindRecord) { func (c cache) setBindRecord(hash string, meta *BindRecord) { metas := c[hash] for i := range metas { - if metas[i].Db == meta.Db && metas[i].OriginalSQL == meta.OriginalSQL { + if metas[i].OriginalSQL == meta.OriginalSQL { metas[i] = meta return } @@ -568,7 +568,7 @@ func copyBindRecordUpdateMap(oldMap map[string]*bindRecordUpdate) map[string]*bi func (c cache) getBindRecord(hash, normdOrigSQL, db string) *BindRecord { bindRecords := c[hash] for _, bindRecord := range bindRecords { - if bindRecord.OriginalSQL == normdOrigSQL && bindRecord.Db == db { + if bindRecord.OriginalSQL == normdOrigSQL { return bindRecord } } @@ -577,9 +577,8 @@ func (c cache) getBindRecord(hash, normdOrigSQL, db string) *BindRecord { func (h *BindHandle) deleteBindInfoSQL(normdOrigSQL, db, bindSQL string) string { sql := fmt.Sprintf( - `DELETE FROM mysql.bind_info WHERE original_sql=%s AND LOWER(default_db)=%s`, + `DELETE FROM mysql.bind_info WHERE original_sql=%s`, expression.Quote(normdOrigSQL), - expression.Quote(db), ) if bindSQL == "" { return sql @@ -601,8 +600,8 @@ func (h *BindHandle) insertBindInfoSQL(orignalSQL string, db string, info Bindin ) } -// lockBindInfoSQL simulates LOCK TABLE by updating a same row in each pessimistic transaction. -func (h *BindHandle) lockBindInfoSQL() string { +// LockBindInfoSQL simulates LOCK TABLE by updating a same row in each pessimistic transaction. +func (h *BindHandle) LockBindInfoSQL() string { return fmt.Sprintf("UPDATE mysql.bind_info SET source=%s WHERE original_sql=%s", expression.Quote(Builtin), expression.Quote(BuiltinPseudoSQL4BindLock)) @@ -610,11 +609,10 @@ func (h *BindHandle) lockBindInfoSQL() string { func (h *BindHandle) logicalDeleteBindInfoSQL(originalSQL, db string, updateTs types.Time, bindingSQL string) string { updateTsStr := updateTs.String() - sql := fmt.Sprintf(`UPDATE mysql.bind_info SET status=%s,update_time=%s WHERE original_sql=%s and LOWER(default_db)=%s and update_time<%s`, + sql := fmt.Sprintf(`UPDATE mysql.bind_info SET status=%s,update_time=%s WHERE original_sql=%s and update_time<%s`, expression.Quote(deleted), expression.Quote(updateTsStr), expression.Quote(originalSQL), - expression.Quote(db), expression.Quote(updateTsStr)) if bindingSQL == "" { return sql @@ -635,12 +633,12 @@ func (h *BindHandle) CaptureBaselines() { if insertStmt, ok := stmt.(*ast.InsertStmt); ok && insertStmt.Select == nil { continue } - normalizedSQL, digest := parser.NormalizeDigest(bindableStmt.Query) dbName := utilparser.GetDefaultDB(stmt, bindableStmt.Schema) + normalizedSQL, digest := parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(stmt, dbName)) if r := h.GetBindRecord(digest, normalizedSQL, dbName); r != nil && r.HasUsingBinding() { continue } - bindSQL := GenerateBindSQL(context.TODO(), stmt, bindableStmt.PlanHint, true) + bindSQL := GenerateBindSQL(context.TODO(), stmt, bindableStmt.PlanHint, true, dbName) if bindSQL == "" { continue } @@ -680,7 +678,7 @@ func getHintsForSQL(sctx sessionctx.Context, sql string) (string, error) { } // GenerateBindSQL generates binding sqls from stmt node and plan hints. -func GenerateBindSQL(ctx context.Context, stmtNode ast.StmtNode, planHint string, captured bool) string { +func GenerateBindSQL(ctx context.Context, stmtNode ast.StmtNode, planHint string, captured bool, defaultDB string) string { // If would be nil for very simple cases such as point get, we do not need to evolve for them. if planHint == "" { return "" @@ -699,6 +697,7 @@ func GenerateBindSQL(ctx context.Context, stmtNode ast.StmtNode, planHint string hint.BindHint(stmtNode, &hint.HintsSet{}) var sb strings.Builder restoreCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, &sb) + restoreCtx.DefaultDB = defaultDB err := stmtNode.Restore(restoreCtx) if err != nil { logutil.Logger(ctx).Debug("[sql-bind] restore SQL failed when generating bind SQL", zap.Error(err)) diff --git a/bindinfo/session_handle.go b/bindinfo/session_handle.go index dc196ccadde39..2604d5b563f52 100644 --- a/bindinfo/session_handle.go +++ b/bindinfo/session_handle.go @@ -88,7 +88,7 @@ func (h *SessionHandle) GetBindRecord(normdOrigSQL, db string) *BindRecord { hash := parser.DigestNormalized(normdOrigSQL) bindRecords := h.ch[hash] for _, bindRecord := range bindRecords { - if bindRecord.OriginalSQL == normdOrigSQL && bindRecord.Db == db { + if bindRecord.OriginalSQL == normdOrigSQL { return bindRecord } } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index bcacb08366887..371f9a0bdf125 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -751,23 +751,52 @@ func (b *PlanBuilder) buildSet(ctx context.Context, v *ast.SetStmt) (Plan, error func (b *PlanBuilder) buildDropBindPlan(v *ast.DropBindingStmt) (Plan, error) { p := &SQLBindPlan{ SQLBindOp: OpSQLBindDrop, - NormdOrigSQL: parser.Normalize(v.OriginNode.Text()), + NormdOrigSQL: parser.Normalize(utilparser.RestoreWithDefaultDB(v.OriginNode, b.ctx.GetSessionVars().CurrentDB)), IsGlobal: v.GlobalScope, Db: utilparser.GetDefaultDB(v.OriginNode, b.ctx.GetSessionVars().CurrentDB), } if v.HintedNode != nil { - p.BindSQL = v.HintedNode.Text() + p.BindSQL = utilparser.RestoreWithDefaultDB(v.HintedNode, b.ctx.GetSessionVars().CurrentDB) } b.visitInfo = appendVisitInfo(b.visitInfo, mysql.SuperPriv, "", "", "", nil) return p, nil } +func checkHintedSQL(sql, charset, collation, db string) error { + p := parser.New() + hintsSet, _, warns, err := hint.ParseHintsSet(p, sql, charset, collation, db) + if err != nil { + return err + } + hintsStr, err := hintsSet.Restore() + if err != nil { + return err + } + // For `create global binding for select * from t using select * from t`, we allow it though hintsStr is empty. + // For `create global binding for select * from t using select /*+ non_exist_hint() */ * from t`, + // the hint is totally invalid, we escalate warning to error. + if hintsStr == "" && len(warns) > 0 { + return warns[0] + } + return nil +} + func (b *PlanBuilder) buildCreateBindPlan(v *ast.CreateBindingStmt) (Plan, error) { charSet, collation := b.ctx.GetSessionVars().GetCharsetInfo() + + // Because we use HintedNode.Restore instead of HintedNode.Text, so we need do some check here + // For example, if HintedNode.Text is `select /*+ non_exist_hint() */ * from t` and the current DB is `test`, + // the HintedNode.Restore will be `select * from test . t`. + // In other words, illegal hints will be deleted during restore. We can't check hinted SQL after restore. + // So we need check here. + if err := checkHintedSQL(v.HintedNode.Text(), charSet, collation, b.ctx.GetSessionVars().CurrentDB); err != nil { + return nil, err + } + p := &SQLBindPlan{ SQLBindOp: OpSQLBindCreate, - NormdOrigSQL: parser.Normalize(v.OriginNode.Text()), - BindSQL: v.HintedNode.Text(), + NormdOrigSQL: parser.Normalize(utilparser.RestoreWithDefaultDB(v.OriginNode, b.ctx.GetSessionVars().CurrentDB)), + BindSQL: utilparser.RestoreWithDefaultDB(v.HintedNode, b.ctx.GetSessionVars().CurrentDB), IsGlobal: v.GlobalScope, BindStmt: v.HintedNode, Db: utilparser.GetDefaultDB(v.OriginNode, b.ctx.GetSessionVars().CurrentDB), diff --git a/planner/core/preprocess.go b/planner/core/preprocess.go index 78b639e15e764..46ae538f5df5a 100644 --- a/planner/core/preprocess.go +++ b/planner/core/preprocess.go @@ -34,6 +34,7 @@ import ( driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/domainutil" + utilparser "github.com/pingcap/tidb/util/parser" ) // PreprocessOpt presents optional parameters to `Preprocess` method. @@ -179,14 +180,14 @@ func (p *preprocessor) Enter(in ast.Node) (out ast.Node, skipChildren bool) { p.stmtTp = TypeCreate EraseLastSemicolon(node.OriginNode) EraseLastSemicolon(node.HintedNode) - p.checkBindGrammar(node.OriginNode, node.HintedNode) + p.checkBindGrammar(node.OriginNode, node.HintedNode, p.ctx.GetSessionVars().CurrentDB) return in, true case *ast.DropBindingStmt: p.stmtTp = TypeDrop EraseLastSemicolon(node.OriginNode) if node.HintedNode != nil { EraseLastSemicolon(node.HintedNode) - p.checkBindGrammar(node.OriginNode, node.HintedNode) + p.checkBindGrammar(node.OriginNode, node.HintedNode, p.ctx.GetSessionVars().CurrentDB) } return in, true case *ast.RecoverTableStmt, *ast.FlashBackTableStmt: @@ -291,7 +292,7 @@ func bindableStmtType(node ast.StmtNode) byte { return TypeInvalid } -func (p *preprocessor) checkBindGrammar(originNode, hintedNode ast.StmtNode) { +func (p *preprocessor) checkBindGrammar(originNode, hintedNode ast.StmtNode, defaultDB string) { origTp := bindableStmtType(originNode) hintedTp := bindableStmtType(hintedNode) if origTp == TypeInvalid || hintedTp == TypeInvalid { @@ -309,8 +310,8 @@ func (p *preprocessor) checkBindGrammar(originNode, hintedNode ast.StmtNode) { return } } - originSQL := parser.Normalize(originNode.Text()) - hintedSQL := parser.Normalize(hintedNode.Text()) + originSQL := parser.Normalize(utilparser.RestoreWithDefaultDB(originNode, defaultDB)) + hintedSQL := parser.Normalize(utilparser.RestoreWithDefaultDB(hintedNode, defaultDB)) if originSQL != hintedSQL { p.err = errors.Errorf("hinted sql and origin sql don't match when hinted sql erase the hint info, after erase hint info, originSQL:%s, hintedSQL:%s", originSQL, hintedSQL) } diff --git a/planner/optimize.go b/planner/optimize.go index 7dd10aed04d48..941eecb123a49 100644 --- a/planner/optimize.go +++ b/planner/optimize.go @@ -38,6 +38,7 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/hint" "github.com/pingcap/tidb/util/logutil" + utilparser "github.com/pingcap/tidb/util/parser" "go.uber.org/zap" ) @@ -273,13 +274,26 @@ func optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is in return finalPlan, names, cost, err } -func extractSelectAndNormalizeDigest(stmtNode ast.StmtNode) (ast.StmtNode, string, string) { +func extractSelectAndNormalizeDigest(stmtNode ast.StmtNode, specifiledDB string) (ast.StmtNode, string, string) { switch x := stmtNode.(type) { case *ast.ExplainStmt: + // This function is only used to find bind record. + // For some SQLs, such as `explain select * from t`, they will be entered here many times, + // but some of them do not want to obtain bind record. + // The difference between them is whether len(x.Text()) is empty. They cannot be distinguished by stmt.restore. + // For these cases, we need return "" as normalize SQL and hash. + if len(x.Text()) == 0 { + return x.Stmt, "", "" + } switch x.Stmt.(type) { case *ast.SelectStmt, *ast.DeleteStmt, *ast.UpdateStmt, *ast.InsertStmt: plannercore.EraseLastSemicolon(x) - normalizeExplainSQL := parser.Normalize(x.Text()) + var normalizeExplainSQL string + if specifiledDB != "" { + normalizeExplainSQL = parser.Normalize(utilparser.RestoreWithDefaultDB(x, specifiledDB)) + } else { + normalizeExplainSQL = parser.Normalize(x.Text()) + } idx := int(0) switch n := x.Stmt.(type) { case *ast.SelectStmt: @@ -300,7 +314,12 @@ func extractSelectAndNormalizeDigest(stmtNode ast.StmtNode) (ast.StmtNode, strin return x.Stmt, normalizeSQL, hash case *ast.SetOprStmt: plannercore.EraseLastSemicolon(x) - normalizeExplainSQL := parser.Normalize(x.Text()) + var normalizeExplainSQL string + if specifiledDB != "" { + normalizeExplainSQL = parser.Normalize(utilparser.RestoreWithDefaultDB(x, specifiledDB)) + } else { + normalizeExplainSQL = parser.Normalize(x.Text()) + } idx := strings.Index(normalizeExplainSQL, "select") parenthesesIdx := strings.Index(normalizeExplainSQL, "(") if parenthesesIdx != -1 && parenthesesIdx < idx { @@ -312,7 +331,20 @@ func extractSelectAndNormalizeDigest(stmtNode ast.StmtNode) (ast.StmtNode, strin } case *ast.SelectStmt, *ast.SetOprStmt, *ast.DeleteStmt, *ast.UpdateStmt, *ast.InsertStmt: plannercore.EraseLastSemicolon(x) - normalizedSQL, hash := parser.NormalizeDigest(x.Text()) + // This function is only used to find bind record. + // For some SQLs, such as `explain select * from t`, they will be entered here many times, + // but some of them do not want to obtain bind record. + // The difference between them is whether len(x.Text()) is empty. They cannot be distinguished by stmt.restore. + // For these cases, we need return "" as normalize SQL and hash. + if len(x.Text()) == 0 { + return x, "", "" + } + var normalizedSQL, hash string + if specifiledDB != "" { + normalizedSQL, hash = parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(x, specifiledDB)) + } else { + normalizedSQL, hash = parser.NormalizeDigest(x.Text()) + } return x, normalizedSQL, hash } return nil, "", "" @@ -323,15 +355,12 @@ func getBindRecord(ctx sessionctx.Context, stmt ast.StmtNode) (*bindinfo.BindRec if ctx.Value(bindinfo.SessionBindInfoKeyType) == nil { return nil, "" } - stmtNode, normalizedSQL, hash := extractSelectAndNormalizeDigest(stmt) + stmtNode, normalizedSQL, hash := extractSelectAndNormalizeDigest(stmt, ctx.GetSessionVars().CurrentDB) if stmtNode == nil { return nil, "" } sessionHandle := ctx.Value(bindinfo.SessionBindInfoKeyType).(*bindinfo.SessionHandle) - bindRecord := sessionHandle.GetBindRecord(normalizedSQL, ctx.GetSessionVars().CurrentDB) - if bindRecord == nil { - bindRecord = sessionHandle.GetBindRecord(normalizedSQL, "") - } + bindRecord := sessionHandle.GetBindRecord(normalizedSQL, "") if bindRecord != nil { if bindRecord.HasUsingBinding() { return bindRecord, metrics.ScopeSession @@ -342,10 +371,7 @@ func getBindRecord(ctx sessionctx.Context, stmt ast.StmtNode) (*bindinfo.BindRec if globalHandle == nil { return nil, "" } - bindRecord = globalHandle.GetBindRecord(hash, normalizedSQL, ctx.GetSessionVars().CurrentDB) - if bindRecord == nil { - bindRecord = globalHandle.GetBindRecord(hash, normalizedSQL, "") - } + bindRecord = globalHandle.GetBindRecord(hash, normalizedSQL, "") return bindRecord, metrics.ScopeGlobal } @@ -364,7 +390,7 @@ func handleInvalidBindRecord(ctx context.Context, sctx sessionctx.Context, level } func handleEvolveTasks(ctx context.Context, sctx sessionctx.Context, br *bindinfo.BindRecord, stmtNode ast.StmtNode, planHint string) { - bindSQL := bindinfo.GenerateBindSQL(ctx, stmtNode, planHint, false) + bindSQL := bindinfo.GenerateBindSQL(ctx, stmtNode, planHint, false, br.Db) if bindSQL == "" { return } diff --git a/session/bootstrap.go b/session/bootstrap.go index c1a766d00432e..bbadab7eb34b5 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -28,6 +28,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/parser" "github.com/pingcap/parser/auth" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" @@ -35,11 +36,15 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/logutil" + utilparser "github.com/pingcap/tidb/util/parser" + "github.com/pingcap/tidb/util/sqlexec" "github.com/pingcap/tidb/util/timeutil" "go.uber.org/zap" ) @@ -449,9 +454,11 @@ const ( version59 = 59 // version60 redesigns `mysql.stats_extended` version60 = 60 + // version61 restore all SQL bindings. + version61 = 61 // please make sure this is the largest version - currentBootstrapVersion = version60 + currentBootstrapVersion = version61 ) var ( @@ -515,6 +522,8 @@ var ( upgradeToVer57, upgradeToVer58, upgradeToVer59, + upgradeToVer60, + upgradeToVer61, } ) @@ -1296,6 +1305,72 @@ func upgradeToVer60(s Session, ver int64) { doReentrantDDL(s, CreateStatsExtended) } +func upgradeToVer61(s Session, ver int64) { + if ver >= version61 { + return + } + h := &bindinfo.BindHandle{} + var err error + mustExecute(s, "BEGIN PESSIMISTIC") + + defer func() { + if err != nil { + mustExecute(s, "ROLLBACK") + return + } + + mustExecute(s, "COMMIT") + }() + mustExecute(s, h.LockBindInfoSQL()) + var recordSets []sqlexec.RecordSet + recordSets, err = s.ExecuteInternal(context.Background(), "SELECT original_sql, bind_sql, default_db, charset, collation from mysql.bind_info where source != 'builtin'") + if err != nil { + logutil.BgLogger().Fatal("upgradeToVer61 error", zap.Error(err)) + } + if len(recordSets) > 0 { + defer terror.Call(recordSets[0].Close) + } + req := recordSets[0].NewChunk() + iter := chunk.NewIterator4Chunk(req) + p := parser.New() + now := types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 3) + for { + err = recordSets[0].Next(context.TODO(), req) + if err != nil { + logutil.BgLogger().Fatal("upgradeToVer61 error", zap.Error(err)) + } + if req.NumRows() == 0 { + break + } + updateBindInfo(s, iter, p, now.String()) + } +} + +func updateBindInfo(s Session, iter *chunk.Iterator4Chunk, p *parser.Parser, now string) { + for row := iter.Begin(); row != iter.End(); row = iter.Next() { + original := row.GetString(0) + bind := row.GetString(1) + db := row.GetString(2) + charset := row.GetString(3) + collation := row.GetString(4) + originStmt, err := p.ParseOneStmt(original, charset, collation) + if err != nil { + logutil.BgLogger().Fatal("updateBindInfo error", zap.Error(err)) + } + bindStmt, err := p.ParseOneStmt(bind, charset, collation) + if err != nil { + logutil.BgLogger().Fatal("updateBindInfo error", zap.Error(err)) + } + originWithDB := parser.Normalize(utilparser.RestoreWithDefaultDB(originStmt, db)) + bindWithDB := utilparser.RestoreWithDefaultDB(bindStmt, db) + mustExecute(s, fmt.Sprintf(`UPDATE mysql.bind_info SET original_sql=%s, bind_sql=%s, default_db='', update_time=%s where original_sql=%s`, + expression.Quote(originWithDB), + expression.Quote(bindWithDB), + expression.Quote(now), + expression.Quote(original))) + } +} + func writeMemoryQuotaQuery(s Session) { comment := "memory_quota_query is 32GB by default in v3.0.x, 1GB by default in v4.0.x+" sql := fmt.Sprintf(`INSERT HIGH_PRIORITY INTO %s.%s VALUES ("%s", '%d', '%s') ON DUPLICATE KEY UPDATE VARIABLE_VALUE='%d'`, diff --git a/session/bootstrap_test.go b/session/bootstrap_test.go index 75a8b49f947a0..222e9d20bb480 100644 --- a/session/bootstrap_test.go +++ b/session/bootstrap_test.go @@ -505,3 +505,33 @@ func (s *testBootstrapSuite) TestStmtSummary(c *C) { c.Assert(row.GetBytes(0), BytesEquals, []byte("ON")) c.Assert(r.Close(), IsNil) } + +func (s *testBootstrapSuite) TestUpdateBindInfo(c *C) { + defer testleak.AfterTest(c)() + ctx := context.Background() + store, dom := newStoreWithBootstrap(c, s.dbName) + defer store.Close() + defer dom.Close() + se := newSession(c, store, s.dbName) + mustExecSQL(c, se, `insert into mysql.bind_info values('select * from t where a > ?','select /*+ use_index(t, idxb) */ * from t where a > 1', 'test','using', '2021-01-04 14:50:58.257','2021-01-04 14:50:58.257', 'utf8', 'utf8_general_ci', 'manual')`) + + upgradeToVer61(se, version60) + r := mustExecSQL(c, se, `select original_sql, bind_sql, default_db, status from mysql.bind_info where source != 'builtin'`) + req := r.NewChunk() + c.Assert(r.Next(ctx, req), IsNil) + row := req.GetRow(0) + c.Assert(row.GetString(0), Equals, "select * from test . t where a > ?") + c.Assert(row.GetString(1), Equals, "SELECT /*+ use_index(t idxb)*/ * FROM test.t WHERE a > 1") + c.Assert(row.GetString(2), Equals, "") + c.Assert(row.GetString(3), Equals, "using") + c.Assert(r.Close(), IsNil) + mustExecSQL(c, se, `drop global binding for select * from test.t where a > 1`) + r = mustExecSQL(c, se, `select original_sql, bind_sql, status from mysql.bind_info where source != 'builtin'`) + c.Assert(r.Next(ctx, req), IsNil) + row = req.GetRow(0) + c.Assert(row.GetString(0), Equals, "select * from test . t where a > ?") + c.Assert(row.GetString(1), Equals, "SELECT /*+ use_index(t idxb)*/ * FROM test.t WHERE a > 1") + c.Assert(row.GetString(2), Equals, "deleted") + c.Assert(r.Close(), IsNil) + mustExecSQL(c, se, `delete from mysql.bind_info where original_sql = 'select * from test . t where a > ?'`) +} diff --git a/types/parser_driver/value_expr.go b/types/parser_driver/value_expr.go index 27816e74f3785..27e9c9d2dd6ee 100644 --- a/types/parser_driver/value_expr.go +++ b/types/parser_driver/value_expr.go @@ -100,7 +100,10 @@ func (n *ValueExpr) Restore(ctx *format.RestoreCtx) error { case types.KindFloat64: ctx.WritePlain(strconv.FormatFloat(n.GetFloat64(), 'e', -1, 64)) case types.KindString: - if n.Type.Charset != "" { + // This part is used to process flag HasStringWithoutDefaultCharset, which means if we have this flag and the + // charset is mysql.DefaultCharset, we don't need to write the default. + if n.Type.Charset != "" && + (!ctx.Flags.HasStringWithoutDefaultCharset() || n.Type.Charset != mysql.DefaultCharset) { ctx.WritePlain("_") ctx.WriteKeyWord(n.Type.Charset) } diff --git a/util/parser/ast.go b/util/parser/ast.go index e7dfeec1a9474..7a1c1ebf4c31c 100644 --- a/util/parser/ast.go +++ b/util/parser/ast.go @@ -14,7 +14,10 @@ package parser import ( + "strings" + "github.com/pingcap/parser/ast" + "github.com/pingcap/parser/format" ) // GetDefaultDB checks if all columns in the AST have explicit DBName. If not, return specified DBName. @@ -45,3 +48,18 @@ func (i *implicitDatabase) Enter(in ast.Node) (out ast.Node, skipChildren bool) func (i *implicitDatabase) Leave(in ast.Node) (out ast.Node, ok bool) { return in, true } + +// RestoreWithDefaultDB returns restore strings for StmtNode with defaultDB +func RestoreWithDefaultDB(node ast.StmtNode, defaultDB string) string { + var sb strings.Builder + // Three flags for restore with default DB: + // 1. RestoreStringSingleQuotes specifies to use single quotes to surround the string; + // 2. RestoreSpacesAroundBinaryOperation specifies to add space around binary operation; + // 3. RestoreStringWithoutDefaultCharset specifies to not print default charset before string; + ctx := format.NewRestoreCtx(format.RestoreStringSingleQuotes|format.RestoreSpacesAroundBinaryOperation|format.RestoreStringWithoutDefaultCharset, &sb) + ctx.DefaultDB = defaultDB + if err := node.Restore(ctx); err != nil { + return "" + } + return sb.String() +} From 76614c6ca9a8ea480636221afbf45180517c0470 Mon Sep 17 00:00:00 2001 From: 9547 Date: Wed, 6 Jan 2021 17:24:42 +0800 Subject: [PATCH 12/16] util/chunk: fix slice out of bound panic (#21155) --- util/chunk/codec_test.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/util/chunk/codec_test.go b/util/chunk/codec_test.go index bbf54464f1391..5d715445ac9f1 100644 --- a/util/chunk/codec_test.go +++ b/util/chunk/codec_test.go @@ -105,14 +105,13 @@ func BenchmarkEncodeChunk(b *testing.B) { numCols := 4 numRows := 1024 - chk := &Chunk{columns: make([]*Column, numCols)} + colTypes := make([]*types.FieldType, numCols) for i := 0; i < numCols; i++ { - chk.columns[i] = &Column{ - length: numRows, - nullBitmap: make([]byte, numRows/8+1), - data: make([]byte, numRows*8), + colTypes[i] = &types.FieldType{ + Tp: mysql.TypeLonglong, } } + chk := NewChunkWithCapacity(colTypes, numRows) codec := &Codec{} @@ -127,17 +126,12 @@ func BenchmarkDecode(b *testing.B) { numRows := 1024 colTypes := make([]*types.FieldType, numCols) - chk := &Chunk{columns: make([]*Column, numCols)} for i := 0; i < numCols; i++ { - chk.columns[i] = &Column{ - length: numRows, - nullBitmap: make([]byte, numRows/8+1), - data: make([]byte, numRows*8), - } colTypes[i] = &types.FieldType{ Tp: mysql.TypeLonglong, } } + chk := NewChunkWithCapacity(colTypes, numRows) codec := &Codec{colTypes} buffer := codec.Encode(chk) From c7165bc51839ea3563cbea989870ba8f878e7514 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 6 Jan 2021 19:03:51 +0800 Subject: [PATCH 13/16] executor, planner: ON DUPLICATE UPDATE can refer to un-project col (#14412) Co-authored-by: Kenan Yao Co-authored-by: ti-srebot <66930949+ti-srebot@users.noreply.github.com> --- executor/builder.go | 1 + executor/insert.go | 22 ++++++-- executor/insert_common.go | 23 +++++++-- executor/insert_test.go | 9 ++++ executor/load_data.go | 2 +- planner/core/common_plans.go | 2 + planner/core/planbuilder.go | 92 ++++++++++++++++++++++++++++++++-- sessionctx/variable/session.go | 4 ++ types/field_name.go | 13 +++++ 9 files changed, 154 insertions(+), 14 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index 2958525d15f97..7fce1d4c71a37 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -761,6 +761,7 @@ func (b *executorBuilder) buildInsert(v *plannercore.Insert) Executor { allAssignmentsAreConstant: v.AllAssignmentsAreConstant, hasRefCols: v.NeedFillDefaultValue, SelectExec: selectExec, + rowLen: v.RowLen, } err := ivs.initInsertColumns() if err != nil { diff --git a/executor/insert.go b/executor/insert.go index ff4d23fa0a628..5af9bcd577ebf 100644 --- a/executor/insert.go +++ b/executor/insert.go @@ -179,11 +179,16 @@ func prefetchDataCache(ctx context.Context, txn kv.Transaction, rows []toBeCheck } // updateDupRow updates a duplicate row to a new row. -func (e *InsertExec) updateDupRow(ctx context.Context, txn kv.Transaction, row toBeCheckedRow, handle kv.Handle, onDuplicate []*expression.Assignment) error { +func (e *InsertExec) updateDupRow(ctx context.Context, idxInBatch int, txn kv.Transaction, row toBeCheckedRow, handle kv.Handle, onDuplicate []*expression.Assignment) error { oldRow, err := getOldRow(ctx, e.ctx, txn, row.t, handle, e.GenExprs) if err != nil { return err } + // get the extra columns from the SELECT clause and get the final `oldRow`. + if len(e.ctx.GetSessionVars().CurrInsertBatchExtraCols) > 0 { + extraCols := e.ctx.GetSessionVars().CurrInsertBatchExtraCols[idxInBatch] + oldRow = append(oldRow, extraCols...) + } err = e.doDupRowUpdate(ctx, handle, oldRow, row.row, e.OnDuplicate) if e.ctx.GetSessionVars().StmtCtx.DupKeyAsWarning && kv.ErrKeyExists.Equal(err) { @@ -229,7 +234,7 @@ func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.D return err } - err = e.updateDupRow(ctx, txn, r, handle, e.OnDuplicate) + err = e.updateDupRow(ctx, i, txn, r, handle, e.OnDuplicate) if err == nil { continue } @@ -251,7 +256,7 @@ func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.D return err } - err = e.updateDupRow(ctx, txn, r, handle, e.OnDuplicate) + err = e.updateDupRow(ctx, i, txn, r, handle, e.OnDuplicate) if err != nil { if kv.IsErrNotFound(err) { // Data index inconsistent? A unique key provide the handle information, but the @@ -297,6 +302,7 @@ func (e *InsertExec) Next(ctx context.Context, req *chunk.Chunk) error { // Close implements the Executor Close interface. func (e *InsertExec) Close() error { e.ctx.GetSessionVars().CurrInsertValues = chunk.Row{} + e.ctx.GetSessionVars().CurrInsertBatchExtraCols = e.ctx.GetSessionVars().CurrInsertBatchExtraCols[0:0:0] e.setMessage() if e.SelectExec != nil { return e.SelectExec.Close() @@ -327,12 +333,20 @@ func (e *InsertExec) initEvalBuffer4Dup() { // Use writable columns for old row for update. numWritableCols := len(e.Table.WritableCols()) - evalBufferTypes := make([]*types.FieldType, 0, numCols+numWritableCols) + extraLen := 0 + if e.SelectExec != nil { + extraLen = e.SelectExec.Schema().Len() - e.rowLen + } + + evalBufferTypes := make([]*types.FieldType, 0, numCols+numWritableCols+extraLen) // Append the old row before the new row, to be consistent with "Schema4OnDuplicate" in the "Insert" PhysicalPlan. for _, col := range e.Table.WritableCols() { evalBufferTypes = append(evalBufferTypes, &col.FieldType) } + if extraLen > 0 { + evalBufferTypes = append(evalBufferTypes, e.SelectExec.base().retFieldTypes[numWritableCols:]...) + } for _, col := range e.Table.Cols() { evalBufferTypes = append(evalBufferTypes, &col.FieldType) } diff --git a/executor/insert_common.go b/executor/insert_common.go index 9270a3f89b400..b15ba87aadb46 100644 --- a/executor/insert_common.go +++ b/executor/insert_common.go @@ -80,6 +80,8 @@ type InsertValues struct { lazyFillAutoID bool memTracker *memory.Tracker + rowLen int + stats *InsertRuntimeStat } @@ -421,7 +423,9 @@ func insertRowsFromSelect(ctx context.Context, base insertCommon) error { batchSize := sessVars.DMLBatchSize batchInsert := sessVars.BatchInsert && !sessVars.InTxn() && config.GetGlobalConfig().EnableBatchDML && batchSize > 0 memUsageOfRows := int64(0) + memUsageOfExtraCols := int64(0) memTracker := e.memTracker + extraColsInSel := make([][]types.Datum, 0, chk.Capacity()) for { err := Next(ctx, selectExec, chk) if err != nil { @@ -439,15 +443,20 @@ func insertRowsFromSelect(ctx context.Context, base insertCommon) error { if err != nil { return err } + extraColsInSel = append(extraColsInSel, innerRow[e.rowLen:]) rows = append(rows, row) if batchInsert && e.rowCount%uint64(batchSize) == 0 { memUsageOfRows = types.EstimatedMemUsage(rows[0], len(rows)) - memTracker.Consume(memUsageOfRows) + memUsageOfExtraCols = types.EstimatedMemUsage(extraColsInSel[0], len(extraColsInSel)) + memTracker.Consume(memUsageOfRows + memUsageOfExtraCols) + e.ctx.GetSessionVars().CurrInsertBatchExtraCols = extraColsInSel if err = base.exec(ctx, rows); err != nil { return err } rows = rows[:0] + extraColsInSel = extraColsInSel[:0] memTracker.Consume(-memUsageOfRows) + memTracker.Consume(-memUsageOfExtraCols) memUsageOfRows = 0 if err = e.doBatchInsert(ctx); err != nil { return err @@ -457,14 +466,18 @@ func insertRowsFromSelect(ctx context.Context, base insertCommon) error { if len(rows) != 0 { memUsageOfRows = types.EstimatedMemUsage(rows[0], len(rows)) - memTracker.Consume(memUsageOfRows) + memUsageOfExtraCols = types.EstimatedMemUsage(extraColsInSel[0], len(extraColsInSel)) + memTracker.Consume(memUsageOfRows + memUsageOfExtraCols) + e.ctx.GetSessionVars().CurrInsertBatchExtraCols = extraColsInSel } err = base.exec(ctx, rows) if err != nil { return err } rows = rows[:0] + extraColsInSel = extraColsInSel[:0] memTracker.Consume(-memUsageOfRows) + memTracker.Consume(-memUsageOfExtraCols) memTracker.Consume(-chkMemUsage) } return nil @@ -490,9 +503,9 @@ func (e *InsertValues) doBatchInsert(ctx context.Context) error { func (e *InsertValues) getRow(ctx context.Context, vals []types.Datum) ([]types.Datum, error) { row := make([]types.Datum, len(e.Table.Cols())) hasValue := make([]bool, len(e.Table.Cols())) - for i, v := range vals { - casted, err := table.CastValue(e.ctx, v, e.insertColumns[i].ToInfo(), false, false) - if e.handleErr(nil, &v, 0, err) != nil { + for i := 0; i < e.rowLen; i++ { + casted, err := table.CastValue(e.ctx, vals[i], e.insertColumns[i].ToInfo(), false, false) + if e.handleErr(nil, &vals[i], 0, err) != nil { return nil, err } diff --git a/executor/insert_test.go b/executor/insert_test.go index 64900d507286e..b2ddb3a784cbe 100644 --- a/executor/insert_test.go +++ b/executor/insert_test.go @@ -198,6 +198,15 @@ func (s *testSuite8) TestInsertOnDuplicateKey(c *C) { c.Assert(tk.Se.AffectedRows(), Equals, uint64(7)) tk.CheckLastMessage("Records: 5 Duplicates: 2 Warnings: 0") + tk.MustExec("drop table if exists a, b") + tk.MustExec("create table a(x int primary key)") + tk.MustExec("create table b(x int, y int)") + tk.MustExec("insert into a values(1)") + tk.MustExec("insert into b values(1, 2)") + tk.MustExec("insert into a select x from b ON DUPLICATE KEY UPDATE a.x=b.y") + c.Assert(tk.Se.AffectedRows(), Equals, uint64(2)) + tk.MustQuery("select * from a").Check(testkit.Rows("2")) + // reproduce insert on duplicate key update bug under new row format. tk.MustExec(`drop table if exists t1`) tk.MustExec(`create table t1(c1 decimal(6,4), primary key(c1))`) diff --git a/executor/load_data.go b/executor/load_data.go index 429b9f3d48cd3..ef3a78ad63a63 100644 --- a/executor/load_data.go +++ b/executor/load_data.go @@ -162,7 +162,7 @@ func (e *LoadDataInfo) initLoadColumns(columnNames []string) error { break } } - + e.rowLen = len(e.insertColumns) // Check column whether is specified only once. err = table.CheckOnce(cols) if err != nil { diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index bc1c65b3cdd8b..72da1438dffaa 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -725,6 +725,8 @@ type Insert struct { NeedFillDefaultValue bool AllAssignmentsAreConstant bool + + RowLen int } // Update represents Update plan. diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 371f9a0bdf125..dd7fa788e6cc8 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -813,6 +813,9 @@ func (b *PlanBuilder) detectSelectAgg(sel *ast.SelectStmt) bool { return true } for _, f := range sel.Fields.Fields { + if f.WildCard != nil { + continue + } if ast.HasAggFlag(f.Expr) { return true } @@ -2668,18 +2671,91 @@ func (b *PlanBuilder) buildValuesListOfInsert(ctx context.Context, insert *ast.I return nil } +type colNameInOnDupExtractor struct { + colNameMap map[types.FieldName]*ast.ColumnNameExpr +} + +func (c *colNameInOnDupExtractor) Enter(node ast.Node) (ast.Node, bool) { + switch x := node.(type) { + case *ast.ColumnNameExpr: + fieldName := types.FieldName{ + DBName: x.Name.Schema, + TblName: x.Name.Table, + ColName: x.Name.Name, + } + c.colNameMap[fieldName] = x + return node, true + // We don't extract the column names from the sub select. + case *ast.SelectStmt, *ast.SetOprStmt: + return node, true + default: + return node, false + } +} + +func (c *colNameInOnDupExtractor) Leave(node ast.Node) (ast.Node, bool) { + return node, true +} + func (b *PlanBuilder) buildSelectPlanOfInsert(ctx context.Context, insert *ast.InsertStmt, insertPlan *Insert) error { affectedValuesCols, err := b.getAffectCols(insert, insertPlan) if err != nil { return err } + actualColLen := -1 + // For MYSQL, it handles the case that the columns in ON DUPLICATE UPDATE is not the project column of the SELECT clause + // but just in the table occurs in the SELECT CLAUSE. + // e.g. insert into a select x from b ON DUPLICATE KEY UPDATE a.x=b.y; the `y` is not a column of select's output. + // MySQL won't throw error and will execute this SQL successfully. + // To make compatible with this strange feature, we add the variable `actualColLen` and the following IF branch. + if len(insert.OnDuplicate) > 0 { + // If the select has aggregation, it cannot see the columns not in the select field. + // e.g. insert into a select x from b ON DUPLICATE KEY UPDATE a.x=b.y; can be executed successfully. + // insert into a select x from b group by x ON DUPLICATE KEY UPDATE a.x=b.y; will report b.y not found. + if sel, ok := insert.Select.(*ast.SelectStmt); ok && !b.detectSelectAgg(sel) { + hasWildCard := false + for _, field := range sel.Fields.Fields { + if field.WildCard != nil { + hasWildCard = true + break + } + } + if !hasWildCard { + colExtractor := &colNameInOnDupExtractor{colNameMap: make(map[types.FieldName]*ast.ColumnNameExpr)} + for _, assign := range insert.OnDuplicate { + assign.Expr.Accept(colExtractor) + } + actualColLen = len(sel.Fields.Fields) + for _, colName := range colExtractor.colNameMap { + // If we found the name from the INSERT's table columns, we don't try to find it in select field anymore. + if insertPlan.tableColNames.FindAstColName(colName.Name) { + continue + } + found := false + for _, field := range sel.Fields.Fields { + if colField, ok := field.Expr.(*ast.ColumnNameExpr); ok && + (colName.Name.Schema.L == "" || colField.Name.Schema.L == "" || colName.Name.Schema.L == colField.Name.Schema.L) && + (colName.Name.Table.L == "" || colField.Name.Table.L == "" || colName.Name.Table.L == colField.Name.Table.L) && + colName.Name.Name.L == colField.Name.Name.L { + found = true + break + } + } + if found { + continue + } + sel.Fields.Fields = append(sel.Fields.Fields, &ast.SelectField{Expr: colName, Offset: len(sel.Fields.Fields)}) + } + } + } + } selectPlan, err := b.Build(ctx, insert.Select) if err != nil { return err } // Check to guarantee that the length of the row returned by select is equal to that of affectedValuesCols. - if selectPlan.Schema().Len() != len(affectedValuesCols) { + if (actualColLen == -1 && selectPlan.Schema().Len() != len(affectedValuesCols)) || (actualColLen != -1 && actualColLen != len(affectedValuesCols)) { return ErrWrongValueCountOnRow.GenWithStackByArgs(1) } @@ -2702,12 +2778,17 @@ func (b *PlanBuilder) buildSelectPlanOfInsert(ctx context.Context, insert *ast.I return err } + if actualColLen == -1 { + actualColLen = selectPlan.Schema().Len() + } + insertPlan.RowLen = actualColLen // schema4NewRow is the schema for the newly created data record based on // the result of the select statement. schema4NewRow := expression.NewSchema(make([]*expression.Column, len(insertPlan.Table.Cols()))...) names4NewRow := make(types.NameSlice, len(insertPlan.Table.Cols())) // TODO: don't clone it. - for i, selCol := range insertPlan.SelectPlan.Schema().Columns { + for i := 0; i < actualColLen; i++ { + selCol := insertPlan.SelectPlan.Schema().Columns[i] ordinal := affectedValuesCols[i].Offset schema4NewRow.Columns[ordinal] = &expression.Column{} *schema4NewRow.Columns[ordinal] = *selCol @@ -2723,8 +2804,11 @@ func (b *PlanBuilder) buildSelectPlanOfInsert(ctx context.Context, insert *ast.I names4NewRow[i] = types.EmptyName } } - insertPlan.Schema4OnDuplicate = expression.MergeSchema(insertPlan.tableSchema, schema4NewRow) - insertPlan.names4OnDuplicate = append(insertPlan.tableColNames.Shallow(), names4NewRow...) + insertPlan.Schema4OnDuplicate = expression.NewSchema(insertPlan.tableSchema.Columns...) + insertPlan.Schema4OnDuplicate.Append(insertPlan.SelectPlan.Schema().Columns[actualColLen:]...) + insertPlan.Schema4OnDuplicate.Append(schema4NewRow.Columns...) + insertPlan.names4OnDuplicate = append(insertPlan.tableColNames.Shallow(), names[actualColLen:]...) + insertPlan.names4OnDuplicate = append(insertPlan.names4OnDuplicate, names4NewRow...) return nil } diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index cafb9bc6b79ef..6b9feb6df1123 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -529,6 +529,10 @@ type SessionVars struct { // See http://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html#function_values CurrInsertValues chunk.Row + // In https://github.com/pingcap/tidb/issues/14164, we can see that MySQL can enter the column that is not in the insert's SELECT's output. + // We store the extra columns in this variable. + CurrInsertBatchExtraCols [][]types.Datum + // Per-connection time zones. Each client that connects has its own time zone setting, given by the session time_zone variable. // See https://dev.mysql.com/doc/refman/5.7/en/time-zone-support.html TimeZone *time.Location diff --git a/types/field_name.go b/types/field_name.go index a026b2a6fe98e..1212d760c5c2d 100644 --- a/types/field_name.go +++ b/types/field_name.go @@ -16,6 +16,7 @@ package types import ( "strings" + "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" ) @@ -60,3 +61,15 @@ func (s NameSlice) Shallow() NameSlice { // EmptyName is to occupy the position in the name slice. If it's set, that column's name is hidden. var EmptyName = &FieldName{Hidden: true} + +// FindAstColName checks whether the given ast.ColumnName is appeared in this slice. +func (s NameSlice) FindAstColName(name *ast.ColumnName) bool { + for _, fieldName := range s { + if (name.Schema.L == "" || name.Schema.L == fieldName.DBName.L) && + (name.Table.L == "" || name.Table.L == fieldName.TblName.L) && + name.Name.L == fieldName.ColName.L { + return true + } + } + return false +} From bf6b222d938562aceaa4deea51b12cdcd74a877d Mon Sep 17 00:00:00 2001 From: tangenta Date: Thu, 7 Jan 2021 11:08:47 +0800 Subject: [PATCH 14/16] .github: update the label for sig/infra (#22237) --- .github/labeler.yml | 7 ++++++- .github/workflows/assign_project.yml | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index dc15adb5d9ad6..2249e87175e37 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -22,10 +22,15 @@ sig/planner: component/statistics: - statistics/* -sig/DDL: +sig/infra: - ddl/* - domain/* - infoschema/* + - session/* + - server/* + - privilege/* + - plugin/* + - config/* - meta/* - owner/* diff --git a/.github/workflows/assign_project.yml b/.github/workflows/assign_project.yml index 3d8999ee5a29f..87a8ee8b3f70b 100644 --- a/.github/workflows/assign_project.yml +++ b/.github/workflows/assign_project.yml @@ -53,7 +53,7 @@ jobs: - name: Run issues assignment to project SIG DDL Kanban uses: srggrs/assign-one-project-github-action@1.2.0 if: | - contains(github.event.issue.labels.*.name, 'sig/DDL') || + contains(github.event.issue.labels.*.name, 'sig/infra') || contains(github.event.issue.labels.*.name, 'component/binlog') || contains(github.event.issue.labels.*.name, 'component/charset') || contains(github.event.issue.labels.*.name, 'component/infoschema') || From 6e98d18e94ad7163bcad503aa7b80ea613245fac Mon Sep 17 00:00:00 2001 From: Zhuhe Fang Date: Thu, 7 Jan 2021 11:41:44 +0800 Subject: [PATCH 15/16] planner: push down projection for tiflash (#21459) --- executor/partition_table.go | 2 + go.mod | 2 +- go.sum | 4 +- planner/core/exhaust_physical_plans.go | 47 +- planner/core/find_best_task.go | 4 +- planner/core/integration_test.go | 78 ++ planner/core/plan_to_pb.go | 25 + planner/core/rule_eliminate_projection.go | 9 + planner/core/task.go | 19 +- .../testdata/integration_serial_suite_in.json | 36 + .../integration_serial_suite_out.json | 1233 +++++++++++------ 11 files changed, 970 insertions(+), 489 deletions(-) diff --git a/executor/partition_table.go b/executor/partition_table.go index 957a2ea5133d4..cfc030e04d7a3 100644 --- a/executor/partition_table.go +++ b/executor/partition_table.go @@ -191,6 +191,8 @@ func updateExecutorTableID(ctx context.Context, exec *tipb.Executor, partitionID child = nil case tipb.ExecType_TypeJoin: child = exec.Join.Children[1-exec.Join.InnerIdx] + case tipb.ExecType_TypeProjection: + child = exec.Projection.Child default: return errors.Trace(fmt.Errorf("unknown new tipb protocol %d", exec.Tp)) } diff --git a/go.mod b/go.mod index 502864a3464d3..fc4080ffa4790 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/pingcap/parser v0.0.0-20201222091346-02c8ff27d0bc github.com/pingcap/sysutil v0.0.0-20201130064824-f0c8aa6a6966 github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible - github.com/pingcap/tipb v0.0.0-20201209065231-aa39b1b86217 + github.com/pingcap/tipb v0.0.0-20201229060814-148bc717ce4c github.com/prometheus/client_golang v1.5.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.9.1 diff --git a/go.sum b/go.sum index f5045709b0b9a..593d7d73545bd 100644 --- a/go.sum +++ b/go.sum @@ -671,8 +671,8 @@ github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= -github.com/pingcap/tipb v0.0.0-20201209065231-aa39b1b86217 h1:Ophn4Ud/QHp1BH0FJOzbAVBW9Mw8BlX0gtWkK7ubDy0= -github.com/pingcap/tipb v0.0.0-20201209065231-aa39b1b86217/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tipb v0.0.0-20201229060814-148bc717ce4c h1:kvrdp2hY+asgSvVXCj4eebA9DH4SSouRVQUZpa1Se/Y= +github.com/pingcap/tipb v0.0.0-20201229060814-148bc717ce4c/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pingcap/tiup v1.2.3 h1:8OCQF7sHhT6VqE8pZU1JTSogPA90OFuWWM/B746x0YY= github.com/pingcap/tiup v1.2.3/go.mod h1:q8WzflNHjE1U49k2qstTL0clx2pKh8pkOzUFV4RTvQo= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index d00b64f73a2af..dfaae5bdacbfd 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -1895,10 +1895,7 @@ func (p *LogicalJoin) tryToGetBroadCastJoinByPreferGlobalIdx(prop *property.Phys // When a sort column will be replaced by scalar function, we refuse it. // When a sort column will be replaced by a constant, we just remove it. func (p *LogicalProjection) TryToGetChildProp(prop *property.PhysicalProperty) (*property.PhysicalProperty, bool) { - if prop.IsFlashProp() { - return nil, false - } - newProp := &property.PhysicalProperty{TaskTp: property.RootTaskType, ExpectedCnt: prop.ExpectedCnt} + newProp := &property.PhysicalProperty{TaskTp: prop.TaskTp, ExpectedCnt: prop.ExpectedCnt} newCols := make([]property.SortItem, 0, len(prop.SortItems)) for _, col := range prop.SortItems { idx := p.schema.ColumnIndex(col.Col) @@ -1928,13 +1925,7 @@ func (p *LogicalProjection) exhaustPhysicalPlans(prop *property.PhysicalProperty } func (lt *LogicalTopN) canPushToCop() bool { - // At present, only Aggregation, Limit, TopN can be pushed to cop task, and Projection will be supported in the future. - // When we push task to coprocessor, finishCopTask will close the cop task and create a root task in the current implementation. - // Thus, we can't push two different tasks to coprocessor now, and can only push task to coprocessor when the child is Datasource. - - // TODO: develop this function after supporting push several tasks to coprecessor and supporting Projection to coprocessor. - _, ok := lt.children[0].(*DataSource) - return ok + return lt.canChildPushDown() } func (lt *LogicalTopN) getPhysTopN(prop *property.PhysicalProperty) []PhysicalPlan { @@ -2092,15 +2083,25 @@ func (p *LogicalWindow) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([ func (p *baseLogicalPlan) exhaustPhysicalPlans(_ *property.PhysicalProperty) ([]PhysicalPlan, bool) { panic("baseLogicalPlan.exhaustPhysicalPlans() should never be called.") } - -func (la *LogicalAggregation) canPushToCop() bool { - // At present, only Aggregation, Limit, TopN can be pushed to cop task, and Projection will be supported in the future. - // When we push task to coprocessor, finishCopTask will close the cop task and create a root task in the current implementation. +func (p *baseLogicalPlan) canChildPushDown() bool { + // At present for TiKV, only Aggregation, Limit, TopN can be pushed to cop task, and Projection will be supported in the future. + // When we push task to coprocessor, convertToRootTask will close the cop task and create a root task in the current implementation. // Thus, we can't push two different tasks to coprocessor now, and can only push task to coprocessor when the child is Datasource. + // But for TiFlash, Projection, Join and DataSource can also be pushed down in most cases. Other operators will be + // supported in the future, such as Aggregation, Limit, TopN. + switch p.children[0].(type) { + case *DataSource: + return true + case *LogicalJoin, *LogicalProjection: + // TiFlash supports pushing down more operators + return p.SCtx().GetSessionVars().AllowBCJ || p.SCtx().GetSessionVars().AllowMPPExecution + default: + return false + } +} - // TODO: develop this function after supporting push several tasks to coprecessor and supporting Projection to coprocessor. - _, ok := la.children[0].(*DataSource) - return ok && !la.noCopPushDown +func (la *LogicalAggregation) canPushToCop() bool { + return la.canChildPushDown() && !la.noCopPushDown } func (la *LogicalAggregation) getEnforcedStreamAggs(prop *property.PhysicalProperty) []PhysicalPlan { @@ -2322,20 +2323,14 @@ func (p *LogicalSelection) exhaustPhysicalPlans(prop *property.PhysicalProperty) } func (p *LogicalLimit) canPushToCop() bool { - // At present, only Aggregation, Limit, TopN can be pushed to cop task, and Projection will be supported in the future. - // When we push task to coprocessor, finishCopTask will close the cop task and create a root task in the current implementation. - // Thus, we can't push two different tasks to coprocessor now, and can only push task to coprocessor when the child is Datasource. - - // TODO: develop this function after supporting push several tasks to coprecessor and supporting Projection to coprocessor. - _, ok := p.children[0].(*DataSource) - return ok + return p.canChildPushDown() } func (p *LogicalLimit) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { if !prop.IsEmpty() { return nil, true } - // allTaskTypes := prop.GetAllPossibleChildTaskTypes() + if p.limitHints.preferLimitToCop { if !p.canPushToCop() { errMsg := "Optimizer Hint LIMIT_TO_COP is inapplicable" diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 58ecf757d55d6..9e33e86b60211 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -244,7 +244,8 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl // Combine best child tasks with parent physical plan. curTask := pp.attach2Task(childTasks...) - if _, ok := curTask.(*mppTask); ok && prop.TaskTp == property.RootTaskType { + // An optimal task could not satisfy the property, so it should be converted here. + if _, ok := curTask.(*rootTask); !ok && prop.TaskTp == property.RootTaskType { curTask = curTask.convertToRootTask(p.ctx) } @@ -266,7 +267,6 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl bestTask = curTask break } - // Get the most efficient one. if curTask.cost() < bestTask.cost() || (bestTask.invalid() && !curTask.invalid()) { bestTask = curTask diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index b7ede55ed52fc..6d3fc1a042f08 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -2426,3 +2426,81 @@ func (s *testIntegrationSuite) TestIssue22071(c *C) { tk.MustQuery("select n in (1,2) from (select a in (1,2) as n from t) g;").Sort().Check(testkit.Rows("0", "1", "1")) tk.MustQuery("select n in (1,n) from (select a in (1,2) as n from t) g;").Check(testkit.Rows("1", "1", "1")) } + +func (s *testIntegrationSerialSuite) TestPushDownProjectionForTiFlash(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (id int, value decimal(6,3))") + tk.MustExec("analyze table t") + + // Create virtual tiflash replica info. + dom := domain.GetDomain(tk.Se) + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + c.Assert(exists, IsTrue) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "t" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + + tk.MustExec("set @@tidb_opt_broadcast_join=1;") + + var input []string + var output []struct { + SQL string + Plan []string + } + s.testData.GetTestCases(c, &input, &output) + for i, tt := range input { + s.testData.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + }) + res := tk.MustQuery(tt) + res.Check(testkit.Rows(output[i].Plan...)) + } +} + +func (s *testIntegrationSerialSuite) TestPushDownProjectionForMPP(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (id int, value decimal(6,3))") + tk.MustExec("analyze table t") + + // Create virtual tiflash replica info. + dom := domain.GetDomain(tk.Se) + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + c.Assert(exists, IsTrue) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "t" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + + tk.MustExec("set @@tidb_allow_mpp=1; set @@tidb_opt_broadcast_join=0;") + + var input []string + var output []struct { + SQL string + Plan []string + } + s.testData.GetTestCases(c, &input, &output) + for i, tt := range input { + s.testData.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + }) + res := tk.MustQuery(tt) + res.Check(testkit.Rows(output[i].Plan...)) + } +} diff --git a/planner/core/plan_to_pb.go b/planner/core/plan_to_pb.go index 385ab9aadfbaf..7f3a34e459ee6 100644 --- a/planner/core/plan_to_pb.go +++ b/planner/core/plan_to_pb.go @@ -110,6 +110,31 @@ func (p *PhysicalSelection) ToPB(ctx sessionctx.Context, storeType kv.StoreType) return &tipb.Executor{Tp: tipb.ExecType_TypeSelection, Selection: selExec, ExecutorId: &executorID}, nil } +// ToPB implements PhysicalPlan ToPB interface. +func (p *PhysicalProjection) ToPB(ctx sessionctx.Context, storeType kv.StoreType) (*tipb.Executor, error) { + sc := ctx.GetSessionVars().StmtCtx + client := ctx.GetClient() + exprs, err := expression.ExpressionsToPBList(sc, p.Exprs, client) + if err != nil { + return nil, err + } + projExec := &tipb.Projection{ + Exprs: exprs, + } + executorID := "" + if storeType == kv.TiFlash { + var err error + projExec.Child, err = p.children[0].ToPB(ctx, storeType) + if err != nil { + return nil, errors.Trace(err) + } + executorID = p.ExplainID().String() + } else { + return nil, errors.Errorf("The projection can only be pushed down to TiFlash now, not %s.", storeType.Name()) + } + return &tipb.Executor{Tp: tipb.ExecType_TypeProjection, Projection: projExec, ExecutorId: &executorID}, nil +} + // ToPB implements PhysicalPlan ToPB interface. func (p *PhysicalTopN) ToPB(ctx sessionctx.Context, storeType kv.StoreType) (*tipb.Executor, error) { sc := ctx.GetSessionVars().StmtCtx diff --git a/planner/core/rule_eliminate_projection.go b/planner/core/rule_eliminate_projection.go index 19326a7cb3ed4..0dd362ad29094 100644 --- a/planner/core/rule_eliminate_projection.go +++ b/planner/core/rule_eliminate_projection.go @@ -17,6 +17,7 @@ import ( "context" "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/kv" ) // canProjectionBeEliminatedLoose checks whether a projection can be eliminated, @@ -82,6 +83,14 @@ func doPhysicalProjectionElimination(p PhysicalPlan) PhysicalPlan { p.Children()[i] = doPhysicalProjectionElimination(child) } + // eliminate projection in a coprocessor task + tableReader, isTableReader := p.(*PhysicalTableReader) + if isTableReader && tableReader.StoreType == kv.TiFlash { + tableReader.tablePlan = eliminatePhysicalProjection(tableReader.tablePlan) + tableReader.TablePlans = flattenPushDownPlan(tableReader.tablePlan) + return p + } + proj, isProj := p.(*PhysicalProjection) if !isProj || !canProjectionBeEliminatedStrict(proj) { return p diff --git a/planner/core/task.go b/planner/core/task.go index 024d5dcee7b40..c24d1c1497f65 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -1042,8 +1042,23 @@ func (p *PhysicalProjection) GetCost(count float64) float64 { } func (p *PhysicalProjection) attach2Task(tasks ...task) task { - // TODO: support projection push down. - var t task = tasks[0].convertToRootTask(p.ctx) + t := tasks[0].copy() + if cop, ok := t.(*copTask); ok { + if len(cop.rootTaskConds) == 0 && cop.getStoreType() == kv.TiFlash && expression.CanExprsPushDown(p.ctx.GetSessionVars().StmtCtx, p.Exprs, p.ctx.GetClient(), cop.getStoreType()) { + copTask := attachPlan2Task(p, cop) + copTask.addCost(p.GetCost(t.count())) + return copTask + } + } else if mpp, ok := t.(*mppTask); ok { + if expression.CanExprsPushDown(p.ctx.GetSessionVars().StmtCtx, p.Exprs, p.ctx.GetClient(), kv.TiFlash) { + p.SetChildren(mpp.p) + mpp.p = p + mpp.addCost(p.GetCost(t.count())) + return mpp + } + } + // TODO: support projection push down for TiKV. + t = t.convertToRootTask(p.ctx) t = attachPlan2Task(p, t) t.addCost(p.GetCost(t.count())) return t diff --git a/planner/core/testdata/integration_serial_suite_in.json b/planner/core/testdata/integration_serial_suite_in.json index 8b5cf78316eba..cfbee02d427d3 100644 --- a/planner/core/testdata/integration_serial_suite_in.json +++ b/planner/core/testdata/integration_serial_suite_in.json @@ -128,5 +128,41 @@ "explain select /*+ inl_hash_join(s) */ * from t join s on t.a=s.a and t.b = s.a", "explain select /*+ inl_hash_join(s) */ * from t join s on t.a=s.a and t.a = s.b" ] + }, + { + "name": "TestPushDownProjectionForTiFlash", + "cases": [ + "desc select /*+ hash_agg()*/ count(b) from (select id + 1 as b from t)A", + "desc select /*+ hash_agg()*/ count(*) from (select id + 1 as b from t)A", + "desc select /*+ hash_agg()*/ sum(b) from (select id + 1 as b from t)A", + "desc select /*+ stream_agg()*/ count(b) from (select id + 1 as b from t)A", + "desc select /*+ stream_agg()*/ count(*) from (select id + 1 as b from t)A", + "desc select /*+ stream_agg()*/ sum(b) from (select id + 1 as b from t)A", + "desc select * from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "desc select * from t join (select id-2 as b from t) A on A.b=t.id", + "desc select * from t left join (select id-2 as b from t) A on A.b=t.id", + "desc select * from t right join (select id-2 as b from t) A on A.b=t.id", + "desc select A.b, B.b from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "desc select A.id from t as A where exists (select 1 from t where t.id=A.id)", + "desc select A.id from t as A where not exists (select 1 from t where t.id=A.id)" + ] + }, + { + "name": "TestPushDownProjectionForMPP", + "cases": [ + "desc select /*+ hash_agg()*/ count(b) from (select id + 1 as b from t)A", + "desc select /*+ hash_agg()*/ count(*) from (select id + 1 as b from t)A", + "desc select /*+ hash_agg()*/ sum(b) from (select id + 1 as b from t)A", + "desc select /*+ stream_agg()*/ count(b) from (select id + 1 as b from t)A", + "desc select /*+ stream_agg()*/ count(*) from (select id + 1 as b from t)A", + "desc select /*+ stream_agg()*/ sum(b) from (select id + 1 as b from t)A", + "desc select * from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "desc select * from t join (select id-2 as b from t) A on A.b=t.id", + "desc select * from t left join (select id-2 as b from t) A on A.b=t.id", + "desc select * from t right join (select id-2 as b from t) A on A.b=t.id", + "desc select A.b, B.b from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "desc select id from t as A where exists (select 1 from t where t.id=A.id)", + "desc select id from t as A where not exists (select 1 from t where t.id=A.id)" + ] } ] diff --git a/planner/core/testdata/integration_serial_suite_out.json b/planner/core/testdata/integration_serial_suite_out.json index ff95dd2edb3bf..9c62e4a120a35 100644 --- a/planner/core/testdata/integration_serial_suite_out.json +++ b/planner/core/testdata/integration_serial_suite_out.json @@ -48,211 +48,215 @@ { "SQL": "explain select count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", "Plan": [ - "StreamAgg_11 1.00 root funcs:count(1)->Column#11", - "└─TableReader_35 8.00 root data:ExchangeSender_34", - " └─ExchangeSender_34 8.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_31 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", - " ├─ExchangeReceiver_19(Build) 2.00 cop[tiflash] ", - " │ └─ExchangeSender_18 2.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_17 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_16 2.00 cop[tiflash] table:d1_t keep order:false", - " └─Selection_21(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_20 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_24 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_26 1.00 root data:ExchangeSender_25", + " └─ExchangeSender_25 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_17 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", + " ├─ExchangeReceiver_21(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender_20 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_19 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_18 2.00 cop[tiflash] table:d1_t keep order:false", + " └─Selection_23(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_22 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t, d1_t, d2_t, d3_t where fact_t.d1_k = d1_t.d1_k and fact_t.d2_k = d2_t.d2_k and fact_t.d3_k = d3_t.d3_k", "Plan": [ - "StreamAgg_18 1.00 root funcs:count(1)->Column#17", - "└─TableReader_68 8.00 root data:ExchangeSender_67", - " └─ExchangeSender_67 8.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_64 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]", - " ├─ExchangeReceiver_38(Build) 2.00 cop[tiflash] ", - " │ └─ExchangeSender_37 2.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_36 2.00 cop[tiflash] not(isnull(test.d3_t.d3_k))", - " │ └─TableFullScan_35 2.00 cop[tiflash] table:d3_t keep order:false", - " └─HashJoin_23(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]", - " ├─ExchangeReceiver_34(Build) 2.00 cop[tiflash] ", - " │ └─ExchangeSender_33 2.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_32 2.00 cop[tiflash] not(isnull(test.d2_t.d2_k))", - " │ └─TableFullScan_31 2.00 cop[tiflash] table:d2_t keep order:false", - " └─HashJoin_24(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", - " ├─ExchangeReceiver_28(Build) 2.00 cop[tiflash] ", - " │ └─ExchangeSender_27 2.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_26 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_25 2.00 cop[tiflash] table:d1_t keep order:false", - " └─Selection_30(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))", - " └─TableFullScan_29 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_41 1.00 root funcs:count(Column#18)->Column#17", + "└─TableReader_43 1.00 root data:ExchangeSender_42", + " └─ExchangeSender_42 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_19 1.00 cop[tiflash] funcs:count(1)->Column#18", + " └─HashJoin_24 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]", + " ├─ExchangeReceiver_40(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender_39 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_38 2.00 cop[tiflash] not(isnull(test.d3_t.d3_k))", + " │ └─TableFullScan_37 2.00 cop[tiflash] table:d3_t keep order:false", + " └─HashJoin_25(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]", + " ├─ExchangeReceiver_36(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender_35 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_34 2.00 cop[tiflash] not(isnull(test.d2_t.d2_k))", + " │ └─TableFullScan_33 2.00 cop[tiflash] table:d2_t keep order:false", + " └─HashJoin_26(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", + " ├─ExchangeReceiver_30(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender_29 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_28 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_27 2.00 cop[tiflash] table:d1_t keep order:false", + " └─Selection_32(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))", + " └─TableFullScan_31 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", "Plan": [ - "StreamAgg_11 1.00 root funcs:count(1)->Column#11", - "└─TableReader_35 8.00 root data:ExchangeSender_34", - " └─ExchangeSender_34 8.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_31 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", - " ├─ExchangeReceiver_19(Build) 2.00 cop[tiflash] ", - " │ └─ExchangeSender_18 2.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_17 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_16 2.00 cop[tiflash] table:d1_t keep order:false", - " └─Selection_21(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_20 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_24 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_26 1.00 root data:ExchangeSender_25", + " └─ExchangeSender_25 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_17 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", + " ├─ExchangeReceiver_21(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender_20 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_19 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_18 2.00 cop[tiflash] table:d1_t keep order:false", + " └─Selection_23(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_22 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k", "Plan": [ - "StreamAgg_8 1.00 root funcs:count(1)->Column#11", - "└─TableReader_28 8.00 root data:ExchangeSender_27", - " └─ExchangeSender_27 8.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_24 8.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─ExchangeReceiver_16(Build) 2.00 cop[tiflash] ", - " │ └─ExchangeSender_15 2.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_14 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_13 2.00 cop[tiflash] table:d1_t keep order:false", - " └─TableFullScan_12(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" + "StreamAgg_12 1.00 root funcs:count(1)->Column#11", + "└─TableReader_36 8.00 root data:ExchangeSender_35", + " └─ExchangeSender_35 8.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_32 8.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─ExchangeReceiver_18(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender_17 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_16 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_15 2.00 cop[tiflash] table:d1_t keep order:false", + " └─TableFullScan_14(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k", "Plan": [ - "StreamAgg_8 1.00 root funcs:count(1)->Column#11", - "└─TableReader_28 8.00 root data:ExchangeSender_27", - " └─ExchangeSender_27 8.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_24 8.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─ExchangeReceiver_15(Build) 8.00 cop[tiflash] ", - " │ └─ExchangeSender_14 8.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_13 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " │ └─TableFullScan_12 8.00 cop[tiflash] table:fact_t keep order:false", - " └─TableFullScan_16(Probe) 2.00 cop[tiflash] table:d1_t keep order:false" + "StreamAgg_12 1.00 root funcs:count(1)->Column#11", + "└─TableReader_36 8.00 root data:ExchangeSender_35", + " └─ExchangeSender_35 8.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_32 8.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─ExchangeReceiver_17(Build) 8.00 cop[tiflash] ", + " │ └─ExchangeSender_16 8.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_15 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " │ └─TableFullScan_14 8.00 cop[tiflash] table:fact_t keep order:false", + " └─TableFullScan_18(Probe) 2.00 cop[tiflash] table:d1_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > d1_t.value", "Plan": [ - "StreamAgg_10 1.00 root funcs:count(1)->Column#11", - "└─TableReader_34 8.00 root data:ExchangeSender_33", - " └─ExchangeSender_33 8.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_30 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)], other cond:gt(test.fact_t.col1, test.d1_t.value)", - " ├─ExchangeReceiver_18(Build) 2.00 cop[tiflash] ", - " │ └─ExchangeSender_17 2.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_16 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", - " │ └─TableFullScan_15 2.00 cop[tiflash] table:d1_t keep order:false", - " └─Selection_20(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_19 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_23 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_25 1.00 root data:ExchangeSender_24", + " └─ExchangeSender_24 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_11 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_16 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)], other cond:gt(test.fact_t.col1, test.d1_t.value)", + " ├─ExchangeReceiver_20(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender_19 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_18 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", + " │ └─TableFullScan_17 2.00 cop[tiflash] table:d1_t keep order:false", + " └─Selection_22(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_21 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > 10", "Plan": [ - "StreamAgg_8 1.00 root funcs:count(1)->Column#11", - "└─TableReader_28 8.00 root data:ExchangeSender_27", - " └─ExchangeSender_27 8.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_24 8.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col1, 10)]", - " ├─ExchangeReceiver_16(Build) 2.00 cop[tiflash] ", - " │ └─ExchangeSender_15 2.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_14 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_13 2.00 cop[tiflash] table:d1_t keep order:false", - " └─TableFullScan_12(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" + "StreamAgg_12 1.00 root funcs:count(1)->Column#11", + "└─TableReader_36 8.00 root data:ExchangeSender_35", + " └─ExchangeSender_35 8.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_32 8.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col1, 10)]", + " ├─ExchangeReceiver_18(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender_17 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_16 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_15 2.00 cop[tiflash] table:d1_t keep order:false", + " └─TableFullScan_14(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col2 > 10 and fact_t.col1 > d1_t.value", "Plan": [ - "StreamAgg_8 1.00 root funcs:count(1)->Column#11", - "└─TableReader_28 8.00 root data:ExchangeSender_27", - " └─ExchangeSender_27 8.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_24 8.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col2, 10)], other cond:gt(test.fact_t.col1, test.d1_t.value)", - " ├─ExchangeReceiver_16(Build) 2.00 cop[tiflash] ", - " │ └─ExchangeSender_15 2.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_14 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", - " │ └─TableFullScan_13 2.00 cop[tiflash] table:d1_t keep order:false", - " └─TableFullScan_12(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" + "StreamAgg_12 1.00 root funcs:count(1)->Column#11", + "└─TableReader_36 8.00 root data:ExchangeSender_35", + " └─ExchangeSender_35 8.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_32 8.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col2, 10)], other cond:gt(test.fact_t.col1, test.d1_t.value)", + " ├─ExchangeReceiver_18(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender_17 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_16 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", + " │ └─TableFullScan_15 2.00 cop[tiflash] table:d1_t keep order:false", + " └─TableFullScan_14(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k and d1_t.value > 10", "Plan": [ - "StreamAgg_8 1.00 root funcs:count(1)->Column#11", - "└─TableReader_28 8.00 root data:ExchangeSender_27", - " └─ExchangeSender_27 8.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_24 8.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10)", - " ├─ExchangeReceiver_15(Build) 8.00 cop[tiflash] ", - " │ └─ExchangeSender_14 8.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_13 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " │ └─TableFullScan_12 8.00 cop[tiflash] table:fact_t keep order:false", - " └─TableFullScan_16(Probe) 2.00 cop[tiflash] table:d1_t keep order:false" + "StreamAgg_12 1.00 root funcs:count(1)->Column#11", + "└─TableReader_36 8.00 root data:ExchangeSender_35", + " └─ExchangeSender_35 8.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_32 8.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10)", + " ├─ExchangeReceiver_17(Build) 8.00 cop[tiflash] ", + " │ └─ExchangeSender_16 8.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_15 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " │ └─TableFullScan_14 8.00 cop[tiflash] table:fact_t keep order:false", + " └─TableFullScan_18(Probe) 2.00 cop[tiflash] table:d1_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k and d1_t.value > 10 and fact_t.col1 > d1_t.value", "Plan": [ - "StreamAgg_8 1.00 root funcs:count(1)->Column#11", - "└─TableReader_28 8.00 root data:ExchangeSender_27", - " └─ExchangeSender_27 8.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_24 8.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10), other cond:gt(test.fact_t.col1, test.d1_t.value)", - " ├─ExchangeReceiver_15(Build) 8.00 cop[tiflash] ", - " │ └─ExchangeSender_14 8.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_13 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", - " │ └─TableFullScan_12 8.00 cop[tiflash] table:fact_t keep order:false", - " └─TableFullScan_16(Probe) 2.00 cop[tiflash] table:d1_t keep order:false" + "StreamAgg_12 1.00 root funcs:count(1)->Column#11", + "└─TableReader_36 8.00 root data:ExchangeSender_35", + " └─ExchangeSender_35 8.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_32 8.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10), other cond:gt(test.fact_t.col1, test.d1_t.value)", + " ├─ExchangeReceiver_17(Build) 8.00 cop[tiflash] ", + " │ └─ExchangeSender_16 8.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_15 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", + " │ └─TableFullScan_14 8.00 cop[tiflash] table:fact_t keep order:false", + " └─TableFullScan_18(Probe) 2.00 cop[tiflash] table:d1_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t where exists (select 1 from d1_t where d1_k = fact_t.d1_k)", "Plan": [ - "StreamAgg_11 1.00 root funcs:count(1)->Column#12", - "└─TableReader_31 6.40 root data:ExchangeSender_30", - " └─ExchangeSender_30 6.40 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_28 6.40 cop[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─ExchangeReceiver_19(Build) 2.00 cop[tiflash] ", - " │ └─ExchangeSender_18 2.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_17 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_16 2.00 cop[tiflash] table:d1_t keep order:false", - " └─Selection_15(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_14 8.00 cop[tiflash] table:fact_t keep order:false" + "StreamAgg_15 1.00 root funcs:count(1)->Column#12", + "└─TableReader_39 6.40 root data:ExchangeSender_38", + " └─ExchangeSender_38 6.40 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_36 6.40 cop[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─ExchangeReceiver_22(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender_21 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_20 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_19 2.00 cop[tiflash] table:d1_t keep order:false", + " └─Selection_18(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_17 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t where exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)", "Plan": [ - "StreamAgg_11 1.00 root funcs:count(1)->Column#12", - "└─TableReader_31 6.40 root data:ExchangeSender_30", - " └─ExchangeSender_30 6.40 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_28 6.40 cop[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", - " ├─ExchangeReceiver_19(Build) 2.00 cop[tiflash] ", - " │ └─ExchangeSender_18 2.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_17 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", - " │ └─TableFullScan_16 2.00 cop[tiflash] table:d1_t keep order:false", - " └─Selection_15(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_14 8.00 cop[tiflash] table:fact_t keep order:false" + "StreamAgg_15 1.00 root funcs:count(1)->Column#12", + "└─TableReader_39 6.40 root data:ExchangeSender_38", + " └─ExchangeSender_38 6.40 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_36 6.40 cop[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", + " ├─ExchangeReceiver_22(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender_21 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_20 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", + " │ └─TableFullScan_19 2.00 cop[tiflash] table:d1_t keep order:false", + " └─Selection_18(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_17 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k)", "Plan": [ - "StreamAgg_11 1.00 root funcs:count(1)->Column#12", - "└─TableReader_27 6.40 root data:ExchangeSender_26", - " └─ExchangeSender_26 6.40 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_24 6.40 cop[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─ExchangeReceiver_17(Build) 2.00 cop[tiflash] ", - " │ └─ExchangeSender_16 2.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─TableFullScan_15 2.00 cop[tiflash] table:d1_t keep order:false", - " └─TableFullScan_14(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" + "StreamAgg_15 1.00 root funcs:count(1)->Column#12", + "└─TableReader_35 6.40 root data:ExchangeSender_34", + " └─ExchangeSender_34 6.40 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_32 6.40 cop[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─ExchangeReceiver_20(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender_19 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─TableFullScan_18 2.00 cop[tiflash] table:d1_t keep order:false", + " └─TableFullScan_17(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)", "Plan": [ - "StreamAgg_11 1.00 root funcs:count(1)->Column#12", - "└─TableReader_27 6.40 root data:ExchangeSender_26", - " └─ExchangeSender_26 6.40 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_24 6.40 cop[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", - " ├─ExchangeReceiver_17(Build) 2.00 cop[tiflash] ", - " │ └─ExchangeSender_16 2.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─TableFullScan_15 2.00 cop[tiflash] table:d1_t keep order:false", - " └─TableFullScan_14(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" + "StreamAgg_15 1.00 root funcs:count(1)->Column#12", + "└─TableReader_35 6.40 root data:ExchangeSender_34", + " └─ExchangeSender_34 6.40 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_32 6.40 cop[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", + " ├─ExchangeReceiver_20(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender_19 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─TableFullScan_18 2.00 cop[tiflash] table:d1_t keep order:false", + " └─TableFullScan_17(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" ] } ] @@ -263,270 +267,285 @@ { "SQL": "explain select count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg_10 1.00 root funcs:count(1)->Column#11", - "└─TableReader_25 32.00 root data:ExchangeSender_24", - " └─ExchangeSender_24 32.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_13 32.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", - " ├─ExchangeReceiver_19(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_18 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " │ └─Selection_17 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_16 4.00 cop[tiflash] table:d1_t keep order:false", - " └─ExchangeReceiver_23(Probe) 16.00 cop[tiflash] ", - " └─ExchangeSender_22 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " └─Selection_21 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_20 16.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_26 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_28 1.00 root data:ExchangeSender_27", + " └─ExchangeSender_27 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_17 32.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", + " ├─ExchangeReceiver_21(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_20 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─Selection_19 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_18 4.00 cop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver_25(Probe) 16.00 cop[tiflash] ", + " └─ExchangeSender_24 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─Selection_23 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_22 16.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t, d1_t, d2_t, d3_t where fact_t.d1_k = d1_t.d1_k and fact_t.d2_k = d2_t.d2_k and fact_t.d3_k = d3_t.d3_k", "Plan": [ - "HashAgg_17 1.00 root funcs:count(1)->Column#17", - "└─TableReader_46 128.00 root data:ExchangeSender_45", - " └─ExchangeSender_45 128.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_20 128.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]", - " ├─ExchangeReceiver_44(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_43 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d3_t.d3_k", - " │ └─Selection_42 4.00 cop[tiflash] not(isnull(test.d3_t.d3_k))", - " │ └─TableFullScan_41 4.00 cop[tiflash] table:d3_t keep order:false", - " └─ExchangeReceiver_40(Probe) 64.00 cop[tiflash] ", - " └─ExchangeSender_39 64.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d3_k", - " └─HashJoin_23 64.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]", - " ├─ExchangeReceiver_38(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_37 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d2_t.d2_k", - " │ └─Selection_36 4.00 cop[tiflash] not(isnull(test.d2_t.d2_k))", - " │ └─TableFullScan_35 4.00 cop[tiflash] table:d2_t keep order:false", - " └─ExchangeReceiver_34(Probe) 32.00 cop[tiflash] ", - " └─ExchangeSender_33 32.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d2_k", - " └─HashJoin_24 32.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", - " ├─ExchangeReceiver_28(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_27 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " │ └─Selection_26 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_25 4.00 cop[tiflash] table:d1_t keep order:false", - " └─ExchangeReceiver_32(Probe) 16.00 cop[tiflash] ", - " └─ExchangeSender_31 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " └─Selection_30 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))", - " └─TableFullScan_29 16.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_47 1.00 root funcs:count(Column#18)->Column#17", + "└─TableReader_49 1.00 root data:ExchangeSender_48", + " └─ExchangeSender_48 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_19 1.00 cop[tiflash] funcs:count(1)->Column#18", + " └─HashJoin_24 128.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]", + " ├─ExchangeReceiver_46(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_45 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d3_t.d3_k", + " │ └─Selection_44 4.00 cop[tiflash] not(isnull(test.d3_t.d3_k))", + " │ └─TableFullScan_43 4.00 cop[tiflash] table:d3_t keep order:false", + " └─ExchangeReceiver_42(Probe) 64.00 cop[tiflash] ", + " └─ExchangeSender_41 64.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d3_k", + " └─HashJoin_25 64.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]", + " ├─ExchangeReceiver_40(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_39 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d2_t.d2_k", + " │ └─Selection_38 4.00 cop[tiflash] not(isnull(test.d2_t.d2_k))", + " │ └─TableFullScan_37 4.00 cop[tiflash] table:d2_t keep order:false", + " └─ExchangeReceiver_36(Probe) 32.00 cop[tiflash] ", + " └─ExchangeSender_35 32.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d2_k", + " └─HashJoin_26 32.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", + " ├─ExchangeReceiver_30(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_29 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─Selection_28 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_27 4.00 cop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver_34(Probe) 16.00 cop[tiflash] ", + " └─ExchangeSender_33 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─Selection_32 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))", + " └─TableFullScan_31 16.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg_10 1.00 root funcs:count(1)->Column#11", - "└─TableReader_25 32.00 root data:ExchangeSender_24", - " └─ExchangeSender_24 32.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_13 32.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", - " ├─ExchangeReceiver_19(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_18 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " │ └─Selection_17 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_16 4.00 cop[tiflash] table:d1_t keep order:false", - " └─ExchangeReceiver_23(Probe) 16.00 cop[tiflash] ", - " └─ExchangeSender_22 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " └─Selection_21 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_20 16.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_26 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_28 1.00 root data:ExchangeSender_27", + " └─ExchangeSender_27 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_17 32.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", + " ├─ExchangeReceiver_21(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_20 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─Selection_19 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_18 4.00 cop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver_25(Probe) 16.00 cop[tiflash] ", + " └─ExchangeSender_24 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─Selection_23 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_22 16.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t, d1_t, d2_t, d3_t where fact_t.d1_k = d1_t.d1_k and fact_t.d1_k = d2_t.value and fact_t.d1_k = d3_t.value", "Plan": [ - "HashAgg_17 1.00 root funcs:count(1)->Column#17", - "└─TableReader_44 128.00 root data:ExchangeSender_43", - " └─ExchangeSender_43 128.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_20 128.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d3_t.value)]", - " ├─ExchangeReceiver_42(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_41 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d3_t.value", - " │ └─Selection_40 4.00 cop[tiflash] not(isnull(test.d3_t.value))", - " │ └─TableFullScan_39 4.00 cop[tiflash] table:d3_t keep order:false", - " └─HashJoin_23(Probe) 64.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d2_t.value)]", - " ├─ExchangeReceiver_38(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_37 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d2_t.value", - " │ └─Selection_36 4.00 cop[tiflash] not(isnull(test.d2_t.value))", - " │ └─TableFullScan_35 4.00 cop[tiflash] table:d2_t keep order:false", - " └─HashJoin_25(Probe) 32.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", - " ├─ExchangeReceiver_30(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_29 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " │ └─Selection_28 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_27 4.00 cop[tiflash] table:d1_t keep order:false", - " └─ExchangeReceiver_34(Probe) 16.00 cop[tiflash] ", - " └─ExchangeSender_33 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " └─Selection_32 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_31 16.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_45 1.00 root funcs:count(Column#18)->Column#17", + "└─TableReader_47 1.00 root data:ExchangeSender_46", + " └─ExchangeSender_46 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_19 1.00 cop[tiflash] funcs:count(1)->Column#18", + " └─HashJoin_24 128.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d3_t.value)]", + " ├─ExchangeReceiver_44(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_43 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d3_t.value", + " │ └─Selection_42 4.00 cop[tiflash] not(isnull(test.d3_t.value))", + " │ └─TableFullScan_41 4.00 cop[tiflash] table:d3_t keep order:false", + " └─HashJoin_25(Probe) 64.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d2_t.value)]", + " ├─ExchangeReceiver_40(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_39 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d2_t.value", + " │ └─Selection_38 4.00 cop[tiflash] not(isnull(test.d2_t.value))", + " │ └─TableFullScan_37 4.00 cop[tiflash] table:d2_t keep order:false", + " └─HashJoin_27(Probe) 32.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", + " ├─ExchangeReceiver_32(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_31 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─Selection_30 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_29 4.00 cop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver_36(Probe) 16.00 cop[tiflash] ", + " └─ExchangeSender_35 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─Selection_34 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_33 16.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg_7 1.00 root funcs:count(1)->Column#11", - "└─TableReader_20 32.00 root data:ExchangeSender_19", - " └─ExchangeSender_19 32.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_9 32.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─ExchangeReceiver_18(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_17 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " │ └─Selection_16 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_15 4.00 cop[tiflash] table:d1_t keep order:false", - " └─ExchangeReceiver_14(Probe) 16.00 cop[tiflash] ", - " └─ExchangeSender_13 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " └─TableFullScan_12 16.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_21 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_23 1.00 root data:ExchangeSender_22", + " └─ExchangeSender_22 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_9 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_13 32.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─ExchangeReceiver_20(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_19 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─Selection_18 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_17 4.00 cop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver_16(Probe) 16.00 cop[tiflash] ", + " └─ExchangeSender_15 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─TableFullScan_14 16.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg_7 1.00 root funcs:count(1)->Column#11", - "└─TableReader_20 32.00 root data:ExchangeSender_19", - " └─ExchangeSender_19 32.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_9 32.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─ExchangeReceiver_15(Build) 16.00 cop[tiflash] ", - " │ └─ExchangeSender_14 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " │ └─Selection_13 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " │ └─TableFullScan_12 16.00 cop[tiflash] table:fact_t keep order:false", - " └─ExchangeReceiver_18(Probe) 4.00 cop[tiflash] ", - " └─ExchangeSender_17 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " └─TableFullScan_16 4.00 cop[tiflash] table:d1_t keep order:false" + "HashAgg_21 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_23 1.00 root data:ExchangeSender_22", + " └─ExchangeSender_22 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_9 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_13 32.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─ExchangeReceiver_17(Build) 16.00 cop[tiflash] ", + " │ └─ExchangeSender_16 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " │ └─Selection_15 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " │ └─TableFullScan_14 16.00 cop[tiflash] table:fact_t keep order:false", + " └─ExchangeReceiver_20(Probe) 4.00 cop[tiflash] ", + " └─ExchangeSender_19 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " └─TableFullScan_18 4.00 cop[tiflash] table:d1_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > d1_t.value", "Plan": [ - "HashAgg_9 1.00 root funcs:count(1)->Column#11", - "└─TableReader_24 32.00 root data:ExchangeSender_23", - " └─ExchangeSender_23 32.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_12 32.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)], other cond:gt(test.fact_t.col1, test.d1_t.value)", - " ├─ExchangeReceiver_18(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_17 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " │ └─Selection_16 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", - " │ └─TableFullScan_15 4.00 cop[tiflash] table:d1_t keep order:false", - " └─ExchangeReceiver_22(Probe) 16.00 cop[tiflash] ", - " └─ExchangeSender_21 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " └─Selection_20 16.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_19 16.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_25 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_27 1.00 root data:ExchangeSender_26", + " └─ExchangeSender_26 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_11 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_16 32.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)], other cond:gt(test.fact_t.col1, test.d1_t.value)", + " ├─ExchangeReceiver_20(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_19 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─Selection_18 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", + " │ └─TableFullScan_17 4.00 cop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver_24(Probe) 16.00 cop[tiflash] ", + " └─ExchangeSender_23 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─Selection_22 16.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_21 16.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > 10", "Plan": [ - "HashAgg_7 1.00 root funcs:count(1)->Column#11", - "└─TableReader_20 32.00 root data:ExchangeSender_19", - " └─ExchangeSender_19 32.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_9 32.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col1, 10)]", - " ├─ExchangeReceiver_18(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_17 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " │ └─Selection_16 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_15 4.00 cop[tiflash] table:d1_t keep order:false", - " └─ExchangeReceiver_14(Probe) 16.00 cop[tiflash] ", - " └─ExchangeSender_13 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " └─TableFullScan_12 16.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_21 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_23 1.00 root data:ExchangeSender_22", + " └─ExchangeSender_22 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_9 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_13 32.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col1, 10)]", + " ├─ExchangeReceiver_20(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_19 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─Selection_18 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_17 4.00 cop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver_16(Probe) 16.00 cop[tiflash] ", + " └─ExchangeSender_15 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─TableFullScan_14 16.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col2 > 10 and fact_t.col1 > d1_t.value", "Plan": [ - "HashAgg_7 1.00 root funcs:count(1)->Column#11", - "└─TableReader_20 32.00 root data:ExchangeSender_19", - " └─ExchangeSender_19 32.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_9 32.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col2, 10)], other cond:gt(test.fact_t.col1, test.d1_t.value)", - " ├─ExchangeReceiver_18(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_17 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " │ └─Selection_16 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", - " │ └─TableFullScan_15 4.00 cop[tiflash] table:d1_t keep order:false", - " └─ExchangeReceiver_14(Probe) 16.00 cop[tiflash] ", - " └─ExchangeSender_13 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " └─TableFullScan_12 16.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_21 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_23 1.00 root data:ExchangeSender_22", + " └─ExchangeSender_22 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_9 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_13 32.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col2, 10)], other cond:gt(test.fact_t.col1, test.d1_t.value)", + " ├─ExchangeReceiver_20(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_19 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─Selection_18 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", + " │ └─TableFullScan_17 4.00 cop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver_16(Probe) 16.00 cop[tiflash] ", + " └─ExchangeSender_15 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─TableFullScan_14 16.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k and d1_t.value > 10", "Plan": [ - "HashAgg_7 1.00 root funcs:count(1)->Column#11", - "└─TableReader_20 32.00 root data:ExchangeSender_19", - " └─ExchangeSender_19 32.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_9 32.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10)", - " ├─ExchangeReceiver_15(Build) 16.00 cop[tiflash] ", - " │ └─ExchangeSender_14 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " │ └─Selection_13 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " │ └─TableFullScan_12 16.00 cop[tiflash] table:fact_t keep order:false", - " └─ExchangeReceiver_18(Probe) 4.00 cop[tiflash] ", - " └─ExchangeSender_17 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " └─TableFullScan_16 4.00 cop[tiflash] table:d1_t keep order:false" + "HashAgg_21 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_23 1.00 root data:ExchangeSender_22", + " └─ExchangeSender_22 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_9 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_13 32.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10)", + " ├─ExchangeReceiver_17(Build) 16.00 cop[tiflash] ", + " │ └─ExchangeSender_16 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " │ └─Selection_15 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " │ └─TableFullScan_14 16.00 cop[tiflash] table:fact_t keep order:false", + " └─ExchangeReceiver_20(Probe) 4.00 cop[tiflash] ", + " └─ExchangeSender_19 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " └─TableFullScan_18 4.00 cop[tiflash] table:d1_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k and d1_t.value > 10 and fact_t.col1 > d1_t.value", "Plan": [ - "HashAgg_7 1.00 root funcs:count(1)->Column#11", - "└─TableReader_20 32.00 root data:ExchangeSender_19", - " └─ExchangeSender_19 32.00 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_9 32.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10), other cond:gt(test.fact_t.col1, test.d1_t.value)", - " ├─ExchangeReceiver_15(Build) 16.00 cop[tiflash] ", - " │ └─ExchangeSender_14 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " │ └─Selection_13 16.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", - " │ └─TableFullScan_12 16.00 cop[tiflash] table:fact_t keep order:false", - " └─ExchangeReceiver_18(Probe) 4.00 cop[tiflash] ", - " └─ExchangeSender_17 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " └─TableFullScan_16 4.00 cop[tiflash] table:d1_t keep order:false" + "HashAgg_21 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_23 1.00 root data:ExchangeSender_22", + " └─ExchangeSender_22 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_9 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_13 32.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10), other cond:gt(test.fact_t.col1, test.d1_t.value)", + " ├─ExchangeReceiver_17(Build) 16.00 cop[tiflash] ", + " │ └─ExchangeSender_16 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " │ └─Selection_15 16.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", + " │ └─TableFullScan_14 16.00 cop[tiflash] table:fact_t keep order:false", + " └─ExchangeReceiver_20(Probe) 4.00 cop[tiflash] ", + " └─ExchangeSender_19 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " └─TableFullScan_18 4.00 cop[tiflash] table:d1_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t where exists (select 1 from d1_t where d1_k = fact_t.d1_k)", "Plan": [ - "StreamAgg_11 1.00 root funcs:count(1)->Column#12", - "└─TableReader_33 12.80 root data:ExchangeSender_32", - " └─ExchangeSender_32 12.80 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_30 12.80 cop[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─ExchangeReceiver_21(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_20 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " │ └─Selection_19 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_18 4.00 cop[tiflash] table:d1_t keep order:false", - " └─ExchangeReceiver_17(Probe) 16.00 cop[tiflash] ", - " └─ExchangeSender_16 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " └─Selection_15 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_14 16.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_25 1.00 root funcs:count(Column#13)->Column#12", + "└─TableReader_27 1.00 root data:ExchangeSender_26", + " └─ExchangeSender_26 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#13", + " └─HashJoin_16 12.80 cop[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─ExchangeReceiver_24(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_23 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─Selection_22 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_21 4.00 cop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver_20(Probe) 16.00 cop[tiflash] ", + " └─ExchangeSender_19 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─Selection_18 16.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_17 16.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t where exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)", "Plan": [ - "StreamAgg_11 1.00 root funcs:count(1)->Column#12", - "└─TableReader_33 12.80 root data:ExchangeSender_32", - " └─ExchangeSender_32 12.80 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_30 12.80 cop[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", - " ├─ExchangeReceiver_21(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_20 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " │ └─Selection_19 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", - " │ └─TableFullScan_18 4.00 cop[tiflash] table:d1_t keep order:false", - " └─ExchangeReceiver_17(Probe) 16.00 cop[tiflash] ", - " └─ExchangeSender_16 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " └─Selection_15 16.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_14 16.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_25 1.00 root funcs:count(Column#13)->Column#12", + "└─TableReader_27 1.00 root data:ExchangeSender_26", + " └─ExchangeSender_26 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#13", + " └─HashJoin_16 12.80 cop[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", + " ├─ExchangeReceiver_24(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_23 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─Selection_22 4.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", + " │ └─TableFullScan_21 4.00 cop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver_20(Probe) 16.00 cop[tiflash] ", + " └─ExchangeSender_19 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─Selection_18 16.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_17 16.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k)", "Plan": [ - "StreamAgg_11 1.00 root funcs:count(1)->Column#12", - "└─TableReader_29 12.80 root data:ExchangeSender_28", - " └─ExchangeSender_28 12.80 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_26 12.80 cop[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─ExchangeReceiver_19(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_18 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " │ └─TableFullScan_17 4.00 cop[tiflash] table:d1_t keep order:false", - " └─ExchangeReceiver_16(Probe) 16.00 cop[tiflash] ", - " └─ExchangeSender_15 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " └─TableFullScan_14 16.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_23 1.00 root funcs:count(Column#13)->Column#12", + "└─TableReader_25 1.00 root data:ExchangeSender_24", + " └─ExchangeSender_24 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#13", + " └─HashJoin_16 12.80 cop[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─ExchangeReceiver_22(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_21 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─TableFullScan_20 4.00 cop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver_19(Probe) 16.00 cop[tiflash] ", + " └─ExchangeSender_18 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─TableFullScan_17 16.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)", "Plan": [ - "StreamAgg_11 1.00 root funcs:count(1)->Column#12", - "└─TableReader_29 12.80 root data:ExchangeSender_28", - " └─ExchangeSender_28 12.80 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_26 12.80 cop[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", - " ├─ExchangeReceiver_19(Build) 4.00 cop[tiflash] ", - " │ └─ExchangeSender_18 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " │ └─TableFullScan_17 4.00 cop[tiflash] table:d1_t keep order:false", - " └─ExchangeReceiver_16(Probe) 16.00 cop[tiflash] ", - " └─ExchangeSender_15 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " └─TableFullScan_14 16.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_23 1.00 root funcs:count(Column#13)->Column#12", + "└─TableReader_25 1.00 root data:ExchangeSender_24", + " └─ExchangeSender_24 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#13", + " └─HashJoin_16 12.80 cop[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", + " ├─ExchangeReceiver_22(Build) 4.00 cop[tiflash] ", + " │ └─ExchangeSender_21 4.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─TableFullScan_20 4.00 cop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver_19(Probe) 16.00 cop[tiflash] ", + " └─ExchangeSender_18 16.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─TableFullScan_17 16.00 cop[tiflash] table:fact_t keep order:false" ] } ] @@ -537,198 +556,198 @@ { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg_20 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader_21 1.00 root data:HashAgg_8", - " └─HashAgg_8 1.00 cop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin_11 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─Selection_19(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_18 2.00 cop[tiflash] table:d1_t keep order:false, global read", - " └─Selection_17(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_16 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_24 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_25 1.00 root data:HashAgg_10", + " └─HashAgg_10 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_15 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─Selection_23(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_22 2.00 cop[tiflash] table:d1_t keep order:false, global read", + " └─Selection_21(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_20 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t,d2_t,d3_t) */ count(*) from fact_t, d1_t, d2_t, d3_t where fact_t.d1_k = d1_t.d1_k and fact_t.d2_k = d2_t.d2_k and fact_t.d3_k = d3_t.d3_k", "Plan": [ - "HashAgg_40 1.00 root funcs:count(Column#18)->Column#17", - "└─TableReader_41 1.00 root data:HashAgg_12", - " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#18", - " └─HashJoin_15 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]", - " ├─Selection_39(Build) 2.00 cop[tiflash] not(isnull(test.d3_t.d3_k))", - " │ └─TableFullScan_38 2.00 cop[tiflash] table:d3_t keep order:false, global read", - " └─HashJoin_29(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]", - " ├─Selection_25(Build) 2.00 cop[tiflash] not(isnull(test.d2_t.d2_k))", - " │ └─TableFullScan_24 2.00 cop[tiflash] table:d2_t keep order:false, global read", - " └─HashJoin_33(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─Selection_23(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_22 2.00 cop[tiflash] table:d1_t keep order:false, global read", - " └─Selection_37(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))", - " └─TableFullScan_36 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_44 1.00 root funcs:count(Column#18)->Column#17", + "└─TableReader_45 1.00 root data:HashAgg_14", + " └─HashAgg_14 1.00 cop[tiflash] funcs:count(1)->Column#18", + " └─HashJoin_19 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]", + " ├─Selection_43(Build) 2.00 cop[tiflash] not(isnull(test.d3_t.d3_k))", + " │ └─TableFullScan_42 2.00 cop[tiflash] table:d3_t keep order:false, global read", + " └─HashJoin_33(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]", + " ├─Selection_29(Build) 2.00 cop[tiflash] not(isnull(test.d2_t.d2_k))", + " │ └─TableFullScan_28 2.00 cop[tiflash] table:d2_t keep order:false, global read", + " └─HashJoin_37(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─Selection_27(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_26 2.00 cop[tiflash] table:d1_t keep order:false, global read", + " └─Selection_41(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))", + " └─TableFullScan_40 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t), broadcast_join_local(d1_t) */ count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg_15 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader_16 1.00 root data:HashAgg_8", - " └─HashAgg_8 1.00 cop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin_10 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─Selection_14(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_13 2.00 cop[tiflash] table:d1_t keep order:false", - " └─Selection_12(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_11 8.00 cop[tiflash] table:fact_t keep order:false, global read" + "HashAgg_19 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_20 1.00 root data:HashAgg_10", + " └─HashAgg_10 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_14 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─Selection_18(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_17 2.00 cop[tiflash] table:d1_t keep order:false", + " └─Selection_16(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_15 8.00 cop[tiflash] table:fact_t keep order:false, global read" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t,d2_t,d3_t), broadcast_join_local(d2_t) */ count(*) from fact_t, d1_t, d2_t, d3_t where fact_t.d1_k = d1_t.d1_k and fact_t.d2_k = d2_t.d2_k and fact_t.d3_k = d3_t.d3_k", "Plan": [ - "HashAgg_26 1.00 root funcs:count(Column#18)->Column#17", - "└─TableReader_27 1.00 root data:HashAgg_12", - " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#18", - " └─HashJoin_14 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]", - " ├─Selection_25(Build) 2.00 cop[tiflash] not(isnull(test.d3_t.d3_k))", - " │ └─TableFullScan_24 2.00 cop[tiflash] table:d3_t keep order:false, global read", - " └─HashJoin_15(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]", - " ├─Selection_23(Build) 2.00 cop[tiflash] not(isnull(test.d2_t.d2_k))", - " │ └─TableFullScan_22 2.00 cop[tiflash] table:d2_t keep order:false", - " └─HashJoin_16(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─Selection_21(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_20 2.00 cop[tiflash] table:d1_t keep order:false, global read", - " └─Selection_19(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))", - " └─TableFullScan_18 8.00 cop[tiflash] table:fact_t keep order:false, global read" + "HashAgg_30 1.00 root funcs:count(Column#18)->Column#17", + "└─TableReader_31 1.00 root data:HashAgg_14", + " └─HashAgg_14 1.00 cop[tiflash] funcs:count(1)->Column#18", + " └─HashJoin_18 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]", + " ├─Selection_29(Build) 2.00 cop[tiflash] not(isnull(test.d3_t.d3_k))", + " │ └─TableFullScan_28 2.00 cop[tiflash] table:d3_t keep order:false, global read", + " └─HashJoin_19(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]", + " ├─Selection_27(Build) 2.00 cop[tiflash] not(isnull(test.d2_t.d2_k))", + " │ └─TableFullScan_26 2.00 cop[tiflash] table:d2_t keep order:false", + " └─HashJoin_20(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─Selection_25(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_24 2.00 cop[tiflash] table:d1_t keep order:false, global read", + " └─Selection_23(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))", + " └─TableFullScan_22 8.00 cop[tiflash] table:fact_t keep order:false, global read" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg_13 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader_14 1.00 root data:HashAgg_7", - " └─HashAgg_7 1.00 cop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin_9 8.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─Selection_12(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_11 2.00 cop[tiflash] table:d1_t keep order:false, global read", - " └─TableFullScan_10(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_17 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_18 1.00 root data:HashAgg_9", + " └─HashAgg_9 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_13 8.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─Selection_16(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_15 2.00 cop[tiflash] table:d1_t keep order:false, global read", + " └─TableFullScan_14(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg_13 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader_14 1.00 root data:HashAgg_7", - " └─HashAgg_7 1.00 cop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin_9 8.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─TableFullScan_12(Build) 2.00 cop[tiflash] table:d1_t keep order:false", - " └─Selection_11(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_10 8.00 cop[tiflash] table:fact_t keep order:false, global read" + "HashAgg_17 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_18 1.00 root data:HashAgg_9", + " └─HashAgg_9 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_13 8.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─TableFullScan_16(Build) 2.00 cop[tiflash] table:d1_t keep order:false", + " └─Selection_15(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_14 8.00 cop[tiflash] table:fact_t keep order:false, global read" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > d1_t.value", "Plan": [ - "HashAgg_19 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader_20 1.00 root data:HashAgg_7", - " └─HashAgg_7 1.00 cop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin_10 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.fact_t.col1, test.d1_t.value)", - " ├─Selection_18(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", - " │ └─TableFullScan_17 2.00 cop[tiflash] table:d1_t keep order:false, global read", - " └─Selection_16(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_15 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_23 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_24 1.00 root data:HashAgg_9", + " └─HashAgg_9 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_14 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.fact_t.col1, test.d1_t.value)", + " ├─Selection_22(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", + " │ └─TableFullScan_21 2.00 cop[tiflash] table:d1_t keep order:false, global read", + " └─Selection_20(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_19 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > 10", "Plan": [ - "HashAgg_13 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader_14 1.00 root data:HashAgg_7", - " └─HashAgg_7 1.00 cop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin_9 8.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col1, 10)]", - " ├─Selection_12(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_11 2.00 cop[tiflash] table:d1_t keep order:false, global read", - " └─TableFullScan_10(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_17 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_18 1.00 root data:HashAgg_9", + " └─HashAgg_9 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_13 8.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col1, 10)]", + " ├─Selection_16(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_15 2.00 cop[tiflash] table:d1_t keep order:false, global read", + " └─TableFullScan_14(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col2 > 10 and fact_t.col1 > d1_t.value", "Plan": [ - "HashAgg_13 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader_14 1.00 root data:HashAgg_7", - " └─HashAgg_7 1.00 cop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin_9 8.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col2, 10)], other cond:gt(test.fact_t.col1, test.d1_t.value)", - " ├─Selection_12(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", - " │ └─TableFullScan_11 2.00 cop[tiflash] table:d1_t keep order:false, global read", - " └─TableFullScan_10(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_17 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_18 1.00 root data:HashAgg_9", + " └─HashAgg_9 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_13 8.00 cop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col2, 10)], other cond:gt(test.fact_t.col1, test.d1_t.value)", + " ├─Selection_16(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", + " │ └─TableFullScan_15 2.00 cop[tiflash] table:d1_t keep order:false, global read", + " └─TableFullScan_14(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k and d1_t.value > 10", "Plan": [ - "HashAgg_13 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader_14 1.00 root data:HashAgg_7", - " └─HashAgg_7 1.00 cop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin_9 8.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10)", - " ├─TableFullScan_12(Build) 2.00 cop[tiflash] table:d1_t keep order:false", - " └─Selection_11(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_10 8.00 cop[tiflash] table:fact_t keep order:false, global read" + "HashAgg_17 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_18 1.00 root data:HashAgg_9", + " └─HashAgg_9 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_13 8.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10)", + " ├─TableFullScan_16(Build) 2.00 cop[tiflash] table:d1_t keep order:false", + " └─Selection_15(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_14 8.00 cop[tiflash] table:fact_t keep order:false, global read" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k and d1_t.value > 10 and fact_t.col1 > d1_t.value", "Plan": [ - "HashAgg_13 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader_14 1.00 root data:HashAgg_7", - " └─HashAgg_7 1.00 cop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin_9 8.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10), other cond:gt(test.fact_t.col1, test.d1_t.value)", - " ├─Selection_11(Build) 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", - " │ └─TableFullScan_10 8.00 cop[tiflash] table:fact_t keep order:false, global read", - " └─TableFullScan_12(Probe) 2.00 cop[tiflash] table:d1_t keep order:false" + "HashAgg_17 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader_18 1.00 root data:HashAgg_9", + " └─HashAgg_9 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin_13 8.00 cop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10), other cond:gt(test.fact_t.col1, test.d1_t.value)", + " ├─Selection_15(Build) 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", + " │ └─TableFullScan_14 8.00 cop[tiflash] table:fact_t keep order:false, global read", + " └─TableFullScan_16(Probe) 2.00 cop[tiflash] table:d1_t keep order:false" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t where exists (select 1 from d1_t where d1_k = fact_t.d1_k)", "Plan": [ - "HashAgg_10 1.00 root funcs:count(1)->Column#12", - "└─HashJoin_13 6.40 root semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─TableReader_23(Build) 2.00 root data:Selection_22", - " │ └─Selection_22 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan_21 2.00 cop[tiflash] table:d1_t keep order:false", - " └─TableReader_20(Probe) 8.00 root data:Selection_19", - " └─Selection_19 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_18 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_21 1.00 root funcs:count(Column#13)->Column#12", + "└─TableReader_22 1.00 root data:HashAgg_12", + " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#13", + " └─HashJoin_16 6.40 cop[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─Selection_20(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan_19 2.00 cop[tiflash] table:d1_t keep order:false, global read", + " └─Selection_18(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_17 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t where exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)", "Plan": [ - "HashAgg_24 1.00 root funcs:count(Column#13)->Column#12", - "└─TableReader_25 1.00 root data:HashAgg_10", - " └─HashAgg_10 1.00 cop[tiflash] funcs:count(1)->Column#13", - " └─HashJoin_12 6.40 cop[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", - " ├─Selection_17(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", - " │ └─TableFullScan_16 2.00 cop[tiflash] table:d1_t keep order:false, global read", - " └─Selection_15(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", - " └─TableFullScan_14 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_21 1.00 root funcs:count(Column#13)->Column#12", + "└─TableReader_22 1.00 root data:HashAgg_12", + " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#13", + " └─HashJoin_16 6.40 cop[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", + " ├─Selection_20(Build) 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", + " │ └─TableFullScan_19 2.00 cop[tiflash] table:d1_t keep order:false, global read", + " └─Selection_18(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", + " └─TableFullScan_17 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k)", "Plan": [ - "HashAgg_10 1.00 root funcs:count(1)->Column#12", - "└─HashJoin_13 6.40 root anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─TableReader_19(Build) 2.00 root data:TableFullScan_18", - " │ └─TableFullScan_18 2.00 cop[tiflash] table:d1_t keep order:false", - " └─TableReader_17(Probe) 8.00 root data:TableFullScan_16", - " └─TableFullScan_16 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_19 1.00 root funcs:count(Column#13)->Column#12", + "└─TableReader_20 1.00 root data:HashAgg_12", + " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#13", + " └─HashJoin_16 6.40 cop[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─TableFullScan_18(Build) 2.00 cop[tiflash] table:d1_t keep order:false, global read", + " └─TableFullScan_17(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)", "Plan": [ - "HashAgg_20 1.00 root funcs:count(Column#13)->Column#12", - "└─TableReader_21 1.00 root data:HashAgg_10", - " └─HashAgg_10 1.00 cop[tiflash] funcs:count(1)->Column#13", - " └─HashJoin_12 6.40 cop[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", - " ├─TableFullScan_15(Build) 2.00 cop[tiflash] table:d1_t keep order:false, global read", - " └─TableFullScan_14(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" + "HashAgg_19 1.00 root funcs:count(Column#13)->Column#12", + "└─TableReader_20 1.00 root data:HashAgg_12", + " └─HashAgg_12 1.00 cop[tiflash] funcs:count(1)->Column#13", + " └─HashJoin_16 6.40 cop[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", + " ├─TableFullScan_18(Build) 2.00 cop[tiflash] table:d1_t keep order:false, global read", + " └─TableFullScan_17(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" ] } ] @@ -1082,5 +1101,307 @@ ] } ] + }, + { + "Name": "TestPushDownProjectionForTiFlash", + "Cases": [ + { + "SQL": "desc select /*+ hash_agg()*/ count(b) from (select id + 1 as b from t)A", + "Plan": [ + "HashAgg_12 1.00 root funcs:count(Column#7)->Column#5", + "└─TableReader_13 1.00 root data:HashAgg_8", + " └─HashAgg_8 1.00 cop[tiflash] funcs:count(Column#4)->Column#7", + " └─Projection_10 10000.00 cop[tiflash] plus(test.t.id, 1)->Column#4", + " └─TableFullScan_11 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select /*+ hash_agg()*/ count(*) from (select id + 1 as b from t)A", + "Plan": [ + "HashAgg_12 1.00 root funcs:count(Column#6)->Column#5", + "└─TableReader_13 1.00 root data:HashAgg_6", + " └─HashAgg_6 1.00 cop[tiflash] funcs:count(1)->Column#6", + " └─TableFullScan_11 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select /*+ hash_agg()*/ sum(b) from (select id + 1 as b from t)A", + "Plan": [ + "HashAgg_12 1.00 root funcs:sum(Column#7)->Column#5", + "└─TableReader_13 1.00 root data:HashAgg_8", + " └─HashAgg_8 1.00 cop[tiflash] funcs:sum(Column#4)->Column#7", + " └─Projection_10 10000.00 cop[tiflash] plus(test.t.id, 1)->Column#4", + " └─TableFullScan_11 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select /*+ stream_agg()*/ count(b) from (select id + 1 as b from t)A", + "Plan": [ + "StreamAgg_11 1.00 root funcs:count(Column#4)->Column#5", + "└─Projection_15 10000.00 root plus(test.t.id, 1)->Column#4", + " └─TableReader_19 10000.00 root data:TableFullScan_18", + " └─TableFullScan_18 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select /*+ stream_agg()*/ count(*) from (select id + 1 as b from t)A", + "Plan": [ + "StreamAgg_17 1.00 root funcs:count(Column#6)->Column#5", + "└─TableReader_18 1.00 root data:StreamAgg_10", + " └─StreamAgg_10 1.00 cop[tiflash] funcs:count(1)->Column#6", + " └─TableFullScan_16 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select /*+ stream_agg()*/ sum(b) from (select id + 1 as b from t)A", + "Plan": [ + "StreamAgg_11 1.00 root funcs:sum(Column#7)->Column#5", + "└─Projection_26 10000.00 root cast(Column#4, decimal(41,0) BINARY)->Column#7", + " └─Projection_15 10000.00 root plus(test.t.id, 1)->Column#4", + " └─TableReader_19 10000.00 root data:TableFullScan_18", + " └─TableFullScan_18 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select * from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "Plan": [ + "TableReader_19 10000.00 root data:HashJoin_9", + "└─HashJoin_9 10000.00 cop[tiflash] inner join, equal:[eq(Column#4, Column#8)]", + " ├─Projection_13(Build) 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#4", + " │ └─Selection_15 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " │ └─TableFullScan_14 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo, global read", + " └─Projection_16(Probe) 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#8", + " └─Selection_18 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " └─TableFullScan_17 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select * from t join (select id-2 as b from t) A on A.b=t.id", + "Plan": [ + "TableReader_23 10000.00 root data:HashJoin_9", + "└─HashJoin_9 10000.00 cop[tiflash] inner join, equal:[eq(test.t.id, Column#7)]", + " ├─Projection_20(Build) 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#7", + " │ └─Selection_22 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " │ └─TableFullScan_21 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo, global read", + " └─Selection_19(Probe) 9990.00 cop[tiflash] not(isnull(test.t.id))", + " └─TableFullScan_18 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select * from t left join (select id-2 as b from t) A on A.b=t.id", + "Plan": [ + "TableReader_14 10000.00 root data:HashJoin_7", + "└─HashJoin_7 10000.00 cop[tiflash] left outer join, equal:[eq(test.t.id, Column#7)]", + " ├─Projection_11(Build) 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#7", + " │ └─Selection_13 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " │ └─TableFullScan_12 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo, global read", + " └─TableFullScan_10(Probe) 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select * from t right join (select id-2 as b from t) A on A.b=t.id", + "Plan": [ + "TableReader_14 12487.50 root data:HashJoin_7", + "└─HashJoin_7 12487.50 cop[tiflash] right outer join, equal:[eq(test.t.id, Column#7)]", + " ├─Selection_11(Build) 9990.00 cop[tiflash] not(isnull(test.t.id))", + " │ └─TableFullScan_10 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo, global read", + " └─Projection_12(Probe) 10000.00 cop[tiflash] minus(test.t.id, 2)->Column#7", + " └─TableFullScan_13 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select A.b, B.b from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "Plan": [ + "Projection_8 10000.00 root Column#8, Column#4", + "└─TableReader_19 10000.00 root data:HashJoin_9", + " └─HashJoin_9 10000.00 cop[tiflash] inner join, equal:[eq(Column#4, Column#8)]", + " ├─Projection_13(Build) 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#4", + " │ └─Selection_15 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " │ └─TableFullScan_14 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo, global read", + " └─Projection_16(Probe) 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#8", + " └─Selection_18 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " └─TableFullScan_17 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select A.id from t as A where exists (select 1 from t where t.id=A.id)", + "Plan": [ + "TableReader_15 7992.00 root data:HashJoin_9", + "└─HashJoin_9 7992.00 cop[tiflash] semi join, equal:[eq(test.t.id, test.t.id)]", + " ├─Selection_14(Build) 9990.00 cop[tiflash] not(isnull(test.t.id))", + " │ └─TableFullScan_13 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo, global read", + " └─Selection_12(Probe) 9990.00 cop[tiflash] not(isnull(test.t.id))", + " └─TableFullScan_11 10000.00 cop[tiflash] table:A keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select A.id from t as A where not exists (select 1 from t where t.id=A.id)", + "Plan": [ + "TableReader_13 8000.00 root data:HashJoin_9", + "└─HashJoin_9 8000.00 cop[tiflash] anti semi join, equal:[eq(test.t.id, test.t.id)]", + " ├─TableFullScan_12(Build) 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo, global read", + " └─TableFullScan_11(Probe) 10000.00 cop[tiflash] table:A keep order:false, stats:pseudo" + ] + } + ] + }, + { + "Name": "TestPushDownProjectionForMPP", + "Cases": [ + { + "SQL": "desc select /*+ hash_agg()*/ count(b) from (select id + 1 as b from t)A", + "Plan": [ + "HashAgg_12 1.00 root funcs:count(Column#7)->Column#5", + "└─TableReader_14 1.00 root data:ExchangeSender_13", + " └─ExchangeSender_13 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_8 1.00 cop[tiflash] funcs:count(Column#4)->Column#7", + " └─Projection_10 10000.00 cop[tiflash] plus(test.t.id, 1)->Column#4", + " └─TableFullScan_11 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select /*+ hash_agg()*/ count(*) from (select id + 1 as b from t)A", + "Plan": [ + "HashAgg_15 1.00 root funcs:count(Column#7)->Column#5", + "└─TableReader_17 1.00 root data:ExchangeSender_16", + " └─ExchangeSender_16 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_8 1.00 cop[tiflash] funcs:count(1)->Column#7", + " └─TableFullScan_14 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select /*+ hash_agg()*/ sum(b) from (select id + 1 as b from t)A", + "Plan": [ + "HashAgg_12 1.00 root funcs:sum(Column#7)->Column#5", + "└─TableReader_14 1.00 root data:ExchangeSender_13", + " └─ExchangeSender_13 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_8 1.00 cop[tiflash] funcs:sum(Column#4)->Column#7", + " └─Projection_10 10000.00 cop[tiflash] plus(test.t.id, 1)->Column#4", + " └─TableFullScan_11 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select /*+ stream_agg()*/ count(b) from (select id + 1 as b from t)A", + "Plan": [ + "StreamAgg_11 1.00 root funcs:count(Column#4)->Column#5", + "└─Projection_15 10000.00 root plus(test.t.id, 1)->Column#4", + " └─TableReader_19 10000.00 root data:TableFullScan_18", + " └─TableFullScan_18 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select /*+ stream_agg()*/ count(*) from (select id + 1 as b from t)A", + "Plan": [ + "StreamAgg_17 1.00 root funcs:count(Column#6)->Column#5", + "└─TableReader_18 1.00 root data:StreamAgg_10", + " └─StreamAgg_10 1.00 cop[tiflash] funcs:count(1)->Column#6", + " └─TableFullScan_16 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select /*+ stream_agg()*/ sum(b) from (select id + 1 as b from t)A", + "Plan": [ + "StreamAgg_11 1.00 root funcs:sum(Column#7)->Column#5", + "└─Projection_26 10000.00 root cast(Column#4, decimal(41,0) BINARY)->Column#7", + " └─Projection_15 10000.00 root plus(test.t.id, 1)->Column#4", + " └─TableReader_19 10000.00 root data:TableFullScan_18", + " └─TableFullScan_18 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select * from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "Plan": [ + "TableReader_22 10000.00 root data:ExchangeSender_21", + "└─ExchangeSender_21 10000.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_9 10000.00 cop[tiflash] inner join, equal:[eq(Column#4, Column#8)]", + " ├─Projection_12(Build) 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#4", + " │ └─Selection_15 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " │ └─TableFullScan_14 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─Projection_18(Probe) 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#8", + " └─Selection_20 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " └─TableFullScan_19 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select * from t join (select id-2 as b from t) A on A.b=t.id", + "Plan": [ + "TableReader_20 10000.00 root data:ExchangeSender_19", + "└─ExchangeSender_19 10000.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_8 10000.00 cop[tiflash] inner join, equal:[eq(test.t.id, Column#7)]", + " ├─Projection_13(Build) 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#7", + " │ └─Selection_16 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " │ └─TableFullScan_15 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─Selection_12(Probe) 9990.00 cop[tiflash] not(isnull(test.t.id))", + " └─TableFullScan_11 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select * from t left join (select id-2 as b from t) A on A.b=t.id", + "Plan": [ + "TableReader_18 10000.00 root data:ExchangeSender_17", + "└─ExchangeSender_17 10000.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_7 10000.00 cop[tiflash] left outer join, equal:[eq(test.t.id, Column#7)]", + " ├─Projection_11(Build) 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#7", + " │ └─Selection_14 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " │ └─TableFullScan_13 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─TableFullScan_10(Probe) 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select * from t right join (select id-2 as b from t) A on A.b=t.id", + "Plan": [ + "TableReader_17 12487.50 root data:ExchangeSender_16", + "└─ExchangeSender_16 12487.50 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_7 12487.50 cop[tiflash] right outer join, equal:[eq(test.t.id, Column#7)]", + " ├─ExchangeReceiver_13(Build) 9990.00 cop[tiflash] ", + " │ └─ExchangeSender_12 9990.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_11 9990.00 cop[tiflash] not(isnull(test.t.id))", + " │ └─TableFullScan_10 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─Projection_14(Probe) 10000.00 cop[tiflash] minus(test.t.id, 2)->Column#7", + " └─TableFullScan_15 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select A.b, B.b from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "Plan": [ + "Projection_8 10000.00 root Column#8, Column#4", + "└─TableReader_22 10000.00 root data:ExchangeSender_21", + " └─ExchangeSender_21 10000.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_9 10000.00 cop[tiflash] inner join, equal:[eq(Column#4, Column#8)]", + " ├─Projection_12(Build) 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#4", + " │ └─Selection_15 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " │ └─TableFullScan_14 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─Projection_18(Probe) 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#8", + " └─Selection_20 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " └─TableFullScan_19 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select id from t as A where exists (select 1 from t where t.id=A.id)", + "Plan": [ + "TableReader_18 7992.00 root data:ExchangeSender_17", + "└─ExchangeSender_17 7992.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_9 7992.00 cop[tiflash] semi join, equal:[eq(test.t.id, test.t.id)]", + " ├─ExchangeReceiver_16(Build) 9990.00 cop[tiflash] ", + " │ └─ExchangeSender_15 9990.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_14 9990.00 cop[tiflash] not(isnull(test.t.id))", + " │ └─TableFullScan_13 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─Selection_12(Probe) 9990.00 cop[tiflash] not(isnull(test.t.id))", + " └─TableFullScan_11 10000.00 cop[tiflash] table:A keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc select id from t as A where not exists (select 1 from t where t.id=A.id)", + "Plan": [ + "TableReader_16 8000.00 root data:ExchangeSender_15", + "└─ExchangeSender_15 8000.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_9 8000.00 cop[tiflash] anti semi join, equal:[eq(test.t.id, test.t.id)]", + " ├─ExchangeReceiver_14(Build) 10000.00 cop[tiflash] ", + " │ └─ExchangeSender_13 10000.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─TableFullScan_12 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─TableFullScan_11(Probe) 10000.00 cop[tiflash] table:A keep order:false, stats:pseudo" + ] + } + ] } ] From 18d3135b6130f72fdb5971d868dc2e43464a70e4 Mon Sep 17 00:00:00 2001 From: xiongjiwei Date: Thu, 7 Jan 2021 12:39:14 +0800 Subject: [PATCH 16/16] test: stable TestTransactionWithWriteOnlyColumn (#22226) --- ddl/db_test.go | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/ddl/db_test.go b/ddl/db_test.go index 77d3902c7a916..a6f45efe30793 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -327,7 +327,7 @@ func (s *testSerialDBSuite) TestAddExpressionIndexRollback(c *C) { _, checkErr = tk1.Exec("use test_db") d := s.dom.DDL() - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} var currJob *model.Job ctx := mock.NewContext() ctx.Store = s.store @@ -447,7 +447,7 @@ LOOP: func (s *testDBSuite5) TestCancelAddPrimaryKey(c *C) { idxName := "primary" addIdxSQL := "alter table t1 add primary key idx_c2 (c2);" - testCancelAddIndex(c, s.store, s.dom.DDL(), s.lease, idxName, addIdxSQL, "") + testCancelAddIndex(c, s.store, s.dom.DDL(), s.lease, idxName, addIdxSQL, "", s.dom) // Check the column's flag when the "add primary key" failed. tk := testkit.NewTestKit(c, s.store) @@ -463,14 +463,14 @@ func (s *testDBSuite5) TestCancelAddPrimaryKey(c *C) { func (s *testDBSuite3) TestCancelAddIndex(c *C) { idxName := "c3_index " addIdxSQL := "create unique index c3_index on t1 (c3)" - testCancelAddIndex(c, s.store, s.dom.DDL(), s.lease, idxName, addIdxSQL, "") + testCancelAddIndex(c, s.store, s.dom.DDL(), s.lease, idxName, addIdxSQL, "", s.dom) tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test_db") tk.MustExec("drop table t1") } -func testCancelAddIndex(c *C, store kv.Storage, d ddl.DDL, lease time.Duration, idxName, addIdxSQL, sqlModeSQL string) { +func testCancelAddIndex(c *C, store kv.Storage, d ddl.DDL, lease time.Duration, idxName, addIdxSQL, sqlModeSQL string, dom *domain.Domain) { tk := testkit.NewTestKit(c, store) tk.MustExec("use test_db") tk.MustExec("drop table if exists t1") @@ -492,7 +492,7 @@ func testCancelAddIndex(c *C, store kv.Storage, d ddl.DDL, lease time.Duration, } var c3IdxInfo *model.IndexInfo - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: dom} originBatchSize := tk.MustQuery("select @@global.tidb_ddl_reorg_batch_size") // Set batch size to lower try to slow down add-index reorganization, This if for hook to cancel this ddl job. tk.MustExec("set @@global.tidb_ddl_reorg_batch_size = 32") @@ -558,7 +558,7 @@ func (s *testDBSuite4) TestCancelAddIndex1(c *C) { } var checkErr error - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} hook.OnJobRunBeforeExported = func(job *model.Job) { if job.Type == model.ActionAddIndex && job.State == model.JobStateRunning && job.SchemaState == model.StateWriteReorganization && job.SnapshotVer == 0 { jobIDs := []int64{job.ID} @@ -612,7 +612,7 @@ func (s *testDBSuite4) TestCancelDropPrimaryKey(c *C) { idxName := "primary" addIdxSQL := "alter table t add primary key idx_c2 (c2);" dropIdxSQL := "alter table t drop primary key;" - testCancelDropIndex(c, s.store, s.dom.DDL(), idxName, addIdxSQL, dropIdxSQL) + testCancelDropIndex(c, s.store, s.dom.DDL(), idxName, addIdxSQL, dropIdxSQL, s.dom) } // TestCancelDropIndex tests cancel ddl job which type is drop index. @@ -620,11 +620,11 @@ func (s *testDBSuite5) TestCancelDropIndex(c *C) { idxName := "idx_c2" addIdxSQL := "alter table t add index idx_c2 (c2);" dropIdxSQL := "alter table t drop index idx_c2;" - testCancelDropIndex(c, s.store, s.dom.DDL(), idxName, addIdxSQL, dropIdxSQL) + testCancelDropIndex(c, s.store, s.dom.DDL(), idxName, addIdxSQL, dropIdxSQL, s.dom) } // testCancelDropIndex tests cancel ddl job which type is drop index. -func testCancelDropIndex(c *C, store kv.Storage, d ddl.DDL, idxName, addIdxSQL, dropIdxSQL string) { +func testCancelDropIndex(c *C, store kv.Storage, d ddl.DDL, idxName, addIdxSQL, dropIdxSQL string, dom *domain.Domain) { tk := testkit.NewTestKit(c, store) tk.MustExec("use test_db") tk.MustExec("drop table if exists t") @@ -647,7 +647,7 @@ func testCancelDropIndex(c *C, store kv.Storage, d ddl.DDL, idxName, addIdxSQL, {true, model.JobStateRunning, model.StateDeleteReorganization, false}, } var checkErr error - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: dom} var jobID int64 testCase := &testCases[0] hook.OnJobRunBeforeExported = func(job *model.Job) { @@ -722,7 +722,7 @@ func (s *testDBSuite5) TestCancelTruncateTable(c *C) { s.mustExec(tk, c, "create table t(c1 int, c2 int)") defer s.mustExec(tk, c, "drop table t;") var checkErr error - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} hook.OnJobRunBeforeExported = func(job *model.Job) { if job.Type == model.ActionTruncateTable && job.State == model.JobStateNone { jobIDs := []int64{job.ID} @@ -765,7 +765,7 @@ func (s *testDBSuite5) TestParallelDropSchemaAndDropTable(c *C) { s.mustExec(tk, c, "use test_drop_schema_table") s.mustExec(tk, c, "create table t(c1 int, c2 int)") var checkErr error - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} dbInfo := testGetSchemaByName(c, tk.Se, "test_drop_schema_table") done := false var wg sync.WaitGroup @@ -827,7 +827,7 @@ func (s *testDBSuite1) TestCancelRenameIndex(c *C) { } s.mustExec(tk, c, "alter table t add index idx_c2(c2)") var checkErr error - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} hook.OnJobRunBeforeExported = func(job *model.Job) { if job.Type == model.ActionRenameIndex && job.State == model.JobStateNone { jobIDs := []int64{job.ID} @@ -894,7 +894,7 @@ func (s *testDBSuite2) TestCancelDropTableAndSchema(c *C) { {true, model.ActionDropSchema, model.JobStateRunning, model.StateDeleteOnly, false}, } var checkErr error - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} var jobID int64 testCase := &testCases[0] s.mustExec(tk, c, "create database if not exists test_drop_db") @@ -1336,7 +1336,7 @@ func (s *testDBSuite1) TestCancelAddTableAndDropTablePartition(c *C) { {model.ActionAddTablePartition, model.JobStateRunning, model.StateReplicaOnly, true}, } var checkErr error - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} testCase := &testCases[0] var jobID int64 hook.OnJobRunBeforeExported = func(job *model.Job) { @@ -1507,7 +1507,7 @@ func (s *testDBSuite3) TestCancelDropColumn(c *C) { {true, model.JobStateRunning, model.StateDeleteReorganization, false}, } var checkErr error - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} var jobID int64 testCase := &testCases[0] hook.OnJobRunBeforeExported = func(job *model.Job) { @@ -1614,7 +1614,7 @@ func (s *testDBSuite3) TestCancelDropColumns(c *C) { {true, model.JobStateRunning, model.StateDeleteReorganization, false}, } var checkErr error - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} var jobID int64 testCase := &testCases[0] hook.OnJobRunBeforeExported = func(job *model.Job) { @@ -2465,7 +2465,7 @@ func (s *testSerialDBSuite) TestCreateTableWithLike2(c *C) { tbl1 := testGetTableByName(c, s.s, "test_db", "t1") doneCh := make(chan error, 2) - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} var onceChecker sync.Map hook.OnJobRunBeforeExported = func(job *model.Job) { if job.Type != model.ActionAddColumn && job.Type != model.ActionDropColumn && @@ -2745,7 +2745,7 @@ func (s *testSerialDBSuite) TestRepairTable(c *C) { // Repaired tableInfo has been filtered by `domain.InfoSchema()`, so get it in repairInfo. originTableInfo, _ := domainutil.RepairInfo.GetRepairedTableInfoByTableName("test", "origin") - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} var repairErr error hook.OnJobRunBeforeExported = func(job *model.Job) { if job.Type != model.ActionRepairTable { @@ -3832,7 +3832,7 @@ func (s *testDBSuite5) TestModifyColumnRollBack(c *C) { var c2 *table.Column var checkErr error - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} hook.OnJobUpdatedExported = func(job *model.Job) { if checkErr != nil { return @@ -3930,7 +3930,7 @@ func (s *testSerialDBSuite) TestModifyColumnnReorgInfo(c *C) { defer s.dom.DDL().(ddl.DDLForTest).SetHook(originalHook) // Check insert null before job first update. - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} var checkErr error var currJob *model.Job var elements []*meta.Element @@ -4155,7 +4155,7 @@ func testModifyColumnNullToNotNull(c *C, s *testDBSuite, enableChangeColumnType // Check insert null before job first update. times := 0 - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} tk.MustExec("delete from t1") var checkErr error hook.OnJobRunBeforeExported = func(job *model.Job) { @@ -4284,7 +4284,7 @@ func (s *testDBSuite3) TestTransactionWithWriteOnlyColumn(c *C) { originHook := s.dom.DDL().GetHook() defer s.dom.DDL().(ddl.DDLForTest).SetHook(originHook) - hook := &ddl.TestDDLCallback{} + hook := &ddl.TestDDLCallback{Do: s.dom} var checkErr error hook.OnJobRunBeforeExported = func(job *model.Job) { if checkErr != nil {