From 0fc64a9f331f201edc32a4dde4b1650a40fe8833 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Wed, 12 Jun 2019 19:50:13 +0800 Subject: [PATCH 01/29] add prepareVirtualColumns --- planner/core/find_best_task.go | 12 ++++++++ planner/core/logical_plan_builder.go | 46 ++++++++++------------------ planner/core/logical_plans.go | 5 +++ 3 files changed, 34 insertions(+), 29 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 40e20ea43287d..e826adca2d2e5 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -501,6 +501,10 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candid }.Init(ds.ctx) ts.SetSchema(ds.schema.Clone()) cop.tablePlan = ts + + // TODO: + // 1. 检查是否有需要处理的虚拟列 + // 2. 如果有, 加一个root Projection } is.initSchema(ds.id, idx, cop.tablePlan != nil) // Only use expectedCnt when it's smaller than the count we calculated. @@ -840,6 +844,14 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid ts.KeepOrder = true copTask.keepOrder = true } + + // TODO: + // 1. 检查是否需要处理虚拟列 + // 2. 替换table filter内的虚拟列 + // 3. 区分table filter中能够下推和不能下推的conditions + // 4. 如果有不能下推的conditions, 加一个root Selection + // 5. 再加一个root Projection + ts.addPushedDownSelection(copTask, ds.stats.ScaleByExpectCnt(prop.ExpectedCnt)) if prop.TaskTp == property.RootTaskType { task = finishCopTask(ds.ctx, task) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index bc2087d1190b3..62d514710086f 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -43,7 +43,7 @@ import ( "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/types" - driver "github.com/pingcap/tidb/types/parser_driver" + "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util/chunk" ) @@ -2311,17 +2311,9 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName) (L result = us } - // If this table contains any virtual generated columns, we need a - // "Projection" to calculate these columns. - proj, err := b.projectVirtualColumns(ctx, ds, columns) - if err != nil { + if err := b.prepareVirtualColumns(ctx, ds, columns); err != nil { return nil, err } - - if proj != nil { - proj.SetChildren(result) - result = proj - } return result, nil } @@ -2382,25 +2374,23 @@ func (b *PlanBuilder) BuildDataSourceFromView(ctx context.Context, dbName model. return projUponView, nil } -// projectVirtualColumns is only for DataSource. If some table has virtual generated columns, -// we add a projection on the original DataSource, and calculate those columns in the projection -// so that plans above it can reference generated columns by their name. -func (b *PlanBuilder) projectVirtualColumns(ctx context.Context, ds *DataSource, columns []*table.Column) (*LogicalProjection, error) { - hasVirtualGeneratedColumn := false +// prepareVirtualColumns is only for DataSource. +// It prepares virtualColExprs and virtualColSchema for table which has virtual generated columns. +// virtualColExprs and virtualColSchema are used to rewrite virtual columns in pushDownSelAndResolveVirtualCols. +func (b *PlanBuilder) prepareVirtualColumns(ctx context.Context, ds *DataSource, columns []*table.Column) error { + ds.hasVirtualCol = false for _, column := range columns { if column.IsGenerated() && !column.GeneratedStored { - hasVirtualGeneratedColumn = true + ds.hasVirtualCol = true break } } - if !hasVirtualGeneratedColumn { - return nil, nil + if !ds.hasVirtualCol { + return nil } - proj := LogicalProjection{ - Exprs: make([]expression.Expression, 0, len(columns)), - calculateGenCols: true, - }.Init(b.ctx) + ds.virtualColSchema = ds.Schema().Clone() + ds.virtualColExprs = make([]expression.Expression, 0, len(columns)) for i, colExpr := range ds.Schema().Columns { var exprIsGen = false var expr expression.Expression @@ -2409,7 +2399,7 @@ func (b *PlanBuilder) projectVirtualColumns(ctx context.Context, ds *DataSource, var err error expr, _, err = b.rewrite(ctx, columns[i].GeneratedExpr, ds, nil, true) if err != nil { - return nil, err + return err } // Because the expression might return different type from // the generated column, we should wrap a CAST on the result. @@ -2420,7 +2410,7 @@ func (b *PlanBuilder) projectVirtualColumns(ctx context.Context, ds *DataSource, if !exprIsGen { expr = colExpr } - proj.Exprs = append(proj.Exprs, expr) + ds.virtualColExprs = append(ds.virtualColExprs, expr) } // Re-iterate expressions to handle those virtual generated columns that refers to the other generated columns, for @@ -2430,12 +2420,10 @@ func (b *PlanBuilder) projectVirtualColumns(ctx context.Context, ds *DataSource, // column a, column b as (a * 2), column c as ((a * 2) + 1) // A generated column definition can refer to only generated columns occurring earlier in the table definition, so // it's safe to iterate in index-ascending order. - for i, expr := range proj.Exprs { - proj.Exprs[i] = expression.ColumnSubstitute(expr, ds.Schema(), proj.Exprs) + for i, expr := range ds.virtualColExprs { + ds.virtualColExprs[i] = expression.ColumnSubstitute(expr, ds.Schema(), ds.virtualColExprs) } - - proj.SetSchema(ds.Schema().Clone()) - return proj, nil + return nil } // buildApplyWithJoinType builds apply plan with outerPlan and innerPlan, which apply join with particular join type for diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 25094e5d0028b..67352e260bb5b 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -353,6 +353,11 @@ type DataSource struct { // handleCol represents the handle column for the datasource, either the // int primary key column or extra handle column. handleCol *expression.Column + + // fields for virtual generated columns + hasVirtualCol bool + virtualColSchema *expression.Schema + virtualColExprs []expression.Expression } // accessPath indicates the way we access a table: by using single index, or by using multiple indexes, From 4e2e924233017b7ad4318395271a92fa25f792ce Mon Sep 17 00:00:00 2001 From: qw4990 Date: Wed, 12 Jun 2019 21:00:00 +0800 Subject: [PATCH 02/29] add resolveVirtualColumns --- planner/core/exhaust_physical_plans.go | 6 +-- planner/core/find_best_task.go | 73 ++++++++++++++++++++------ 2 files changed, 61 insertions(+), 18 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 5ff411d6e5e7f..944a8fdae929b 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -538,8 +538,8 @@ func (p *LogicalJoin) constructInnerTableScan(ds *DataSource, pk *expression.Col indexPlanFinished: true, } selStats := ts.stats.Scale(selectionFactor) - ts.addPushedDownSelection(copTask, selStats) - t := finishCopTask(ds.ctx, copTask) + t := ds.resolveVirtualColumns(copTask, selStats) + t = finishCopTask(ds.ctx, copTask) reader := t.plan() return p.constructInnerUnionScan(us, reader) } @@ -894,7 +894,7 @@ func (ijHelper *indexJoinBuildHelper) analyzeLookUpFilters(indexInfo *model.Inde } func (ijHelper *indexJoinBuildHelper) updateBestChoice(ranges []*ranger.Range, idxInfo *model.IndexInfo, accesses, - remained []expression.Expression, lastColManager *ColWithCmpFuncManager) { +remained []expression.Expression, lastColManager *ColWithCmpFuncManager) { // We choose the index by the number of used columns of the range, the much the better. // Notice that there may be the cases like `t1.a=t2.a and b > 2 and b < 1`. So ranges can be nil though the conditions are valid. // But obviously when the range is nil, we don't need index join. diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index e826adca2d2e5..f99f78e11193d 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -501,10 +501,6 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candid }.Init(ds.ctx) ts.SetSchema(ds.schema.Clone()) cop.tablePlan = ts - - // TODO: - // 1. 检查是否有需要处理的虚拟列 - // 2. 如果有, 加一个root Projection } is.initSchema(ds.id, idx, cop.tablePlan != nil) // Only use expectedCnt when it's smaller than the count we calculated. @@ -521,7 +517,6 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candid } cop.cst = rowCount * scanFactor - task = cop if candidate.isMatchProp { if prop.Items[0].Desc { is.Desc = true @@ -538,6 +533,8 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candid // so we can just use prop.ExpectedCnt as parameter of addPushedDownSelection. finalStats := ds.stats.ScaleByExpectCnt(prop.ExpectedCnt) is.addPushedDownSelection(cop, ds, path, finalStats) + + task = ds.resolveVirtualColumns(cop, finalStats) if prop.TaskTp == property.RootTaskType { task = finishCopTask(ds.ctx, task) } else if _, ok := task.(*rootTask); ok { @@ -810,7 +807,6 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid tablePlan: ts, indexPlanFinished: true, } - task = copTask // Adjust number of rows we actually need to scan if prop.ExpectedCnt is smaller than the count we calculated. if prop.ExpectedCnt < ds.stats.RowCount { count, ok, corr := ds.crossEstimateRowCount(path, prop.ExpectedCnt, candidate.isMatchProp && prop.Items[0].Desc) @@ -845,14 +841,7 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid copTask.keepOrder = true } - // TODO: - // 1. 检查是否需要处理虚拟列 - // 2. 替换table filter内的虚拟列 - // 3. 区分table filter中能够下推和不能下推的conditions - // 4. 如果有不能下推的conditions, 加一个root Selection - // 5. 再加一个root Projection - - ts.addPushedDownSelection(copTask, ds.stats.ScaleByExpectCnt(prop.ExpectedCnt)) + task = ds.resolveVirtualColumns(copTask, ds.stats.ScaleByExpectCnt(prop.ExpectedCnt)) if prop.TaskTp == property.RootTaskType { task = finishCopTask(ds.ctx, task) } else if _, ok := task.(*rootTask); ok { @@ -861,7 +850,61 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid return task, nil } -func (ts *PhysicalTableScan) addPushedDownSelection(copTask *copTask, stats *property.StatsInfo) { +// TODO: more comments +// 1. check if there are some virtual generated columns to handle; +// 2. remove virtual columns in this table plan; +// 3. substitute table filters in this table plan; +// 4. distinguish table filters that can't be pushed down from these can be; +// 5. if there are some table filters that can't be pushed down, add a Selection upon this DataSource; +// 6. add a Projection upon this DataSource/Selection; +func (ds *DataSource) resolveVirtualColumns(copTask *copTask, stats *property.StatsInfo) (t task) { + // step 1 + t = copTask + ts := copTask.tablePlan.(*PhysicalTableScan) + if !ds.hasVirtualCol { + ds.addPushedDownSelection(copTask, ts, stats) + return + } + + // step 2 + virSchema := ts.Schema().Clone() + for i := 0; i < len(ts.Columns); i++ { + if ts.Columns[i].IsGenerated() && !ts.Columns[i].GeneratedStored { + ts.Columns = append(ts.Columns[i:], ts.Columns[i+1:]...) + ts.schema.Columns = append(ts.schema.Columns[i:], ts.schema.Columns[i+1:]...) + } + } + + // step 3 + for i, expr := range ts.filterCondition { + ts.filterCondition[i] = expression.ColumnSubstitute(expr, ds.virtualColSchema, ds.virtualColExprs) + } + + // step 4: since Cast functions cannot be pushed down, so all filter cannot be pushed down now. + cantBePushed := ts.filterCondition + ts.filterCondition = ts.filterCondition[:0] + + // step 4.5: add a Cop Selection + ds.addPushedDownSelection(copTask, ts, stats) + + // step 5 + if len(cantBePushed) > 0 { + sel := PhysicalSelection{Conditions: cantBePushed}.Init(ds.ctx, stats) + t = sel.attach2Task(copTask) + } + + // step 6 + projExprs := make([]expression.Expression, 0, len(virSchema.Columns)) + for _, c := range virSchema.Columns { + projExprs = append(projExprs, expression.ColumnSubstitute(c, ds.virtualColSchema, ds.virtualColExprs)) + } + proj := PhysicalProjection{Exprs: projExprs}.Init(ds.ctx, stats) + proj.SetSchema(virSchema) + t = proj.attach2Task(t) + return +} + +func (ds *DataSource) addPushedDownSelection(copTask *copTask, ts *PhysicalTableScan, stats *property.StatsInfo) { // Add filter condition to table plan now. if len(ts.filterCondition) > 0 { copTask.cst += copTask.count() * cpuFactor From c0ede5ecf6174e3e3b710c62ef99f9175f154b78 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Wed, 12 Jun 2019 21:21:15 +0800 Subject: [PATCH 03/29] bugfix --- planner/core/find_best_task.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index f99f78e11193d..b4f2e0cf2dd26 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -870,8 +870,8 @@ func (ds *DataSource) resolveVirtualColumns(copTask *copTask, stats *property.St virSchema := ts.Schema().Clone() for i := 0; i < len(ts.Columns); i++ { if ts.Columns[i].IsGenerated() && !ts.Columns[i].GeneratedStored { - ts.Columns = append(ts.Columns[i:], ts.Columns[i+1:]...) - ts.schema.Columns = append(ts.schema.Columns[i:], ts.schema.Columns[i+1:]...) + ts.Columns = append(ts.Columns[:i], ts.Columns[i+1:]...) + ts.schema.Columns = append(ts.schema.Columns[:i], ts.schema.Columns[i+1:]...) } } From d2be0059f3087eea9614c702b100b1bb07ef7fbf Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 13 Jun 2019 14:29:06 +0800 Subject: [PATCH 04/29] fix --- planner/core/find_best_task.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index b4f2e0cf2dd26..7f915673aad0d 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -860,6 +860,9 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid func (ds *DataSource) resolveVirtualColumns(copTask *copTask, stats *property.StatsInfo) (t task) { // step 1 t = copTask + if copTask.tablePlan == nil { + return + } ts := copTask.tablePlan.(*PhysicalTableScan) if !ds.hasVirtualCol { ds.addPushedDownSelection(copTask, ts, stats) @@ -867,13 +870,18 @@ func (ds *DataSource) resolveVirtualColumns(copTask *copTask, stats *property.St } // step 2 - virSchema := ts.Schema().Clone() - for i := 0; i < len(ts.Columns); i++ { - if ts.Columns[i].IsGenerated() && !ts.Columns[i].GeneratedStored { - ts.Columns = append(ts.Columns[:i], ts.Columns[i+1:]...) - ts.schema.Columns = append(ts.schema.Columns[:i], ts.schema.Columns[i+1:]...) + newSchema := ts.Schema().Clone() + newColumns := make([]*model.ColumnInfo, 0, len(ts.Columns)) + for i := 0; i < len(newSchema.Columns); i++ { + if i < len(newColumns) { + if newColumns[i].IsGenerated() && !newColumns[i].GeneratedStored { + newColumns = append(newColumns[:i], newColumns[i+1:]...) + newSchema.Columns = append(newSchema.Columns[:i], newSchema.Columns[i+1:]...) + } } } + ts.schema = newSchema + ts.Columns = newColumns // step 3 for i, expr := range ts.filterCondition { @@ -894,12 +902,12 @@ func (ds *DataSource) resolveVirtualColumns(copTask *copTask, stats *property.St } // step 6 - projExprs := make([]expression.Expression, 0, len(virSchema.Columns)) - for _, c := range virSchema.Columns { + projExprs := make([]expression.Expression, 0, len(ds.Schema().Columns)) + for _, c := range ds.Schema().Columns { projExprs = append(projExprs, expression.ColumnSubstitute(c, ds.virtualColSchema, ds.virtualColExprs)) } proj := PhysicalProjection{Exprs: projExprs}.Init(ds.ctx, stats) - proj.SetSchema(virSchema) + proj.SetSchema(ds.Schema()) t = proj.attach2Task(t) return } From fedff2e9c988863802b473103c60f46acab116a0 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 13 Jun 2019 15:16:24 +0800 Subject: [PATCH 05/29] fixup --- planner/core/find_best_task.go | 68 +++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 7f915673aad0d..c5d179fac23f2 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -852,7 +852,7 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid // TODO: more comments // 1. check if there are some virtual generated columns to handle; -// 2. remove virtual columns in this table plan; +// 2. substitute virtual columns in this table plan; // 3. substitute table filters in this table plan; // 4. distinguish table filters that can't be pushed down from these can be; // 5. if there are some table filters that can't be pushed down, add a Selection upon this DataSource; @@ -869,19 +869,8 @@ func (ds *DataSource) resolveVirtualColumns(copTask *copTask, stats *property.St return } - // step 2 - newSchema := ts.Schema().Clone() - newColumns := make([]*model.ColumnInfo, 0, len(ts.Columns)) - for i := 0; i < len(newSchema.Columns); i++ { - if i < len(newColumns) { - if newColumns[i].IsGenerated() && !newColumns[i].GeneratedStored { - newColumns = append(newColumns[:i], newColumns[i+1:]...) - newSchema.Columns = append(newSchema.Columns[:i], newSchema.Columns[i+1:]...) - } - } - } - ts.schema = newSchema - ts.Columns = newColumns + // step 2: substitute virtual columns + ds.substituteVirtualColumns(ts) // step 3 for i, expr := range ts.filterCondition { @@ -912,6 +901,57 @@ func (ds *DataSource) resolveVirtualColumns(copTask *copTask, stats *property.St return } +// substituteVirtualColumns substitute virtual columns to physical columns. +func (ds *DataSource) substituteVirtualColumns(ts *PhysicalTableScan) { + // clone a new Schema to modify for safety + schema := ts.Schema().Clone() + colInfos := make([]*model.ColumnInfo, 0, len(ts.Columns)) + colInfos = append(colInfos, ts.Columns...) + + // derive physical columns from virtual columns + phyCols := make(map[int64]*expression.Column) // physical columns this ts already has + virCols := make(map[int64]*expression.Column) // virtual columns this ts has + phyColsFromVir := make(map[int64]*expression.Column) // physical columns derived from virtual columns + for i := 0; i < len(schema.Columns); i++ { + if i < len(colInfos) { + if colInfos[i].IsGenerated() && !colInfos[i].GeneratedStored { + virCols[schema.Columns[i].UniqueID] = schema.Columns[i] + expr := expression.ColumnSubstitute(schema.Columns[i], ds.virtualColSchema, ds.virtualColExprs) + for _, phyColFromVir := range expression.ExtractColumns(expr) { + phyColsFromVir[phyColFromVir.UniqueID] = phyColFromVir + } + } else { + phyCols[schema.Columns[i].UniqueID] = schema.Columns[i] + } + } + } + + // remove virtual columns and add new derived physical columns + for i := 0; i < len(schema.Columns); i++ { + if _, ok := virCols[schema.Columns[i].UniqueID]; ok { + schema.Columns = append(schema.Columns[:i], schema.Columns[i+1:]...) + colInfos = append(colInfos[:i], colInfos[i+1:]...) + } + } + for id, col := range phyColsFromVir { + if _, ok := phyCols[id]; !ok { + schema.Columns = append(schema.Columns, col) + var colInfo *model.ColumnInfo + for _, ci := range ds.tableInfo.Cols() { + if ci.Name.O == col.ColName.O { + colInfo = ci + break + } + } + colInfos = append(colInfos, colInfo) + phyCols[id] = col + } + } + + ts.Columns = colInfos + ts.SetSchema(schema) +} + func (ds *DataSource) addPushedDownSelection(copTask *copTask, ts *PhysicalTableScan, stats *property.StatsInfo) { // Add filter condition to table plan now. if len(ts.filterCondition) > 0 { From 94ec9fc594a973d20cb393c3fdd3c03cab61b8b9 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 13 Jun 2019 15:40:33 +0800 Subject: [PATCH 06/29] fixup --- planner/core/exhaust_physical_plans.go | 6 +++--- planner/core/find_best_task.go | 27 +++++++++++++------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 944a8fdae929b..94e6ef832bc90 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -538,7 +538,7 @@ func (p *LogicalJoin) constructInnerTableScan(ds *DataSource, pk *expression.Col indexPlanFinished: true, } selStats := ts.stats.Scale(selectionFactor) - t := ds.resolveVirtualColumns(copTask, selStats) + t := ds.pushDownSelAndResolveVirtualCols(copTask, nil, selStats) t = finishCopTask(ds.ctx, copTask) reader := t.plan() return p.constructInnerUnionScan(us, reader) @@ -611,8 +611,8 @@ func (p *LogicalJoin) constructInnerIndexScan(ds *DataSource, idx *model.IndexIn } selectivity := ds.stats.RowCount / ds.tableStats.RowCount finalStats := ds.stats.ScaleByExpectCnt(selectivity * rowCount) - is.addPushedDownSelection(cop, ds, path, finalStats) - t := finishCopTask(ds.ctx, cop) + t := ds.pushDownSelAndResolveVirtualCols(cop, path, finalStats) + t = finishCopTask(ds.ctx, t) reader := t.plan() return p.constructInnerUnionScan(us, reader) } diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index c5d179fac23f2..aa1ff5568ef24 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -532,9 +532,8 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candid // prop.IsEmpty() would always return true when coming to here, // so we can just use prop.ExpectedCnt as parameter of addPushedDownSelection. finalStats := ds.stats.ScaleByExpectCnt(prop.ExpectedCnt) - is.addPushedDownSelection(cop, ds, path, finalStats) - task = ds.resolveVirtualColumns(cop, finalStats) + task = ds.pushDownSelAndResolveVirtualCols(cop, path, finalStats) if prop.TaskTp == property.RootTaskType { task = finishCopTask(ds.ctx, task) } else if _, ok := task.(*rootTask); ok { @@ -575,8 +574,9 @@ func (is *PhysicalIndexScan) initSchema(id int, idx *model.IndexInfo, isDoubleRe is.SetSchema(expression.NewSchema(indexCols...)) } -func (is *PhysicalIndexScan) addPushedDownSelection(copTask *copTask, p *DataSource, path *accessPath, finalStats *property.StatsInfo) { +func (ds *DataSource) addPushedDownIndexScan(copTask *copTask, path *accessPath, finalStats *property.StatsInfo) { // Add filter condition to table plan now. + is := copTask.indexPlan.(*PhysicalIndexScan) indexConds, tableConds := path.indexFilters, path.tableFilters if indexConds != nil { copTask.cst += copTask.count() * cpuFactor @@ -592,10 +592,8 @@ func (is *PhysicalIndexScan) addPushedDownSelection(copTask *copTask, p *DataSou } if tableConds != nil { copTask.finishIndexPlan() - copTask.cst += copTask.count() * cpuFactor - tableSel := PhysicalSelection{Conditions: tableConds}.Init(is.ctx, finalStats) - tableSel.SetChildren(copTask.tablePlan) - copTask.tablePlan = tableSel + ts := copTask.tablePlan.(*PhysicalTableScan) + ts.filterCondition = append(ts.filterCondition, tableConds...) } } @@ -841,7 +839,7 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid copTask.keepOrder = true } - task = ds.resolveVirtualColumns(copTask, ds.stats.ScaleByExpectCnt(prop.ExpectedCnt)) + task = ds.pushDownSelAndResolveVirtualCols(copTask, path, ds.stats.ScaleByExpectCnt(prop.ExpectedCnt)) if prop.TaskTp == property.RootTaskType { task = finishCopTask(ds.ctx, task) } else if _, ok := task.(*rootTask); ok { @@ -857,15 +855,18 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid // 4. distinguish table filters that can't be pushed down from these can be; // 5. if there are some table filters that can't be pushed down, add a Selection upon this DataSource; // 6. add a Projection upon this DataSource/Selection; -func (ds *DataSource) resolveVirtualColumns(copTask *copTask, stats *property.StatsInfo) (t task) { +func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *accessPath, stats *property.StatsInfo) (t task) { // step 1 t = copTask - if copTask.tablePlan == nil { + if copTask.indexPlan != nil { + ds.addPushedDownIndexScan(copTask, path, stats) + } + if copTask.tablePlan == nil { // don't need to handle virtual columns in IndexScan return } ts := copTask.tablePlan.(*PhysicalTableScan) if !ds.hasVirtualCol { - ds.addPushedDownSelection(copTask, ts, stats) + ds.addPushedDownTableScan(copTask, ts, stats) return } @@ -882,7 +883,7 @@ func (ds *DataSource) resolveVirtualColumns(copTask *copTask, stats *property.St ts.filterCondition = ts.filterCondition[:0] // step 4.5: add a Cop Selection - ds.addPushedDownSelection(copTask, ts, stats) + ds.addPushedDownTableScan(copTask, ts, stats) // step 5 if len(cantBePushed) > 0 { @@ -952,7 +953,7 @@ func (ds *DataSource) substituteVirtualColumns(ts *PhysicalTableScan) { ts.SetSchema(schema) } -func (ds *DataSource) addPushedDownSelection(copTask *copTask, ts *PhysicalTableScan, stats *property.StatsInfo) { +func (ds *DataSource) addPushedDownTableScan(copTask *copTask, ts *PhysicalTableScan, stats *property.StatsInfo) { // Add filter condition to table plan now. if len(ts.filterCondition) > 0 { copTask.cst += copTask.count() * cpuFactor From 8c0191738f547fd53f65240bc6476078b76339fa Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 13 Jun 2019 15:41:23 +0800 Subject: [PATCH 07/29] fmt --- planner/core/exhaust_physical_plans.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 94e6ef832bc90..aec947fb5611e 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -894,7 +894,7 @@ func (ijHelper *indexJoinBuildHelper) analyzeLookUpFilters(indexInfo *model.Inde } func (ijHelper *indexJoinBuildHelper) updateBestChoice(ranges []*ranger.Range, idxInfo *model.IndexInfo, accesses, -remained []expression.Expression, lastColManager *ColWithCmpFuncManager) { + remained []expression.Expression, lastColManager *ColWithCmpFuncManager) { // We choose the index by the number of used columns of the range, the much the better. // Notice that there may be the cases like `t1.a=t2.a and b > 2 and b < 1`. So ranges can be nil though the conditions are valid. // But obviously when the range is nil, we don't need index join. From d7f65f300e050bc8f6d5aec5b64cbb9cc28207e0 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 13 Jun 2019 16:10:25 +0800 Subject: [PATCH 08/29] fix UT --- planner/core/cbo_test.go | 46 ++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index 69f8fbe284ece..eff47989e3174 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -959,32 +959,36 @@ func (s *testAnalyzeSuite) TestIssue9805(c *C) { // Expected output is like: // - // +--------------------------------+----------+------+----------------------------------------------------------------------------------+----------------------------------+ - // | id | count | task | operator info | execution info | - // +--------------------------------+----------+------+----------------------------------------------------------------------------------+----------------------------------+ - // | Projection_9 | 10.00 | root | test.t1.id, test.t2.a | time:203.355µs, loops:1, rows:0 | - // | └─IndexJoin_13 | 10.00 | root | inner join, inner:IndexLookUp_12, outer key:test.t1.a, inner key:test.t2.d | time:199.633µs, loops:1, rows:0 | - // | ├─Projection_16 | 8.00 | root | test.t1.id, test.t1.a, test.t1.b, cast(mod(test.t1.a, 30)) | time:164.587µs, loops:1, rows:0 | - // | │ └─Selection_17 | 8.00 | root | eq(cast(mod(test.t1.a, 30)), 4) | time:157.768µs, loops:1, rows:0 | - // | │ └─TableReader_20 | 10.00 | root | data:Selection_19 | time:154.61µs, loops:1, rows:0 | - // | │ └─Selection_19 | 10.00 | cop | eq(test.t1.b, "t2") | time:28.824µs, loops:1, rows:0 | - // | │ └─TableScan_18 | 10000.00 | cop | table:t1, range:[-inf,+inf], keep order:false, stats:pseudo | time:27.654µs, loops:1, rows:0 | - // | └─IndexLookUp_12 | 10.00 | root | | time:0ns, loops:0, rows:0 | - // | ├─IndexScan_10 | 10.00 | cop | table:t2, index:d, range: decided by [test.t1.a], keep order:false, stats:pseudo | time:0ns, loops:0, rows:0 | - // | └─TableScan_11 | 10.00 | cop | table:t2, keep order:false, stats:pseudo | time:0ns, loops:0, rows:0 | - // +--------------------------------+----------+------+----------------------------------------------------------------------------------+----------------------------------+ - // 10 rows in set (0.00 sec) + //+----------------------------+-------+------+-------------------------------------------------------------------------------------------------+----------------------------------+ + //| id | count | task | operator info | execution info | + //+----------------------------+-------+------+-------------------------------------------------------------------------------------------------+----------------------------------+ + //| Projection_7 | 0.12 | root | test.t1.id, test.t2.a | time:3.844593ms, loops:1, rows:0 | + //| └─IndexJoin_11 | 0.12 | root | inner join, inner:IndexLookUp_10, outer key:test.t1.a, inner key:test.t2.d | time:3.830714ms, loops:1, rows:0 | + //| ├─Projection_20 | 0.10 | root | test.t1.id, test.t1.a, test.t1.b, cast(mod(test.t1.a, 30)) | time:3.735174ms, loops:1, rows:0 | + //| │ └─IndexLookUp_21 | 0.10 | root | | time:3.569946ms, loops:1, rows:0 | + //| │ ├─IndexScan_18 | 0.10 | cop | table:t1, index:d, b, c, range:[4 "t2",4 "t2"], keep order:false, stats:pseudo | time:50.542µs, loops:1, rows:0 | + //| │ └─TableScan_19 | 0.10 | cop | table:t1, keep order:false, stats:pseudo | time:0s, loops:0, rows:0 | + //| └─IndexLookUp_10 | 10.00 | root | | time:0ns, loops:0, rows:0 | + //| ├─IndexScan_8 | 10.00 | cop | table:t2, index:d, range: decided by [eq(test.t2.d, test.t1.a)], keep order:false, stats:pseudo | time:0ns, loops:0, rows:0 | + //| └─TableScan_9 | 10.00 | cop | table:t2, keep order:false, stats:pseudo | time:0ns, loops:0, rows:0 | + //+----------------------------+-------+------+-------------------------------------------------------------------------------------------------+----------------------------------+ + //9 rows in set (0.01 sec) // - c.Assert(rs.Rows(), HasLen, 10) - hasIndexLookUp12 := false + c.Assert(rs.Rows(), HasLen, 9) + hasIndexLookUp10 := false + hasIndexLookUp21 := false for _, row := range rs.Rows() { - c.Assert(row, HasLen, 6) - if strings.HasSuffix(row[0].(string), "IndexLookUp_12") { - hasIndexLookUp12 = true + c.Assert(row, HasLen, 5) + if strings.HasSuffix(row[0].(string), "IndexLookUp_10") { + hasIndexLookUp10 = true c.Assert(row[4], Equals, "time:0ns, loops:0, rows:0") } + if strings.HasSuffix(row[0].(string), "IndexLookUp_21") { + hasIndexLookUp21 = true + } } - c.Assert(hasIndexLookUp12, IsTrue) + c.Assert(hasIndexLookUp10, IsTrue) + c.Assert(hasIndexLookUp21, IsTrue) } func (s *testAnalyzeSuite) TestLimitCrossEstimation(c *C) { From b2050fe9c43070bc4b0f9c0582eb627d393d8829 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 13 Jun 2019 16:28:11 +0800 Subject: [PATCH 09/29] add UT --- planner/core/cbo_test.go | 41 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index eff47989e3174..8ea5e6c7952b4 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -991,6 +991,47 @@ func (s *testAnalyzeSuite) TestIssue9805(c *C) { c.Assert(hasIndexLookUp21, IsTrue) } +func (s *testAnalyzeSuite) TestVirtualGeneratedColumn(c *C) { + defer testleak.AfterTest(c)() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + tk := testkit.NewTestKit(c, store) + defer func() { + dom.Close() + store.Close() + }() + tk.MustExec("use test") + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1(a int, b int as (a+1) virtual, c int as (b+1) virtual, d int as (c+1) virtual, key(b), index idx(c, d))") + tk.MustExec("insert into t1 (a) values (0)") + + tk.MustQuery("explain select b from t1 where b=1").Check(testkit.Rows( + "IndexReader_6 10.00 root index:IndexScan_5", + "└─IndexScan_5 10.00 cop table:t1, index:b, range:[1,1], keep order:false, stats:pseudo")) + tk.MustQuery("select b from t1 where b=1").Check(testkit.Rows("1")) + + tk.MustQuery("explain select b, c, d from t1 where b=1").Check(testkit.Rows( + "Projection_11 10.00 root cast(plus(test.t1.a, 1)), cast(plus(cast(plus(test.t1.a, 1)), 1)), cast(plus(cast(plus(cast(plus(test.t1.a, 1)), 1)), 1))", + "└─IndexLookUp_12 10.00 root ", + " ├─IndexScan_9 10.00 cop table:t1, index:b, range:[1,1], keep order:false, stats:pseudo", + " └─TableScan_10 10.00 cop table:t1, keep order:false, stats:pseudo")) + tk.MustQuery("select b, c, d from t1 where b=1").Check(testkit.Rows("1 2 3")) + + tk.MustQuery("explain select * from t1 where b=1").Check(testkit.Rows( + "Projection_11 10.00 root test.t1.a, cast(plus(test.t1.a, 1)), cast(plus(cast(plus(test.t1.a, 1)), 1)), cast(plus(cast(plus(cast(plus(test.t1.a, 1)), 1)), 1))", + "└─IndexLookUp_12 10.00 root ", + " ├─IndexScan_9 10.00 cop table:t1, index:b, range:[1,1], keep order:false, stats:pseudo", + " └─TableScan_10 10.00 cop table:t1, keep order:false, stats:pseudo")) + tk.MustQuery("select * from t1 where b=1").Check(testkit.Rows("0 1 2 3")) + + tk.MustQuery("explain select c from t1 where c=2 and d=3").Check(testkit.Rows( + "Projection_4 0.10 root test.t1.c", + "└─IndexReader_6 0.10 root index:IndexScan_5", + " └─IndexScan_5 0.10 cop table:t1, index:c, d, range:[2 3,2 3], keep order:false, stats:pseudo", + )) + tk.MustQuery("select c from t1 where c=2 and d=3").Check(testkit.Rows("2")) +} + func (s *testAnalyzeSuite) TestLimitCrossEstimation(c *C) { defer testleak.AfterTest(c)() store, dom, err := newStoreWithBootstrap() From a1abd5c5a23f19c7d5d88c8dde9a741100b85566 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 13 Jun 2019 16:29:37 +0800 Subject: [PATCH 10/29] add comments --- planner/core/find_best_task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index aa1ff5568ef24..b82b4ed1f2665 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -848,7 +848,7 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid return task, nil } -// TODO: more comments +// pushDownSelAndResolveVirtualCols push some filters down to coprocessor and resolve virtual columns // 1. check if there are some virtual generated columns to handle; // 2. substitute virtual columns in this table plan; // 3. substitute table filters in this table plan; From b5f3dbaeb53a6403be3ac0614146588e6ac861a8 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 13 Jun 2019 19:25:16 +0800 Subject: [PATCH 11/29] more test --- planner/core/cbo_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index 8ea5e6c7952b4..9dd011fc25773 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -1030,6 +1030,21 @@ func (s *testAnalyzeSuite) TestVirtualGeneratedColumn(c *C) { " └─IndexScan_5 0.10 cop table:t1, index:c, d, range:[2 3,2 3], keep order:false, stats:pseudo", )) tk.MustQuery("select c from t1 where c=2 and d=3").Check(testkit.Rows("2")) + + tk.MustExec(`CREATE TABLE person ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + address_info JSON, + city_no INT AS (JSON_EXTRACT(address_info, '$.city_no')) VIRTUAL, + KEY(city_no))`) + tk.MustExec(`INSERT INTO person (name, address_info) VALUES ("John", CAST('{"city_no": 1}' AS JSON))`) + tk.MustQuery(`EXPLAIN SELECT name FROM person where city_no=1`).Check(testkit.Rows( + "Projection_4 10.00 root test.person.name", + `└─Projection_11 10.00 root test.person.name, cast(json_extract(test.person.address_info, "$.city_no"))`, + ` └─IndexLookUp_12 10.00 root `, + ` ├─IndexScan_9 10.00 cop table:person, index:city_no, range:[1,1], keep order:false, stats:pseudo`, + ` └─TableScan_10 10.00 cop table:person, keep order:false, stats:pseudo`)) + tk.MustQuery(`SELECT name FROM person where city_no=1`).Check(testkit.Rows("John")) } func (s *testAnalyzeSuite) TestLimitCrossEstimation(c *C) { From d5441661f0b285bd33f8291967bbdf9b9760ba30 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 13 Jun 2019 19:41:40 +0800 Subject: [PATCH 12/29] add more comments --- planner/core/find_best_task.go | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index b82b4ed1f2665..5df78e165057f 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -848,13 +848,11 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid return task, nil } -// pushDownSelAndResolveVirtualCols push some filters down to coprocessor and resolve virtual columns -// 1. check if there are some virtual generated columns to handle; -// 2. substitute virtual columns in this table plan; -// 3. substitute table filters in this table plan; -// 4. distinguish table filters that can't be pushed down from these can be; -// 5. if there are some table filters that can't be pushed down, add a Selection upon this DataSource; -// 6. add a Projection upon this DataSource/Selection; +// pushDownSelAndResolveVirtualCols push some filters down to coprocessor and resolve virtual columns. +// 1. push down filters and check if there is virtual column. +// 2. substitute virtual columns in TableScan's Schema to corresponding physical columns. +// 3. substitute virtual columns in TableScan's filters and push them down. +// 4. add a Projection upon this DataSource. func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *accessPath, stats *property.StatsInfo) (t task) { // step 1 t = copTask @@ -870,28 +868,22 @@ func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *a return } - // step 2: substitute virtual columns + // step 2 ds.substituteVirtualColumns(ts) // step 3 for i, expr := range ts.filterCondition { ts.filterCondition[i] = expression.ColumnSubstitute(expr, ds.virtualColSchema, ds.virtualColExprs) } - - // step 4: since Cast functions cannot be pushed down, so all filter cannot be pushed down now. - cantBePushed := ts.filterCondition + cantBePushed := ts.filterCondition // all Cast cannot be pushed down now ts.filterCondition = ts.filterCondition[:0] - - // step 4.5: add a Cop Selection ds.addPushedDownTableScan(copTask, ts, stats) - - // step 5 - if len(cantBePushed) > 0 { + if len(cantBePushed) > 0 { // for filters cannot be pushed down, we add a root Selection to handle them sel := PhysicalSelection{Conditions: cantBePushed}.Init(ds.ctx, stats) t = sel.attach2Task(copTask) } - // step 6 + // step 4 projExprs := make([]expression.Expression, 0, len(ds.Schema().Columns)) for _, c := range ds.Schema().Columns { projExprs = append(projExprs, expression.ColumnSubstitute(c, ds.virtualColSchema, ds.virtualColExprs)) @@ -902,7 +894,7 @@ func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *a return } -// substituteVirtualColumns substitute virtual columns to physical columns. +// substituteVirtualColumns substitute virtual columns in TableScan's Schema to physical columns. func (ds *DataSource) substituteVirtualColumns(ts *PhysicalTableScan) { // clone a new Schema to modify for safety schema := ts.Schema().Clone() @@ -949,6 +941,7 @@ func (ds *DataSource) substituteVirtualColumns(ts *PhysicalTableScan) { } } + // update TableScan's Schema ts.Columns = colInfos ts.SetSchema(schema) } From 09a754f36337923cea9f4d5e8d7d92800d76d11c Mon Sep 17 00:00:00 2001 From: qw4990 Date: Wed, 26 Jun 2019 11:33:01 +0800 Subject: [PATCH 13/29] address comments --- planner/core/find_best_task.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 5df78e165057f..3eef05e1d622d 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -574,7 +574,7 @@ func (is *PhysicalIndexScan) initSchema(id int, idx *model.IndexInfo, isDoubleRe is.SetSchema(expression.NewSchema(indexCols...)) } -func (ds *DataSource) addPushedDownIndexScan(copTask *copTask, path *accessPath, finalStats *property.StatsInfo) { +func (ds *DataSource) addPushedDownIndexScan(copTask *copTask, path *accessPath) { // Add filter condition to table plan now. is := copTask.indexPlan.(*PhysicalIndexScan) indexConds, tableConds := path.indexFilters, path.tableFilters @@ -857,7 +857,7 @@ func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *a // step 1 t = copTask if copTask.indexPlan != nil { - ds.addPushedDownIndexScan(copTask, path, stats) + ds.addPushedDownIndexScan(copTask, path) } if copTask.tablePlan == nil { // don't need to handle virtual columns in IndexScan return From 5e79b353bc02186fedbc750f601f0ff8773c677e Mon Sep 17 00:00:00 2001 From: qw4990 Date: Wed, 26 Jun 2019 12:28:55 +0800 Subject: [PATCH 14/29] address comments --- planner/core/find_best_task.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 3eef05e1d622d..d5a86beaca567 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -875,8 +875,8 @@ func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *a for i, expr := range ts.filterCondition { ts.filterCondition[i] = expression.ColumnSubstitute(expr, ds.virtualColSchema, ds.virtualColExprs) } - cantBePushed := ts.filterCondition // all Cast cannot be pushed down now - ts.filterCondition = ts.filterCondition[:0] + var cantBePushed []expression.Expression + _, ts.filterCondition, cantBePushed = expression.ExpressionsToPB(ds.ctx.GetSessionVars().StmtCtx, ts.filterCondition, ds.ctx.GetClient()) ds.addPushedDownTableScan(copTask, ts, stats) if len(cantBePushed) > 0 { // for filters cannot be pushed down, we add a root Selection to handle them sel := PhysicalSelection{Conditions: cantBePushed}.Init(ds.ctx, stats) From a4f587118683c200510bc6001e00abfd42913c54 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Wed, 26 Jun 2019 13:03:05 +0800 Subject: [PATCH 15/29] fix CI --- planner/core/cbo_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index 9dd011fc25773..fdcd773bf52dc 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -964,10 +964,10 @@ func (s *testAnalyzeSuite) TestIssue9805(c *C) { //+----------------------------+-------+------+-------------------------------------------------------------------------------------------------+----------------------------------+ //| Projection_7 | 0.12 | root | test.t1.id, test.t2.a | time:3.844593ms, loops:1, rows:0 | //| └─IndexJoin_11 | 0.12 | root | inner join, inner:IndexLookUp_10, outer key:test.t1.a, inner key:test.t2.d | time:3.830714ms, loops:1, rows:0 | - //| ├─Projection_20 | 0.10 | root | test.t1.id, test.t1.a, test.t1.b, cast(mod(test.t1.a, 30)) | time:3.735174ms, loops:1, rows:0 | - //| │ └─IndexLookUp_21 | 0.10 | root | | time:3.569946ms, loops:1, rows:0 | - //| │ ├─IndexScan_18 | 0.10 | cop | table:t1, index:d, b, c, range:[4 "t2",4 "t2"], keep order:false, stats:pseudo | time:50.542µs, loops:1, rows:0 | - //| │ └─TableScan_19 | 0.10 | cop | table:t1, keep order:false, stats:pseudo | time:0s, loops:0, rows:0 | + //| ├─Projection_21 | 0.10 | root | test.t1.id, test.t1.a, test.t1.b, cast(mod(test.t1.a, 30)) | time:3.735174ms, loops:1, rows:0 | + //| │ └─IndexLookUp_22 | 0.10 | root | | time:3.569946ms, loops:1, rows:0 | + //| │ ├─IndexScan_19 | 0.10 | cop | table:t1, index:d, b, c, range:[4 "t2",4 "t2"], keep order:false, stats:pseudo | time:50.542µs, loops:1, rows:0 | + //| │ └─TableScan_20 | 0.10 | cop | table:t1, keep order:false, stats:pseudo | time:0s, loops:0, rows:0 | //| └─IndexLookUp_10 | 10.00 | root | | time:0ns, loops:0, rows:0 | //| ├─IndexScan_8 | 10.00 | cop | table:t2, index:d, range: decided by [eq(test.t2.d, test.t1.a)], keep order:false, stats:pseudo | time:0ns, loops:0, rows:0 | //| └─TableScan_9 | 10.00 | cop | table:t2, keep order:false, stats:pseudo | time:0ns, loops:0, rows:0 | @@ -976,19 +976,19 @@ func (s *testAnalyzeSuite) TestIssue9805(c *C) { // c.Assert(rs.Rows(), HasLen, 9) hasIndexLookUp10 := false - hasIndexLookUp21 := false + hasIndexLookUp22 := false for _, row := range rs.Rows() { c.Assert(row, HasLen, 5) - if strings.HasSuffix(row[0].(string), "IndexLookUp_10") { + if strings.Contains(row[0].(string), "IndexLookUp_10") { hasIndexLookUp10 = true c.Assert(row[4], Equals, "time:0ns, loops:0, rows:0") } - if strings.HasSuffix(row[0].(string), "IndexLookUp_21") { - hasIndexLookUp21 = true + if strings.Contains(row[0].(string), "IndexLookUp_22") { + hasIndexLookUp22 = true } } c.Assert(hasIndexLookUp10, IsTrue) - c.Assert(hasIndexLookUp21, IsTrue) + c.Assert(hasIndexLookUp22, IsTrue) } func (s *testAnalyzeSuite) TestVirtualGeneratedColumn(c *C) { From 74e3e82a4d31fe7dcffcd560a3189b54aaff9fe4 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Tue, 2 Jul 2019 10:51:35 +0800 Subject: [PATCH 16/29] update UT --- planner/core/cbo_test.go | 37 +++++-------------------------------- util/testkit/testkit.go | 10 +++++++++- 2 files changed, 14 insertions(+), 33 deletions(-) diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index fdcd773bf52dc..8e80286b63225 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -1005,31 +1005,10 @@ func (s *testAnalyzeSuite) TestVirtualGeneratedColumn(c *C) { tk.MustExec("create table t1(a int, b int as (a+1) virtual, c int as (b+1) virtual, d int as (c+1) virtual, key(b), index idx(c, d))") tk.MustExec("insert into t1 (a) values (0)") - tk.MustQuery("explain select b from t1 where b=1").Check(testkit.Rows( - "IndexReader_6 10.00 root index:IndexScan_5", - "└─IndexScan_5 10.00 cop table:t1, index:b, range:[1,1], keep order:false, stats:pseudo")) - tk.MustQuery("select b from t1 where b=1").Check(testkit.Rows("1")) - - tk.MustQuery("explain select b, c, d from t1 where b=1").Check(testkit.Rows( - "Projection_11 10.00 root cast(plus(test.t1.a, 1)), cast(plus(cast(plus(test.t1.a, 1)), 1)), cast(plus(cast(plus(cast(plus(test.t1.a, 1)), 1)), 1))", - "└─IndexLookUp_12 10.00 root ", - " ├─IndexScan_9 10.00 cop table:t1, index:b, range:[1,1], keep order:false, stats:pseudo", - " └─TableScan_10 10.00 cop table:t1, keep order:false, stats:pseudo")) - tk.MustQuery("select b, c, d from t1 where b=1").Check(testkit.Rows("1 2 3")) - - tk.MustQuery("explain select * from t1 where b=1").Check(testkit.Rows( - "Projection_11 10.00 root test.t1.a, cast(plus(test.t1.a, 1)), cast(plus(cast(plus(test.t1.a, 1)), 1)), cast(plus(cast(plus(cast(plus(test.t1.a, 1)), 1)), 1))", - "└─IndexLookUp_12 10.00 root ", - " ├─IndexScan_9 10.00 cop table:t1, index:b, range:[1,1], keep order:false, stats:pseudo", - " └─TableScan_10 10.00 cop table:t1, keep order:false, stats:pseudo")) - tk.MustQuery("select * from t1 where b=1").Check(testkit.Rows("0 1 2 3")) - - tk.MustQuery("explain select c from t1 where c=2 and d=3").Check(testkit.Rows( - "Projection_4 0.10 root test.t1.c", - "└─IndexReader_6 0.10 root index:IndexScan_5", - " └─IndexScan_5 0.10 cop table:t1, index:c, d, range:[2 3,2 3], keep order:false, stats:pseudo", - )) - tk.MustQuery("select c from t1 where c=2 and d=3").Check(testkit.Rows("2")) + tk.MustIndexRead("select b from t1 where b=1").Check(testkit.Rows("1")) + tk.MustIndexLookup("select b, c, d from t1 where b=1").Check(testkit.Rows("1 2 3")) + tk.MustIndexLookup("select * from t1 where b=1").Check(testkit.Rows("0 1 2 3")) + tk.MustIndexRead("select c from t1 where c=2 and d=3").Check(testkit.Rows("2")) tk.MustExec(`CREATE TABLE person ( id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, @@ -1038,13 +1017,7 @@ func (s *testAnalyzeSuite) TestVirtualGeneratedColumn(c *C) { city_no INT AS (JSON_EXTRACT(address_info, '$.city_no')) VIRTUAL, KEY(city_no))`) tk.MustExec(`INSERT INTO person (name, address_info) VALUES ("John", CAST('{"city_no": 1}' AS JSON))`) - tk.MustQuery(`EXPLAIN SELECT name FROM person where city_no=1`).Check(testkit.Rows( - "Projection_4 10.00 root test.person.name", - `└─Projection_11 10.00 root test.person.name, cast(json_extract(test.person.address_info, "$.city_no"))`, - ` └─IndexLookUp_12 10.00 root `, - ` ├─IndexScan_9 10.00 cop table:person, index:city_no, range:[1,1], keep order:false, stats:pseudo`, - ` └─TableScan_10 10.00 cop table:person, keep order:false, stats:pseudo`)) - tk.MustQuery(`SELECT name FROM person where city_no=1`).Check(testkit.Rows("John")) + tk.MustIndexLookup(`SELECT name FROM person where city_no=1`).Check(testkit.Rows("John")) } func (s *testAnalyzeSuite) TestLimitCrossEstimation(c *C) { diff --git a/util/testkit/testkit.go b/util/testkit/testkit.go index 947afe70961de..dcf0812c93f23 100644 --- a/util/testkit/testkit.go +++ b/util/testkit/testkit.go @@ -187,10 +187,14 @@ func (tk *TestKit) MustExec(sql string, args ...interface{}) { // MustIndexLookup checks whether the plan for the sql is Point_Get. func (tk *TestKit) MustIndexLookup(sql string, args ...interface{}) *Result { + return tk.MustPlan("IndexLookUp", sql, args...) +} + +func (tk *TestKit) MustPlan(plan, sql string, args ...interface{}) *Result { rs := tk.MustQuery("explain "+sql, args...) hasIndexLookup := false for i := range rs.rows { - if strings.Contains(rs.rows[i][0], "IndexLookUp") { + if strings.Contains(rs.rows[i][0], plan) { hasIndexLookup = true break } @@ -199,6 +203,10 @@ func (tk *TestKit) MustIndexLookup(sql string, args ...interface{}) *Result { return tk.MustQuery(sql, args...) } +func (tk *TestKit) MustIndexRead(sql string, args ...interface{}) *Result { + return tk.MustPlan("IndexReader", sql, args...) +} + // MustPointGet checks whether the plan for the sql is Point_Get. func (tk *TestKit) MustPointGet(sql string, args ...interface{}) *Result { rs := tk.MustQuery("explain "+sql, args...) From 25457c06a97e99a901ffbdf577a7ba72f6946a34 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Tue, 2 Jul 2019 11:01:45 +0800 Subject: [PATCH 17/29] update explain-test --- cmd/explaintest/r/generated_columns.result | 39 ++++++++++++++++++++++ cmd/explaintest/t/generated_columns.test | 21 ++++++++++++ 2 files changed, 60 insertions(+) diff --git a/cmd/explaintest/r/generated_columns.result b/cmd/explaintest/r/generated_columns.result index 0add5d3921876..33865a38aeae5 100644 --- a/cmd/explaintest/r/generated_columns.result +++ b/cmd/explaintest/r/generated_columns.result @@ -136,3 +136,42 @@ Union_13 23263.33 root └─TableReader_34 3323.33 root data:Selection_33 └─Selection_33 3323.33 cop lt(test.sgc3.a, 7) └─TableScan_32 10000.00 cop table:sgc3, partition:max, range:[-inf,+inf], keep order:false, stats:pseudo +DROP TABLE IF EXISTS t1; +CREATE TABLE t1(a INT, b INT AS (a+1) VIRTUAL, c INT AS (b+1) VIRTUAL, d INT AS (c+1) VIRTUAL, KEY(b), INDEX IDX(c, d)); +INSERT INTO t1 (a) VALUES (0); +EXPLAIN SELECT b FROM t1 WHERE b=1; +id count task operator info +IndexReader_6 10.00 root index:IndexScan_5 +└─IndexScan_5 10.00 cop table:t1, index:b, range:[1,1], keep order:false, stats:pseudo +EXPLAIN SELECT b, c, d FROM t1 WHERE b=1; +id count task operator info +Projection_11 10.00 root cast(plus(test.t1.a, 1)), cast(plus(cast(plus(test.t1.a, 1)), 1)), cast(plus(cast(plus(cast(plus(test.t1.a, 1)), 1)), 1)) +└─IndexLookUp_12 10.00 root + ├─IndexScan_9 10.00 cop table:t1, index:b, range:[1,1], keep order:false, stats:pseudo + └─TableScan_10 10.00 cop table:t1, keep order:false, stats:pseudo +EXPLAIN SELECT * FROM t1 WHERE b=1; +id count task operator info +Projection_11 10.00 root test.t1.a, cast(plus(test.t1.a, 1)), cast(plus(cast(plus(test.t1.a, 1)), 1)), cast(plus(cast(plus(cast(plus(test.t1.a, 1)), 1)), 1)) +└─IndexLookUp_12 10.00 root + ├─IndexScan_9 10.00 cop table:t1, index:b, range:[1,1], keep order:false, stats:pseudo + └─TableScan_10 10.00 cop table:t1, keep order:false, stats:pseudo +EXPLAIN SELECT c FROM t1 WHERE c=2 AND d=3; +id count task operator info +Projection_4 0.10 root test.t1.c +└─IndexReader_6 0.10 root index:IndexScan_5 + └─IndexScan_5 0.10 cop table:t1, index:c, d, range:[2 3,2 3], keep order:false, stats:pseudo +DROP TABLE IF EXISTS person; +CREATE TABLE person ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +name VARCHAR(255) NOT NULL, +address_info JSON, +city_no INT AS (JSON_EXTRACT(address_info, '$.city_no')) VIRTUAL, +KEY(city_no)); +INSERT INTO person (name, address_info) VALUES ("John", CAST('{"city_no": 1}' AS JSON)); +EXPLAIN SELECT name FROM person where city_no=1; +id count task operator info +Projection_4 10.00 root test.person.name +└─Projection_11 10.00 root test.person.name, cast(json_extract(test.person.address_info, "$.city_no")) + └─IndexLookUp_12 10.00 root + ├─IndexScan_9 10.00 cop table:person, index:city_no, range:[1,1], keep order:false, stats:pseudo + └─TableScan_10 10.00 cop table:person, keep order:false, stats:pseudo diff --git a/cmd/explaintest/t/generated_columns.test b/cmd/explaintest/t/generated_columns.test index 5d783c1f4daac..61c6cae64a373 100644 --- a/cmd/explaintest/t/generated_columns.test +++ b/cmd/explaintest/t/generated_columns.test @@ -90,3 +90,24 @@ PARTITION max VALUES LESS THAN MAXVALUE); EXPLAIN SELECT * FROM sgc3 WHERE a <= 1; EXPLAIN SELECT * FROM sgc3 WHERE a < 7; +-- Virtual generated columns as indices + +DROP TABLE IF EXISTS t1; +CREATE TABLE t1(a INT, b INT AS (a+1) VIRTUAL, c INT AS (b+1) VIRTUAL, d INT AS (c+1) VIRTUAL, KEY(b), INDEX IDX(c, d)); +INSERT INTO t1 (a) VALUES (0); + +EXPLAIN SELECT b FROM t1 WHERE b=1; +EXPLAIN SELECT b, c, d FROM t1 WHERE b=1; +EXPLAIN SELECT * FROM t1 WHERE b=1; +EXPLAIN SELECT c FROM t1 WHERE c=2 AND d=3; + +DROP TABLE IF EXISTS person; +CREATE TABLE person ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +name VARCHAR(255) NOT NULL, +address_info JSON, +city_no INT AS (JSON_EXTRACT(address_info, '$.city_no')) VIRTUAL, +KEY(city_no)); + +INSERT INTO person (name, address_info) VALUES ("John", CAST('{"city_no": 1}' AS JSON)); +EXPLAIN SELECT name FROM person where city_no=1; From 1d5e27d2341bcb9ad952e2a7c4c2f6c9ca8f25d9 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Tue, 2 Jul 2019 11:06:08 +0800 Subject: [PATCH 18/29] add more comments --- util/testkit/testkit.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/testkit/testkit.go b/util/testkit/testkit.go index dcf0812c93f23..d169c2a9d4867 100644 --- a/util/testkit/testkit.go +++ b/util/testkit/testkit.go @@ -190,6 +190,7 @@ func (tk *TestKit) MustIndexLookup(sql string, args ...interface{}) *Result { return tk.MustPlan("IndexLookUp", sql, args...) } +// MustPlan checks whether the plan for the sql is the specific plan. func (tk *TestKit) MustPlan(plan, sql string, args ...interface{}) *Result { rs := tk.MustQuery("explain "+sql, args...) hasIndexLookup := false @@ -203,6 +204,7 @@ func (tk *TestKit) MustPlan(plan, sql string, args ...interface{}) *Result { return tk.MustQuery(sql, args...) } +// MustIndexRead checks whether the plan for this sql is IndexReader. func (tk *TestKit) MustIndexRead(sql string, args ...interface{}) *Result { return tk.MustPlan("IndexReader", sql, args...) } From 25bcf81407ffea1ea95fc81a276c3923d4240888 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Tue, 2 Jul 2019 13:16:28 +0800 Subject: [PATCH 19/29] rename addPushedDownSelection --- planner/core/find_best_task.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index d5a86beaca567..5195a385644e5 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -574,7 +574,7 @@ func (is *PhysicalIndexScan) initSchema(id int, idx *model.IndexInfo, isDoubleRe is.SetSchema(expression.NewSchema(indexCols...)) } -func (ds *DataSource) addPushedDownIndexScan(copTask *copTask, path *accessPath) { +func (ds *DataSource) addPushedDownSelection(copTask *copTask, path *accessPath) { // Add filter condition to table plan now. is := copTask.indexPlan.(*PhysicalIndexScan) indexConds, tableConds := path.indexFilters, path.tableFilters @@ -857,7 +857,7 @@ func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *a // step 1 t = copTask if copTask.indexPlan != nil { - ds.addPushedDownIndexScan(copTask, path) + ds.addPushedDownSelection(copTask, path) } if copTask.tablePlan == nil { // don't need to handle virtual columns in IndexScan return From a7c39b34e9f4a15a556bbd620bbdb6fdaa31194c Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Mon, 8 Jul 2019 13:24:56 +0800 Subject: [PATCH 20/29] address comments --- planner/core/find_best_task.go | 2 +- planner/core/logical_plan_builder.go | 11 ++++++++--- planner/core/logical_plans.go | 1 - util/testkit/testkit.go | 5 +---- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 5195a385644e5..96e8daa2aeab0 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -863,7 +863,7 @@ func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *a return } ts := copTask.tablePlan.(*PhysicalTableScan) - if !ds.hasVirtualCol { + if len(ds.virtualColExprs) > 0 { ds.addPushedDownTableScan(copTask, ts, stats) return } diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 62d514710086f..9f1682427181e 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -2377,15 +2377,20 @@ func (b *PlanBuilder) BuildDataSourceFromView(ctx context.Context, dbName model. // prepareVirtualColumns is only for DataSource. // It prepares virtualColExprs and virtualColSchema for table which has virtual generated columns. // virtualColExprs and virtualColSchema are used to rewrite virtual columns in pushDownSelAndResolveVirtualCols. +//<<<<<<< HEAD +//func (b *PlanBuilder) prepareVirtualColumns(ctx context.Context, ds *DataSource, columns []*table.Column) error { +// ds.hasVirtualCol = false +//======= func (b *PlanBuilder) prepareVirtualColumns(ctx context.Context, ds *DataSource, columns []*table.Column) error { - ds.hasVirtualCol = false + hasVirtualCol := false + //>>>>>>> address comments for _, column := range columns { if column.IsGenerated() && !column.GeneratedStored { - ds.hasVirtualCol = true + hasVirtualCol = true break } } - if !ds.hasVirtualCol { + if !hasVirtualCol { return nil } diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 67352e260bb5b..79557c1cc15be 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -355,7 +355,6 @@ type DataSource struct { handleCol *expression.Column // fields for virtual generated columns - hasVirtualCol bool virtualColSchema *expression.Schema virtualColExprs []expression.Expression } diff --git a/util/testkit/testkit.go b/util/testkit/testkit.go index d169c2a9d4867..398b645df04df 100644 --- a/util/testkit/testkit.go +++ b/util/testkit/testkit.go @@ -211,10 +211,7 @@ func (tk *TestKit) MustIndexRead(sql string, args ...interface{}) *Result { // MustPointGet checks whether the plan for the sql is Point_Get. func (tk *TestKit) MustPointGet(sql string, args ...interface{}) *Result { - rs := tk.MustQuery("explain "+sql, args...) - tk.c.Assert(len(rs.rows), check.Equals, 1) - tk.c.Assert(strings.Contains(rs.rows[0][0], "Point_Get"), check.IsTrue) - return tk.MustQuery(sql, args...) + return tk.MustPlan("Point_Get", sql, args...) } // MustQuery query the statements and returns result rows. From 9370a88de07cdf053379c59b67bec31a100026b2 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Mon, 8 Jul 2019 13:43:35 +0800 Subject: [PATCH 21/29] address comments --- planner/core/exhaust_physical_plans.go | 32 +++++++++++++++------- planner/core/find_best_task.go | 37 ++++++++++++++++++++------ 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index aec947fb5611e..f01eef02e9fcc 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -435,7 +435,11 @@ func (p *LogicalJoin) getIndexJoinByOuterIdx(prop *property.PhysicalProperty, ou keyOff2IdxOff[i] = 0 } if pkMatched { - innerPlan := p.constructInnerTableScan(ds, pkCol, outerJoinKeys, us) + innerPlan, err := p.constructInnerTableScan(ds, pkCol, outerJoinKeys, us) + if err != nil { + logutil.BgLogger().Error("construct inner table scan error", zap.Error(err)) + return nil + } // Since the primary key means one value corresponding to exact one row, this will always be a no worse one // comparing to other index. return p.constructIndexJoin(prop, outerIdx, innerPlan, nil, keyOff2IdxOff, nil, nil) @@ -464,7 +468,11 @@ func (p *LogicalJoin) getIndexJoinByOuterIdx(prop *property.PhysicalProperty, ou } idxCols, lens := expression.IndexInfo2Cols(ds.schema.Columns, helper.chosenIndexInfo) rangeInfo := helper.buildRangeDecidedByInformation(idxCols, outerJoinKeys) - innerPlan := p.constructInnerIndexScan(ds, helper.chosenIndexInfo, helper.chosenRemained, outerJoinKeys, us, rangeInfo) + innerPlan, err := p.constructInnerIndexScan(ds, helper.chosenIndexInfo, helper.chosenRemained, outerJoinKeys, us, rangeInfo) + if err != nil { + logutil.BgLogger().Error("construct inner index scan error", zap.Error(err)) + return nil + } return p.constructIndexJoin(prop, outerIdx, innerPlan, helper.chosenRanges, keyOff2IdxOff, lens, helper.lastColManager) } return nil @@ -514,7 +522,7 @@ func (ijHelper *indexJoinBuildHelper) buildRangeDecidedByInformation(idxCols []* } // constructInnerTableScan is specially used to construct the inner plan for PhysicalIndexJoin. -func (p *LogicalJoin) constructInnerTableScan(ds *DataSource, pk *expression.Column, outerJoinKeys []*expression.Column, us *LogicalUnionScan) PhysicalPlan { +func (p *LogicalJoin) constructInnerTableScan(ds *DataSource, pk *expression.Column, outerJoinKeys []*expression.Column, us *LogicalUnionScan) (PhysicalPlan, error) { ranges := ranger.FullIntRange(mysql.HasUnsignedFlag(pk.RetType.Flag)) ts := PhysicalTableScan{ Table: ds.tableInfo, @@ -538,10 +546,13 @@ func (p *LogicalJoin) constructInnerTableScan(ds *DataSource, pk *expression.Col indexPlanFinished: true, } selStats := ts.stats.Scale(selectionFactor) - t := ds.pushDownSelAndResolveVirtualCols(copTask, nil, selStats) + t, err := ds.pushDownSelAndResolveVirtualCols(copTask, nil, selStats) + if err != nil { + return nil, err + } t = finishCopTask(ds.ctx, copTask) reader := t.plan() - return p.constructInnerUnionScan(us, reader) + return p.constructInnerUnionScan(us, reader), nil } func (p *LogicalJoin) constructInnerUnionScan(us *LogicalUnionScan, reader PhysicalPlan) PhysicalPlan { @@ -557,7 +568,7 @@ func (p *LogicalJoin) constructInnerUnionScan(us *LogicalUnionScan, reader Physi // constructInnerIndexScan is specially used to construct the inner plan for PhysicalIndexJoin. func (p *LogicalJoin) constructInnerIndexScan(ds *DataSource, idx *model.IndexInfo, filterConds []expression.Expression, - outerJoinKeys []*expression.Column, us *LogicalUnionScan, rangeInfo string) PhysicalPlan { + outerJoinKeys []*expression.Column, us *LogicalUnionScan, rangeInfo string) (PhysicalPlan, error) { is := PhysicalIndexScan{ Table: ds.tableInfo, TableAsName: ds.TableAsName, @@ -611,10 +622,13 @@ func (p *LogicalJoin) constructInnerIndexScan(ds *DataSource, idx *model.IndexIn } selectivity := ds.stats.RowCount / ds.tableStats.RowCount finalStats := ds.stats.ScaleByExpectCnt(selectivity * rowCount) - t := ds.pushDownSelAndResolveVirtualCols(cop, path, finalStats) + t, err := ds.pushDownSelAndResolveVirtualCols(cop, path, finalStats) + if err != nil { + return nil, err + } t = finishCopTask(ds.ctx, t) reader := t.plan() - return p.constructInnerUnionScan(us, reader) + return p.constructInnerUnionScan(us, reader), nil } var symmetricOp = map[string]string{ @@ -894,7 +908,7 @@ func (ijHelper *indexJoinBuildHelper) analyzeLookUpFilters(indexInfo *model.Inde } func (ijHelper *indexJoinBuildHelper) updateBestChoice(ranges []*ranger.Range, idxInfo *model.IndexInfo, accesses, - remained []expression.Expression, lastColManager *ColWithCmpFuncManager) { +remained []expression.Expression, lastColManager *ColWithCmpFuncManager) { // We choose the index by the number of used columns of the range, the much the better. // Notice that there may be the cases like `t1.a=t2.a and b > 2 and b < 1`. So ranges can be nil though the conditions are valid. // But obviously when the range is nil, we don't need index join. diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 96e8daa2aeab0..1f60dce1237ce 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -14,7 +14,10 @@ package core import ( + "fmt" + "github.com/pingcap/errors" "math" + "reflect" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -533,7 +536,10 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candid // so we can just use prop.ExpectedCnt as parameter of addPushedDownSelection. finalStats := ds.stats.ScaleByExpectCnt(prop.ExpectedCnt) - task = ds.pushDownSelAndResolveVirtualCols(cop, path, finalStats) + task, err = ds.pushDownSelAndResolveVirtualCols(cop, path, finalStats) + if err != nil { + return invalidTask, err + } if prop.TaskTp == property.RootTaskType { task = finishCopTask(ds.ctx, task) } else if _, ok := task.(*rootTask); ok { @@ -574,9 +580,12 @@ func (is *PhysicalIndexScan) initSchema(id int, idx *model.IndexInfo, isDoubleRe is.SetSchema(expression.NewSchema(indexCols...)) } -func (ds *DataSource) addPushedDownSelection(copTask *copTask, path *accessPath) { +func (ds *DataSource) addPushedDownSelection(copTask *copTask, path *accessPath) error { // Add filter condition to table plan now. - is := copTask.indexPlan.(*PhysicalIndexScan) + is, ok := copTask.indexPlan.(*PhysicalIndexScan) + if !ok { + return errors.Trace(fmt.Errorf("type assertion fail, expect PhysicalIndexScan, but got %v", reflect.TypeOf(copTask.indexPlan))) + } indexConds, tableConds := path.indexFilters, path.tableFilters if indexConds != nil { copTask.cst += copTask.count() * cpuFactor @@ -592,9 +601,13 @@ func (ds *DataSource) addPushedDownSelection(copTask *copTask, path *accessPath) } if tableConds != nil { copTask.finishIndexPlan() - ts := copTask.tablePlan.(*PhysicalTableScan) + ts, ok := copTask.tablePlan.(*PhysicalTableScan) + if !ok { + return errors.Trace(fmt.Errorf("type assertion fail, expect PhysicalTableScan, but got %v", reflect.TypeOf(copTask.tablePlan))) + } ts.filterCondition = append(ts.filterCondition, tableConds...) } + return nil } func matchIndicesProp(idxCols []*model.IndexColumn, propItems []property.Item) bool { @@ -839,7 +852,10 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid copTask.keepOrder = true } - task = ds.pushDownSelAndResolveVirtualCols(copTask, path, ds.stats.ScaleByExpectCnt(prop.ExpectedCnt)) + task, err = ds.pushDownSelAndResolveVirtualCols(copTask, path, ds.stats.ScaleByExpectCnt(prop.ExpectedCnt)) + if err != nil { + return invalidTask, err + } if prop.TaskTp == property.RootTaskType { task = finishCopTask(ds.ctx, task) } else if _, ok := task.(*rootTask); ok { @@ -853,16 +869,21 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid // 2. substitute virtual columns in TableScan's Schema to corresponding physical columns. // 3. substitute virtual columns in TableScan's filters and push them down. // 4. add a Projection upon this DataSource. -func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *accessPath, stats *property.StatsInfo) (t task) { +func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *accessPath, stats *property.StatsInfo) (t task, err error) { // step 1 t = copTask if copTask.indexPlan != nil { - ds.addPushedDownSelection(copTask, path) + if err := ds.addPushedDownSelection(copTask, path); err != nil { + return invalidTask, err + } } if copTask.tablePlan == nil { // don't need to handle virtual columns in IndexScan return } - ts := copTask.tablePlan.(*PhysicalTableScan) + ts, ok := copTask.tablePlan.(*PhysicalTableScan) + if !ok { + return invalidTask, errors.Trace(fmt.Errorf("type assertion fail, expect PhysicalTableScan, but got %v", reflect.TypeOf(copTask.tablePlan))) + } if len(ds.virtualColExprs) > 0 { ds.addPushedDownTableScan(copTask, ts, stats) return From 69d0aeb1e9b268f397f0c6500bab14e85adf94ad Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Mon, 8 Jul 2019 13:44:53 +0800 Subject: [PATCH 22/29] refmt --- planner/core/exhaust_physical_plans.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index f01eef02e9fcc..50f202b76c1f8 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -908,7 +908,7 @@ func (ijHelper *indexJoinBuildHelper) analyzeLookUpFilters(indexInfo *model.Inde } func (ijHelper *indexJoinBuildHelper) updateBestChoice(ranges []*ranger.Range, idxInfo *model.IndexInfo, accesses, -remained []expression.Expression, lastColManager *ColWithCmpFuncManager) { + remained []expression.Expression, lastColManager *ColWithCmpFuncManager) { // We choose the index by the number of used columns of the range, the much the better. // Notice that there may be the cases like `t1.a=t2.a and b > 2 and b < 1`. So ranges can be nil though the conditions are valid. // But obviously when the range is nil, we don't need index join. From 93b4d4480f658dcbbd4555e7b426c66917c510fe Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Mon, 8 Jul 2019 13:49:21 +0800 Subject: [PATCH 23/29] fixup --- planner/core/find_best_task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 1f60dce1237ce..48ece87e76dd7 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -884,7 +884,7 @@ func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *a if !ok { return invalidTask, errors.Trace(fmt.Errorf("type assertion fail, expect PhysicalTableScan, but got %v", reflect.TypeOf(copTask.tablePlan))) } - if len(ds.virtualColExprs) > 0 { + if len(ds.virtualColExprs) == 0 { ds.addPushedDownTableScan(copTask, ts, stats) return } From baa35a83b02b0ce5c2e6426602ad76ca3591bf2f Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Tue, 9 Jul 2019 14:29:34 +0800 Subject: [PATCH 24/29] update error --- planner/core/find_best_task.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 48ece87e76dd7..1353aceea76be 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -14,7 +14,6 @@ package core import ( - "fmt" "github.com/pingcap/errors" "math" "reflect" @@ -584,7 +583,7 @@ func (ds *DataSource) addPushedDownSelection(copTask *copTask, path *accessPath) // Add filter condition to table plan now. is, ok := copTask.indexPlan.(*PhysicalIndexScan) if !ok { - return errors.Trace(fmt.Errorf("type assertion fail, expect PhysicalIndexScan, but got %v", reflect.TypeOf(copTask.indexPlan))) + return errors.Errorf("type assertion fail, expect PhysicalIndexScan, but got %v", reflect.TypeOf(copTask.indexPlan)) } indexConds, tableConds := path.indexFilters, path.tableFilters if indexConds != nil { @@ -603,7 +602,7 @@ func (ds *DataSource) addPushedDownSelection(copTask *copTask, path *accessPath) copTask.finishIndexPlan() ts, ok := copTask.tablePlan.(*PhysicalTableScan) if !ok { - return errors.Trace(fmt.Errorf("type assertion fail, expect PhysicalTableScan, but got %v", reflect.TypeOf(copTask.tablePlan))) + return errors.Errorf("type assertion fail, expect PhysicalTableScan, but got %v", reflect.TypeOf(copTask.tablePlan)) } ts.filterCondition = append(ts.filterCondition, tableConds...) } @@ -882,7 +881,7 @@ func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *a } ts, ok := copTask.tablePlan.(*PhysicalTableScan) if !ok { - return invalidTask, errors.Trace(fmt.Errorf("type assertion fail, expect PhysicalTableScan, but got %v", reflect.TypeOf(copTask.tablePlan))) + return invalidTask, errors.Errorf("type assertion fail, expect PhysicalTableScan, but got %v", reflect.TypeOf(copTask.tablePlan)) } if len(ds.virtualColExprs) == 0 { ds.addPushedDownTableScan(copTask, ts, stats) From 04e68c0b9fe6f5944dff8351cb312421ae5eff57 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Wed, 24 Jul 2019 11:31:57 +0800 Subject: [PATCH 25/29] fix UT --- planner/core/cbo_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index 8e80286b63225..dce94b3d4e0e9 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -978,7 +978,7 @@ func (s *testAnalyzeSuite) TestIssue9805(c *C) { hasIndexLookUp10 := false hasIndexLookUp22 := false for _, row := range rs.Rows() { - c.Assert(row, HasLen, 5) + c.Assert(row, HasLen, 6) if strings.Contains(row[0].(string), "IndexLookUp_10") { hasIndexLookUp10 = true c.Assert(row[4], Equals, "time:0ns, loops:0, rows:0") From 0ea33a181934924bff359677efeada2c40ad57ed Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Wed, 24 Jul 2019 11:36:16 +0800 Subject: [PATCH 26/29] address comments --- planner/core/find_best_task.go | 32 +++++++++++++++----------------- util/testkit/testkit.go | 2 +- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 1353aceea76be..2471940095e1f 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -14,10 +14,10 @@ package core import ( - "github.com/pingcap/errors" "math" "reflect" + "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" @@ -532,7 +532,7 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candid is.KeepOrder = true } // prop.IsEmpty() would always return true when coming to here, - // so we can just use prop.ExpectedCnt as parameter of addPushedDownSelection. + // so we can just use prop.ExpectedCnt as parameter of addIndexScanSelection. finalStats := ds.stats.ScaleByExpectCnt(prop.ExpectedCnt) task, err = ds.pushDownSelAndResolveVirtualCols(cop, path, finalStats) @@ -579,7 +579,7 @@ func (is *PhysicalIndexScan) initSchema(id int, idx *model.IndexInfo, isDoubleRe is.SetSchema(expression.NewSchema(indexCols...)) } -func (ds *DataSource) addPushedDownSelection(copTask *copTask, path *accessPath) error { +func (ds *DataSource) addIndexScanSelection(copTask *copTask, path *accessPath) error { // Add filter condition to table plan now. is, ok := copTask.indexPlan.(*PhysicalIndexScan) if !ok { @@ -872,7 +872,7 @@ func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *a // step 1 t = copTask if copTask.indexPlan != nil { - if err := ds.addPushedDownSelection(copTask, path); err != nil { + if err := ds.addIndexScanSelection(copTask, path); err != nil { return invalidTask, err } } @@ -884,7 +884,7 @@ func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *a return invalidTask, errors.Errorf("type assertion fail, expect PhysicalTableScan, but got %v", reflect.TypeOf(copTask.tablePlan)) } if len(ds.virtualColExprs) == 0 { - ds.addPushedDownTableScan(copTask, ts, stats) + ds.addTableScanSelection(copTask, ts, stats) return } @@ -897,7 +897,7 @@ func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *a } var cantBePushed []expression.Expression _, ts.filterCondition, cantBePushed = expression.ExpressionsToPB(ds.ctx.GetSessionVars().StmtCtx, ts.filterCondition, ds.ctx.GetClient()) - ds.addPushedDownTableScan(copTask, ts, stats) + ds.addTableScanSelection(copTask, ts, stats) if len(cantBePushed) > 0 { // for filters cannot be pushed down, we add a root Selection to handle them sel := PhysicalSelection{Conditions: cantBePushed}.Init(ds.ctx, stats) t = sel.attach2Task(copTask) @@ -925,17 +925,15 @@ func (ds *DataSource) substituteVirtualColumns(ts *PhysicalTableScan) { phyCols := make(map[int64]*expression.Column) // physical columns this ts already has virCols := make(map[int64]*expression.Column) // virtual columns this ts has phyColsFromVir := make(map[int64]*expression.Column) // physical columns derived from virtual columns - for i := 0; i < len(schema.Columns); i++ { - if i < len(colInfos) { - if colInfos[i].IsGenerated() && !colInfos[i].GeneratedStored { - virCols[schema.Columns[i].UniqueID] = schema.Columns[i] - expr := expression.ColumnSubstitute(schema.Columns[i], ds.virtualColSchema, ds.virtualColExprs) - for _, phyColFromVir := range expression.ExtractColumns(expr) { - phyColsFromVir[phyColFromVir.UniqueID] = phyColFromVir - } - } else { - phyCols[schema.Columns[i].UniqueID] = schema.Columns[i] + for i := 0; i < len(schema.Columns) && i < len(colInfos); i++ { + if colInfos[i].IsGenerated() && !colInfos[i].GeneratedStored { + virCols[schema.Columns[i].UniqueID] = schema.Columns[i] + expr := expression.ColumnSubstitute(schema.Columns[i], ds.virtualColSchema, ds.virtualColExprs) + for _, phyColFromVir := range expression.ExtractColumns(expr) { + phyColsFromVir[phyColFromVir.UniqueID] = phyColFromVir } + } else { + phyCols[schema.Columns[i].UniqueID] = schema.Columns[i] } } @@ -966,7 +964,7 @@ func (ds *DataSource) substituteVirtualColumns(ts *PhysicalTableScan) { ts.SetSchema(schema) } -func (ds *DataSource) addPushedDownTableScan(copTask *copTask, ts *PhysicalTableScan, stats *property.StatsInfo) { +func (ds *DataSource) addTableScanSelection(copTask *copTask, ts *PhysicalTableScan, stats *property.StatsInfo) { // Add filter condition to table plan now. if len(ts.filterCondition) > 0 { copTask.cst += copTask.count() * cpuFactor diff --git a/util/testkit/testkit.go b/util/testkit/testkit.go index 398b645df04df..04af0919b7009 100644 --- a/util/testkit/testkit.go +++ b/util/testkit/testkit.go @@ -185,7 +185,7 @@ func (tk *TestKit) MustExec(sql string, args ...interface{}) { } } -// MustIndexLookup checks whether the plan for the sql is Point_Get. +// MustIndexLookup checks whether the plan for the sql is IndexLookUp. func (tk *TestKit) MustIndexLookup(sql string, args ...interface{}) *Result { return tk.MustPlan("IndexLookUp", sql, args...) } From 044b74187a06670134f11162c4c09b5175abe994 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Wed, 24 Jul 2019 15:15:41 +0800 Subject: [PATCH 27/29] remove reflect --- planner/core/find_best_task.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 2471940095e1f..cf63c0c024451 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -14,9 +14,6 @@ package core import ( - "math" - "reflect" - "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -30,6 +27,7 @@ import ( "github.com/pingcap/tidb/util/ranger" "github.com/pingcap/tidb/util/set" "golang.org/x/tools/container/intsets" + "math" ) const ( @@ -583,7 +581,7 @@ func (ds *DataSource) addIndexScanSelection(copTask *copTask, path *accessPath) // Add filter condition to table plan now. is, ok := copTask.indexPlan.(*PhysicalIndexScan) if !ok { - return errors.Errorf("type assertion fail, expect PhysicalIndexScan, but got %v", reflect.TypeOf(copTask.indexPlan)) + return errors.Errorf("type assertion fail, expect PhysicalIndexScan, but got %T", copTask.indexPlan) } indexConds, tableConds := path.indexFilters, path.tableFilters if indexConds != nil { @@ -602,7 +600,7 @@ func (ds *DataSource) addIndexScanSelection(copTask *copTask, path *accessPath) copTask.finishIndexPlan() ts, ok := copTask.tablePlan.(*PhysicalTableScan) if !ok { - return errors.Errorf("type assertion fail, expect PhysicalTableScan, but got %v", reflect.TypeOf(copTask.tablePlan)) + return errors.Errorf("type assertion fail, expect PhysicalTableScan, but got %T", copTask.tablePlan) } ts.filterCondition = append(ts.filterCondition, tableConds...) } @@ -881,7 +879,7 @@ func (ds *DataSource) pushDownSelAndResolveVirtualCols(copTask *copTask, path *a } ts, ok := copTask.tablePlan.(*PhysicalTableScan) if !ok { - return invalidTask, errors.Errorf("type assertion fail, expect PhysicalTableScan, but got %v", reflect.TypeOf(copTask.tablePlan)) + return invalidTask, errors.Errorf("type assertion fail, expect PhysicalTableScan, but got %T", copTask.tablePlan) } if len(ds.virtualColExprs) == 0 { ds.addTableScanSelection(copTask, ts, stats) From ba1b73ef3eab502f5c0ce542f45a29ea124e1518 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Thu, 25 Jul 2019 10:47:24 +0800 Subject: [PATCH 28/29] rebase --- planner/core/logical_plan_builder.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 9f1682427181e..418756b82b4fb 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -2377,13 +2377,8 @@ func (b *PlanBuilder) BuildDataSourceFromView(ctx context.Context, dbName model. // prepareVirtualColumns is only for DataSource. // It prepares virtualColExprs and virtualColSchema for table which has virtual generated columns. // virtualColExprs and virtualColSchema are used to rewrite virtual columns in pushDownSelAndResolveVirtualCols. -//<<<<<<< HEAD -//func (b *PlanBuilder) prepareVirtualColumns(ctx context.Context, ds *DataSource, columns []*table.Column) error { -// ds.hasVirtualCol = false -//======= func (b *PlanBuilder) prepareVirtualColumns(ctx context.Context, ds *DataSource, columns []*table.Column) error { hasVirtualCol := false - //>>>>>>> address comments for _, column := range columns { if column.IsGenerated() && !column.GeneratedStored { hasVirtualCol = true From 4d9f40dbe07ec2f3ee48588220733a36bcbb3dde Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Thu, 25 Jul 2019 10:48:54 +0800 Subject: [PATCH 29/29] refmt fmt fixup fixup refmt add UT --- executor/builder.go | 28 +++++++++++++++++++++++++- executor/projection.go | 4 ++++ planner/core/cbo_test.go | 14 +++++++++++++ planner/core/exhaust_physical_plans.go | 12 +++++++---- planner/core/find_best_task.go | 21 +++++++++++-------- 5 files changed, 66 insertions(+), 13 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index ebd8295088404..85fdf930ec86b 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1946,7 +1946,12 @@ type dataReaderBuilder struct { func (builder *dataReaderBuilder) buildExecutorForIndexJoin(ctx context.Context, lookUpContents []*indexJoinLookUpContent, IndexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager) (Executor, error) { - switch v := builder.Plan.(type) { + return builder.buildReaderForIndexJoin(ctx, builder.Plan, lookUpContents, IndexRanges, keyOff2IdxOff, cwc) +} + +func (builder *dataReaderBuilder) buildReaderForIndexJoin(ctx context.Context, p plannercore.Plan, + lookUpContents []*indexJoinLookUpContent, IndexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager) (Executor, error) { + switch v := p.(type) { case *plannercore.PhysicalTableReader: return builder.buildTableReaderForIndexJoin(ctx, v, lookUpContents) case *plannercore.PhysicalIndexReader: @@ -1955,10 +1960,31 @@ func (builder *dataReaderBuilder) buildExecutorForIndexJoin(ctx context.Context, return builder.buildIndexLookUpReaderForIndexJoin(ctx, v, lookUpContents, IndexRanges, keyOff2IdxOff, cwc) case *plannercore.PhysicalUnionScan: return builder.buildUnionScanForIndexJoin(ctx, v, lookUpContents, IndexRanges, keyOff2IdxOff, cwc) + case *plannercore.PhysicalProjection: + return builder.buildProjectionForIndexJoin(ctx, v, lookUpContents, IndexRanges, keyOff2IdxOff, cwc) } + return nil, errors.New("Wrong plan type for dataReaderBuilder") } +func (builder *dataReaderBuilder) buildProjectionForIndexJoin(ctx context.Context, v *plannercore.PhysicalProjection, + lookUpContents []*indexJoinLookUpContent, IndexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager) (Executor, error) { + childExec, err := builder.buildReaderForIndexJoin(ctx, v.Children()[0], lookUpContents, IndexRanges, keyOff2IdxOff, cwc) + if err != nil { + return nil, err + } + e := &ProjectionExec{ + baseExecutor: newBaseExecutor(builder.ctx, v.Schema(), v.ExplainID(), childExec), + numWorkers: 0, // always run in un-parallel mode to avoid too many number of goroutines. + evaluatorSuit: expression.NewEvaluatorSuite(v.Exprs, v.AvoidColumnEvaluator), + calculateNoDelay: v.CalculateNoDelay, + } + if err := e.open(ctx); err != nil { + return nil, nil + } + return e, nil +} + func (builder *dataReaderBuilder) buildUnionScanForIndexJoin(ctx context.Context, v *plannercore.PhysicalUnionScan, values []*indexJoinLookUpContent, indexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager) (Executor, error) { childBuilder := &dataReaderBuilder{Plan: v.Children()[0], executorBuilder: builder.executorBuilder} diff --git a/executor/projection.go b/executor/projection.go index be6aafeeb1b42..225e9b1205778 100644 --- a/executor/projection.go +++ b/executor/projection.go @@ -78,6 +78,10 @@ func (e *ProjectionExec) Open(ctx context.Context) error { return err } + return e.open(ctx) +} + +func (e *ProjectionExec) open(ctx context.Context) error { e.prepared = false e.parentReqRows = int64(e.maxChunkSize) diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index dce94b3d4e0e9..384742de53feb 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -1010,6 +1010,20 @@ func (s *testAnalyzeSuite) TestVirtualGeneratedColumn(c *C) { tk.MustIndexLookup("select * from t1 where b=1").Check(testkit.Rows("0 1 2 3")) tk.MustIndexRead("select c from t1 where c=2 and d=3").Check(testkit.Rows("2")) + tk.MustExec("insert into t1 (a) values (1)") + tk.MustQuery("select /*+ TIDB_INLJ(o, i) */ i.b, o.a from t1 o, t1 i where i.b = o.a").Check(testkit.Rows("1 1")) + tk.MustQuery("explain select /*+ TIDB_INLJ(o, i) */ * from t1 o, t1 i where i.b = o.a").Check(testkit.Rows( + "IndexJoin_12 12487.50 root inner join, inner:Projection_10, outer key:test.o.a, inner key:test.i.b", + "├─Projection_15 9990.00 root test.o.a, cast(plus(test.o.a, 1)), cast(plus(cast(plus(test.o.a, 1)), 1)), cast(plus(cast(plus(cast(plus(test.o.a, 1)), 1)), 1))", + "│ └─TableReader_16 9990.00 root data:Selection_14", + "│ └─Selection_14 9990.00 cop not(isnull(test.o.a))", + "│ └─TableScan_13 10000.00 cop table:o, range:[-inf,+inf], keep order:false, stats:pseudo", + "└─Projection_10 9.99 root test.i.a, cast(plus(test.i.a, 1)), cast(plus(cast(plus(test.i.a, 1)), 1)), cast(plus(cast(plus(cast(plus(test.i.a, 1)), 1)), 1))", + " └─IndexLookUp_11 9.99 root ", + " ├─Selection_9 9.99 cop not(isnull(test.i.b))", + " │ └─IndexScan_7 10.00 cop table:i, index:b, range: decided by [eq(test.i.b, test.o.a)], keep order:false, stats:pseudo", + " └─TableScan_8 9.99 cop table:t1, keep order:false, stats:pseudo")) + tk.MustExec(`CREATE TABLE person ( id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, name VARCHAR(255) NOT NULL, diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 50f202b76c1f8..d9cfc3cb72e8e 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -541,16 +541,18 @@ func (p *LogicalJoin) constructInnerTableScan(ds *DataSource, pk *expression.Col ts.stats.StatsVersion = statistics.PseudoVersion } - copTask := &copTask{ + cop := &copTask{ tablePlan: ts, indexPlanFinished: true, } selStats := ts.stats.Scale(selectionFactor) - t, err := ds.pushDownSelAndResolveVirtualCols(copTask, nil, selStats) + t, err := ds.pushDownSelAndResolveVirtualCols(cop, nil, selStats) if err != nil { return nil, err } - t = finishCopTask(ds.ctx, copTask) + if cop, ok := t.(*copTask); ok { + t = finishCopTask(ds.ctx, cop) + } reader := t.plan() return p.constructInnerUnionScan(us, reader), nil } @@ -626,7 +628,9 @@ func (p *LogicalJoin) constructInnerIndexScan(ds *DataSource, idx *model.IndexIn if err != nil { return nil, err } - t = finishCopTask(ds.ctx, t) + if cop, ok := t.(*copTask); ok { + t = finishCopTask(ds.ctx, cop) + } reader := t.plan() return p.constructInnerUnionScan(us, reader), nil } diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index cf63c0c024451..dea9ddd30f64a 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -14,6 +14,8 @@ package core import ( + "math" + "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -27,7 +29,6 @@ import ( "github.com/pingcap/tidb/util/ranger" "github.com/pingcap/tidb/util/set" "golang.org/x/tools/container/intsets" - "math" ) const ( @@ -538,7 +539,9 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candid return invalidTask, err } if prop.TaskTp == property.RootTaskType { - task = finishCopTask(ds.ctx, task) + if cop, ok := task.(*copTask); ok { + task = finishCopTask(ds.ctx, cop) + } } else if _, ok := task.(*rootTask); ok { return invalidTask, nil } @@ -811,7 +814,7 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid ts.Ranges = path.ranges ts.AccessCondition, ts.filterCondition = path.accessConds, path.tableFilters rowCount := path.countAfterAccess - copTask := &copTask{ + cop := &copTask{ tablePlan: ts, indexPlanFinished: true, } @@ -839,22 +842,24 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid ts.stats.StatsVersion = statistics.PseudoVersion } - copTask.cst = rowCount * scanFactor + cop.cst = rowCount * scanFactor if candidate.isMatchProp { if prop.Items[0].Desc { ts.Desc = true - copTask.cst = rowCount * descScanFactor + cop.cst = rowCount * descScanFactor } ts.KeepOrder = true - copTask.keepOrder = true + cop.keepOrder = true } - task, err = ds.pushDownSelAndResolveVirtualCols(copTask, path, ds.stats.ScaleByExpectCnt(prop.ExpectedCnt)) + task, err = ds.pushDownSelAndResolveVirtualCols(cop, path, ds.stats.ScaleByExpectCnt(prop.ExpectedCnt)) if err != nil { return invalidTask, err } if prop.TaskTp == property.RootTaskType { - task = finishCopTask(ds.ctx, task) + if cop, ok := task.(*copTask); ok { + task = finishCopTask(ds.ctx, cop) + } } else if _, ok := task.(*rootTask); ok { return invalidTask, nil }