From 7c27a72bf15564c71c2f89e1ca6c095d3a025f53 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Thu, 1 Dec 2022 12:17:10 +0800 Subject: [PATCH 1/3] executor: add tests for property pushed down for partition table --- executor/partition_table_test.go | 64 ++++++++++++++++++++++++++++++++ planner/core/task.go | 3 ++ testkit/testkit.go | 10 +++++ 3 files changed, 77 insertions(+) diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index f9ca811e87626..09f33cd50058b 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -439,6 +439,31 @@ func TestOrderByandLimit(t *testing.T) { tk.MustExec("insert into thash_clustered values " + strings.Join(dedupValsAB, ",")) tk.MustExec("insert into tregular_clustered values " + strings.Join(dedupValsAB, ",")) + tk.MustExec("analyze table trange") + tk.MustExec("analyze table trange_intpk") + tk.MustExec("analyze table trange_clustered") + tk.MustExec("analyze table thash") + tk.MustExec("analyze table thash_intpk") + tk.MustExec("analyze table thash_clustered") + tk.MustExec("analyze table tregular") + tk.MustExec("analyze table tregular_intpk") + tk.MustExec("analyze table tregular_clustered") + + // Create virtual tiflash replica info. + dom := domain.GetDomain(tk.Session()) + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test_orderby_limit")) + require.True(t, exists) + for _, tblInfo := range db.Tables { + if strings.HasPrefix(tblInfo.Name.L, "tr") || strings.HasPrefix(tblInfo.Name.L, "thash") { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + tk.MustExec("set @@session.tidb_isolation_read_engines=\"tikv\"") + // test indexLookUp for i := 0; i < 100; i++ { // explain select * from t where a > {y} use index(idx_a) order by a limit {x}; // check if IndexLookUp is used @@ -529,6 +554,45 @@ func TestOrderByandLimit(t *testing.T) { regularResult = tk.MustQuery(queryRegular).Rows() tk.MustQuery(queryRangePartition).Check(regularResult) tk.MustQuery(queryHashPartition).Check(regularResult) + + tk.MustExec(" set @@tidb_allow_mpp=1;") + tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash,tikv\"") + queryPartitionWithTiFlash := fmt.Sprintf("select /*+ read_from_storage(tiflash[trange_intpk]) */ * from trange_intpk where a > %v order by a limit %v", x, y) + // check if tiflash is used + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + // but order is not pushed + require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[trange_intpk]) */ /*+ LIMIT_TO_COP() */ * from trange_intpk where a > %v order by a limit %v", x, y) + // check if tiflash is used + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + // but order is not pushed + require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[trange_clustered]) */ * from trange_clustered where a > %v order by a limit %v", x, y) + // check if tiflash is used + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[trange_clustered]) */ /*+ LIMIT_TO_COP() */ * from trange_clustered where a > %v order by a limit %v", x, y) + // check if tiflash is used + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash)) + // but order is not pushed + require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[thash_intpk]) */ * from thash_intpk where a > %v order by a limit %v", x, y) + // check if tiflash is used + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[thash_intpk]) */ /*+ LIMIT_TO_COP() */ * from thash_intpk where a > %v order by a limit %v", x, y) + // check if tiflash is used + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash)) + // but order is not pushed + require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[thash_clustered]) */ * from thash_clustered where a > %v order by a limit %v", x, y) + // check if tiflash is used + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[thash_clustered]) */ /*+ LIMIT_TO_COP() */ * from thash_clustered where a > %v order by a limit %v", x, y) + // check if tiflash is used + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash)) + // but order is not pushed + require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + tk.MustExec(" set @@tidb_allow_mpp=0;") + tk.MustExec("set @@session.tidb_isolation_read_engines=\"tikv\"") } // test indexReader diff --git a/planner/core/task.go b/planner/core/task.go index ac3d10a323a7b..4426fd46f8510 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -989,6 +989,9 @@ func (p *PhysicalTopN) attach2Task(tasks ...task) task { // Before that is done, we use this logic to provide a way to keep the order property when reading from TiKV, so that we can use the orderliness of index to speed up the query. // Here we can change the execution plan to TopN(TiDB)->Reader(TiDB)->Limit(TiKV)->Scan(TiKV).(TiFlash is not supported). func (p *PhysicalTopN) pushTopNDownToDynamicPartition(copTsk *copTask) (task, bool) { + if copTsk.getStoreType() != kv.TiKV { + return nil, false + } copTsk = copTsk.copy().(*copTask) if len(copTsk.rootTaskConds) > 0 { return nil, false diff --git a/testkit/testkit.go b/testkit/testkit.go index 50ba5c24aaec1..dbf9508218388 100644 --- a/testkit/testkit.go +++ b/testkit/testkit.go @@ -231,6 +231,16 @@ func (tk *TestKit) HasPlan(sql string, plan string, args ...interface{}) bool { return false } +func (tk *TestKit) HasTiFlashPlan(sql string, args ...interface{}) bool { + rs := tk.MustQuery("explain "+sql, args...) + for i := range rs.rows { + if strings.Contains(rs.rows[i][2], "tiflash") { + return true + } + } + return false +} + // HasPlanForLastExecution checks if the execution plan of the last execution contains specific plan. func (tk *TestKit) HasPlanForLastExecution(plan string) bool { connID := tk.session.GetSessionVars().ConnectionID From 6b8a1126473d3f49be603ef5464318a54ba1e186 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Thu, 1 Dec 2022 20:04:05 +0800 Subject: [PATCH 2/3] fix fmt --- executor/partition_table_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index 09f33cd50058b..15d2c2872ca9c 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -559,38 +559,38 @@ func TestOrderByandLimit(t *testing.T) { tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash,tikv\"") queryPartitionWithTiFlash := fmt.Sprintf("select /*+ read_from_storage(tiflash[trange_intpk]) */ * from trange_intpk where a > %v order by a limit %v", x, y) // check if tiflash is used - require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) // but order is not pushed - require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[trange_intpk]) */ /*+ LIMIT_TO_COP() */ * from trange_intpk where a > %v order by a limit %v", x, y) // check if tiflash is used - require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) // but order is not pushed require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[trange_clustered]) */ * from trange_clustered where a > %v order by a limit %v", x, y) // check if tiflash is used - require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[trange_clustered]) */ /*+ LIMIT_TO_COP() */ * from trange_clustered where a > %v order by a limit %v", x, y) // check if tiflash is used - require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash)) + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash)) // but order is not pushed - require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[thash_intpk]) */ * from thash_intpk where a > %v order by a limit %v", x, y) // check if tiflash is used - require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[thash_intpk]) */ /*+ LIMIT_TO_COP() */ * from thash_intpk where a > %v order by a limit %v", x, y) // check if tiflash is used - require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash)) + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash)) // but order is not pushed - require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[thash_clustered]) */ * from thash_clustered where a > %v order by a limit %v", x, y) // check if tiflash is used - require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) queryPartitionWithTiFlash = fmt.Sprintf("select /*+ read_from_storage(tiflash[thash_clustered]) */ /*+ LIMIT_TO_COP() */ * from thash_clustered where a > %v order by a limit %v", x, y) // check if tiflash is used - require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash)) + require.True(t, tk.HasTiFlashPlan(queryPartitionWithTiFlash)) // but order is not pushed - require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) + require.False(t, tk.HasPlan(queryPartitionWithTiFlash, "Limit"), fmt.Sprintf("%v", tk.MustQuery("explain "+queryPartitionWithTiFlash).Rows())) tk.MustExec(" set @@tidb_allow_mpp=0;") tk.MustExec("set @@session.tidb_isolation_read_engines=\"tikv\"") } From 504a74c43d23665b9455473785c95bcd5aff8228 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Thu, 1 Dec 2022 21:52:33 +0800 Subject: [PATCH 3/3] fix lint --- testkit/testkit.go | 1 + 1 file changed, 1 insertion(+) diff --git a/testkit/testkit.go b/testkit/testkit.go index dbf9508218388..db86548ee3bfd 100644 --- a/testkit/testkit.go +++ b/testkit/testkit.go @@ -231,6 +231,7 @@ func (tk *TestKit) HasPlan(sql string, plan string, args ...interface{}) bool { return false } +// HasTiFlashPlan checks if the result execution plan contains TiFlash plan. func (tk *TestKit) HasTiFlashPlan(sql string, args ...interface{}) bool { rs := tk.MustQuery("explain "+sql, args...) for i := range rs.rows {