Skip to content

Commit

Permalink
Merge branch 'master' into br_0604
Browse files Browse the repository at this point in the history
  • Loading branch information
@AndreMouche
AndreMouche authored Jun 4, 2021
2 parents c8c10e9 + b1beb1b commit 4a4cbda
Show file tree
Hide file tree
Showing 17 changed files with 677 additions and 23 deletions.
11 changes: 6 additions & 5 deletions cmd/explaintest/r/explain_easy.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,11 +124,12 @@ Projection 10000.00 root eq(test.t1.c2, test.t2.c2)->Column#11
└─Apply 10000.00 root CARTESIAN left outer join
├─TableReader(Build) 10000.00 root data:TableFullScan
│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
└─Projection(Probe) 1.00 root test.t2.c1, test.t2.c2
└─IndexLookUp 1.00 root limit embedded(offset:0, count:1)
├─Limit(Build) 1.00 cop[tikv] offset:0, count:1
│ └─IndexRangeScan 1.00 cop[tikv] table:t2, index:c1(c1) range: decided by [eq(test.t1.c1, test.t2.c1)], keep order:true, stats:pseudo
└─TableRowIDScan(Probe) 1.00 cop[tikv] table:t2 keep order:false, stats:pseudo
└─Limit(Probe) 1.00 root offset:0, count:1
└─Projection 1.00 root test.t2.c1, test.t2.c2
└─IndexLookUp 1.00 root
├─Limit(Build) 1.00 cop[tikv] offset:0, count:1
│ └─IndexRangeScan 1.00 cop[tikv] table:t2, index:c1(c1) range: decided by [eq(test.t1.c1, test.t2.c1)], keep order:true, stats:pseudo
└─TableRowIDScan(Probe) 1.00 cop[tikv] table:t2 keep order:false, stats:pseudo
explain format = 'brief' select * from t1 order by c1 desc limit 1;
id estRows task access object operator info
Limit 1.00 root offset:0, count:1
Expand Down
11 changes: 6 additions & 5 deletions cmd/explaintest/r/explain_easy_stats.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,11 +106,12 @@ Projection 1999.00 root eq(test.t1.c2, test.t2.c2)->Column#11
└─Apply 1999.00 root CARTESIAN left outer join
├─TableReader(Build) 1999.00 root data:TableFullScan
│ └─TableFullScan 1999.00 cop[tikv] table:t1 keep order:false
└─Projection(Probe) 1.00 root test.t2.c1, test.t2.c2
└─IndexLookUp 1.00 root limit embedded(offset:0, count:1)
├─Limit(Build) 1.00 cop[tikv] offset:0, count:1
│ └─IndexRangeScan 1.25 cop[tikv] table:t2, index:c1(c1) range: decided by [eq(test.t1.c1, test.t2.c1)], keep order:true
└─TableRowIDScan(Probe) 1.00 cop[tikv] table:t2 keep order:false, stats:pseudo
└─Limit(Probe) 1.00 root offset:0, count:1
└─Projection 1.00 root test.t2.c1, test.t2.c2
└─IndexLookUp 1.00 root
├─Limit(Build) 1.00 cop[tikv] offset:0, count:1
│ └─IndexRangeScan 1.25 cop[tikv] table:t2, index:c1(c1) range: decided by [eq(test.t1.c1, test.t2.c1)], keep order:true
└─TableRowIDScan(Probe) 1.00 cop[tikv] table:t2 keep order:false, stats:pseudo
explain format = 'brief' select * from t1 order by c1 desc limit 1;
id estRows task access object operator info
Limit 1.00 root offset:0, count:1
Expand Down
36 changes: 36 additions & 0 deletions executor/partition_table_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2403,6 +2403,42 @@ func (s *partitionTableSuite) TestDirectReadingWithAgg(c *C) {
}
}

func (s *partitionTableSuite) TestIssue24636(c *C) {
tk := testkit.NewTestKitWithInit(c, s.store)
tk.MustExec("create database test_issue_24636")
tk.MustExec("use test_issue_24636")

tk.MustExec(`CREATE TABLE t (a int, b date, c int, PRIMARY KEY (a,b))
PARTITION BY RANGE ( TO_DAYS(b) ) (
PARTITION p0 VALUES LESS THAN (737821),
PARTITION p1 VALUES LESS THAN (738289)
)`)
tk.MustExec(`INSERT INTO t (a, b, c) VALUES(0, '2021-05-05', 0)`)
tk.MustQuery(`select c from t use index(primary) where a=0 limit 1`).Check(testkit.Rows("0"))

tk.MustExec(`
CREATE TABLE test_partition (
a varchar(100) NOT NULL,
b date NOT NULL,
c varchar(100) NOT NULL,
d datetime DEFAULT NULL,
e datetime DEFAULT NULL,
f bigint(20) DEFAULT NULL,
g bigint(20) DEFAULT NULL,
h bigint(20) DEFAULT NULL,
i bigint(20) DEFAULT NULL,
j bigint(20) DEFAULT NULL,
k bigint(20) DEFAULT NULL,
l bigint(20) DEFAULT NULL,
PRIMARY KEY (a,b,c) /*T![clustered_index] NONCLUSTERED */
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin
PARTITION BY RANGE ( TO_DAYS(b) ) (
PARTITION pmin VALUES LESS THAN (737821),
PARTITION p20200601 VALUES LESS THAN (738289))`)
tk.MustExec(`INSERT INTO test_partition (a, b, c, d, e, f, g, h, i, j, k, l) VALUES('aaa', '2021-05-05', '428ff6a1-bb37-42ac-9883-33d7a29961e6', '2021-05-06 08:13:38', '2021-05-06 13:28:08', 0, 8, 3, 0, 9, 1, 0)`)
tk.MustQuery(`select c,j,l from test_partition where c='428ff6a1-bb37-42ac-9883-33d7a29961e6' and a='aaa' limit 0, 200`).Check(testkit.Rows("428ff6a1-bb37-42ac-9883-33d7a29961e6 9 0"))
}

func (s *partitionTableSuite) TestIdexMerge(c *C) {
if israce.RaceEnabled {
c.Skip("exhaustive types test, skip race test")
Expand Down
277 changes: 277 additions & 0 deletions executor/tiflash_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ package executor_test

import (
"fmt"
"math/rand"
"strings"
"sync"
"sync/atomic"
"time"
Expand Down Expand Up @@ -248,6 +250,142 @@ func (s *tiflashTestSuite) TestInjectExtraProj(c *C) {
tk.MustQuery("select avg(a), a from t group by a").Check(testkit.Rows("9223372036854775807.0000 9223372036854775807"))
}

func (s *tiflashTestSuite) TestTiFlashPartitionTableShuffledHashJoin(c *C) {
if israce.RaceEnabled {
c.Skip("exhaustive types test, skip race test")
}

tk := testkit.NewTestKit(c, s.store)
tk.MustExec(`create database tiflash_partition_SHJ`)
tk.MustExec("use tiflash_partition_SHJ")
tk.MustExec(`create table thash (a int, b int) partition by hash(a) partitions 4`)
tk.MustExec(`create table trange (a int, b int) partition by range(a) (
partition p0 values less than (100), partition p1 values less than (200),
partition p2 values less than (300), partition p3 values less than (400))`)
listPartitions := make([]string, 4)
for i := 0; i < 400; i++ {
idx := i % 4
if listPartitions[idx] != "" {
listPartitions[idx] += ", "
}
listPartitions[idx] = listPartitions[idx] + fmt.Sprintf("%v", i)
}
tk.MustExec(`create table tlist (a int, b int) partition by list(a) (
partition p0 values in (` + listPartitions[0] + `), partition p1 values in (` + listPartitions[1] + `),
partition p2 values in (` + listPartitions[2] + `), partition p3 values in (` + listPartitions[3] + `))`)
tk.MustExec(`create table tnormal (a int, b int)`)

for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} {
tk.MustExec("alter table " + tbl + " set tiflash replica 1")
tb := testGetTableByName(c, tk.Se, "tiflash_partition_SHJ", tbl)
err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true)
c.Assert(err, IsNil)
}

vals := make([]string, 0, 100)
for i := 0; i < 100; i++ {
vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(400), rand.Intn(400)))
}
for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} {
tk.MustExec(fmt.Sprintf("insert into %v values %v", tbl, strings.Join(vals, ", ")))
tk.MustExec(fmt.Sprintf("analyze table %v", tbl))
}

tk.MustExec("SET tidb_allow_mpp=2")
tk.MustExec("SET tidb_opt_broadcast_join=0")
tk.MustExec("SET tidb_broadcast_join_threshold_count=0")
tk.MustExec("SET tidb_broadcast_join_threshold_size=0")
tk.MustExec("set @@session.tidb_isolation_read_engines='tiflash'")

lr := func() (int, int) {
l, r := rand.Intn(400), rand.Intn(400)
if l > r {
l, r = r, l
}
return l, r
}
for i := 0; i < 2; i++ {
l1, r1 := lr()
l2, r2 := lr()
cond := fmt.Sprintf("t1.b>=%v and t1.b<=%v and t2.b>=%v and t2.b<=%v", l1, r1, l2, r2)
var res [][]interface{}
for _, mode := range []string{"static", "dynamic"} {
tk.MustExec(fmt.Sprintf("set @@tidb_partition_prune_mode = '%v'", mode))
for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} {
q := fmt.Sprintf("select count(*) from %v t1 join %v t2 on t1.a=t2.a where %v", tbl, tbl, cond)
if res == nil {
res = tk.MustQuery(q).Sort().Rows()
} else {
tk.MustQuery(q).Check(res)
}
}
}
}
}

func (s *tiflashTestSuite) TestTiFlashPartitionTableReader(c *C) {
if israce.RaceEnabled {
c.Skip("exhaustive types test, skip race test")
}

tk := testkit.NewTestKit(c, s.store)
tk.MustExec(`create database tiflash_partition_tablereader`)
tk.MustExec("use tiflash_partition_tablereader")
tk.MustExec(`create table thash (a int, b int) partition by hash(a) partitions 4`)
tk.MustExec(`create table trange (a int, b int) partition by range(a) (
partition p0 values less than (100), partition p1 values less than (200),
partition p2 values less than (300), partition p3 values less than (400))`)
listPartitions := make([]string, 4)
for i := 0; i < 400; i++ {
idx := i % 4
if listPartitions[idx] != "" {
listPartitions[idx] += ", "
}
listPartitions[idx] = listPartitions[idx] + fmt.Sprintf("%v", i)
}
tk.MustExec(`create table tlist (a int, b int) partition by list(a) (
partition p0 values in (` + listPartitions[0] + `), partition p1 values in (` + listPartitions[1] + `),
partition p2 values in (` + listPartitions[2] + `), partition p3 values in (` + listPartitions[3] + `))`)
tk.MustExec(`create table tnormal (a int, b int)`)

for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} {
tk.MustExec("alter table " + tbl + " set tiflash replica 1")
tb := testGetTableByName(c, tk.Se, "tiflash_partition_tablereader", tbl)
err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true)
c.Assert(err, IsNil)
}

vals := make([]string, 0, 500)
for i := 0; i < 500; i++ {
vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(400), rand.Intn(400)))
}
for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} {
tk.MustExec(fmt.Sprintf("insert into %v values %v", tbl, strings.Join(vals, ", ")))
}

tk.MustExec("SET tidb_allow_mpp=2")
tk.MustExec("set @@session.tidb_isolation_read_engines='tiflash'")
for i := 0; i < 100; i++ {
l, r := rand.Intn(400), rand.Intn(400)
if l > r {
l, r = r, l
}
cond := fmt.Sprintf("a>=%v and a<=%v", l, r)
var res [][]interface{}
for _, mode := range []string{"static", "dynamic"} {
tk.MustExec(fmt.Sprintf("set @@tidb_partition_prune_mode = '%v'", mode))
for _, tbl := range []string{"thash", "trange", "tlist", "tnormal"} {
q := fmt.Sprintf("select * from %v where %v", tbl, cond)
if res == nil {
res = tk.MustQuery(q).Sort().Rows()
} else {
tk.MustQuery(q).Sort().Check(res)
}
}
}
}
}

func (s *tiflashTestSuite) TestPartitionTable(c *C) {
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
Expand Down Expand Up @@ -526,3 +664,142 @@ func (s *tiflashTestSuite) TestTiFlashVirtualColumn(c *C) {
tk.MustQuery("select /*+ hash_agg() */ count(*) from t2 where c > 1").Check(testkit.Rows("2"))
tk.MustQuery("select /*+ hash_agg() */ count(*) from t3 where c > b'01'").Check(testkit.Rows("3"))
}

func (s *tiflashTestSuite) TestTiFlashPartitionTableShuffledHashAggregation(c *C) {
if israce.RaceEnabled {
c.Skip("exhaustive types test, skip race test")
}

tk := testkit.NewTestKit(c, s.store)
tk.MustExec("create database tiflash_partition_AGG")
tk.MustExec("use tiflash_partition_AGG")
tk.MustExec(`create table thash (a int, b int) partition by hash(a) partitions 4`)
tk.MustExec(`create table trange (a int, b int) partition by range(a) (
partition p0 values less than (100), partition p1 values less than (200),
partition p2 values less than (300), partition p3 values less than (400))`)
listPartitions := make([]string, 4)
for i := 0; i < 400; i++ {
idx := i % 4
if listPartitions[idx] != "" {
listPartitions[idx] += ", "
}
listPartitions[idx] = listPartitions[idx] + fmt.Sprintf("%v", i)
}
tk.MustExec(`create table tlist (a int, b int) partition by list(a) (
partition p0 values in (` + listPartitions[0] + `), partition p1 values in (` + listPartitions[1] + `),
partition p2 values in (` + listPartitions[2] + `), partition p3 values in (` + listPartitions[3] + `))`)
tk.MustExec(`create table tnormal (a int, b int) partition by hash(a) partitions 4`)

for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} {
tk.MustExec("alter table " + tbl + " set tiflash replica 1")
tb := testGetTableByName(c, tk.Se, "tiflash_partition_AGG", tbl)
err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true)
c.Assert(err, IsNil)
}

vals := make([]string, 0, 100)
for i := 0; i < 100; i++ {
vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(400), rand.Intn(400)))
}
for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} {
tk.MustExec(fmt.Sprintf("insert into %v values %v", tbl, strings.Join(vals, ", ")))
tk.MustExec(fmt.Sprintf("analyze table %v", tbl))
}
tk.MustExec("set @@session.tidb_isolation_read_engines='tiflash'")
tk.MustExec("set @@session.tidb_allow_mpp=2")

lr := func() (int, int) {
l, r := rand.Intn(400), rand.Intn(400)
if l > r {
l, r = r, l
}
return l, r
}
for i := 0; i < 2; i++ {
l1, r1 := lr()
cond := fmt.Sprintf("t1.b>=%v and t1.b<=%v", l1, r1)
var res [][]interface{}
for _, mode := range []string{"static", "dynamic"} {
tk.MustExec(fmt.Sprintf("set @@tidb_partition_prune_mode = '%v'", mode))
for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} {
q := fmt.Sprintf("select /*+ HASH_AGG() */ count(*) from %v t1 where %v", tbl, cond)
c.Assert(tk.HasPlan(q, "HashAgg"), IsTrue)
if res == nil {
res = tk.MustQuery(q).Sort().Rows()
} else {
tk.MustQuery(q).Check(res)
}
}
}
}
}

func (s *tiflashTestSuite) TestTiFlashPartitionTableBroadcastJoin(c *C) {
if israce.RaceEnabled {
c.Skip("exhaustive types test, skip race test")
}

tk := testkit.NewTestKit(c, s.store)
tk.MustExec("create database tiflash_partition_BCJ")
tk.MustExec("use tiflash_partition_BCJ")
tk.MustExec(`create table thash (a int, b int) partition by hash(a) partitions 4`)
tk.MustExec(`create table trange (a int, b int) partition by range(a) (
partition p0 values less than (100), partition p1 values less than (200),
partition p2 values less than (300), partition p3 values less than (400))`)
listPartitions := make([]string, 4)
for i := 0; i < 400; i++ {
idx := i % 4
if listPartitions[idx] != "" {
listPartitions[idx] += ", "
}
listPartitions[idx] = listPartitions[idx] + fmt.Sprintf("%v", i)
}
tk.MustExec(`create table tlist (a int, b int) partition by list(a) (
partition p0 values in (` + listPartitions[0] + `), partition p1 values in (` + listPartitions[1] + `),
partition p2 values in (` + listPartitions[2] + `), partition p3 values in (` + listPartitions[3] + `))`)
tk.MustExec(`create table tnormal (a int, b int) partition by hash(a) partitions 4`)

for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} {
tk.MustExec("alter table " + tbl + " set tiflash replica 1")
tb := testGetTableByName(c, tk.Se, "tiflash_partition_BCJ", tbl)
err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true)
c.Assert(err, IsNil)
}

vals := make([]string, 0, 100)
for i := 0; i < 100; i++ {
vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(400), rand.Intn(400)))
}
for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} {
tk.MustExec(fmt.Sprintf("insert into %v values %v", tbl, strings.Join(vals, ", ")))
tk.MustExec(fmt.Sprintf("analyze table %v", tbl))
}
tk.MustExec("set @@session.tidb_isolation_read_engines='tiflash'")
tk.MustExec("set @@session.tidb_allow_mpp=2")
tk.MustExec("set @@session.tidb_opt_broadcast_join=ON")

lr := func() (int, int) {
l, r := rand.Intn(400), rand.Intn(400)
if l > r {
l, r = r, l
}
return l, r
}
for i := 0; i < 2; i++ {
l1, r1 := lr()
l2, r2 := lr()
cond := fmt.Sprintf("t1.b>=%v and t1.b<=%v and t2.b>=%v and t2.b<=%v", l1, r1, l2, r2)
var res [][]interface{}
for _, mode := range []string{"static", "dynamic"} {
tk.MustExec(fmt.Sprintf("set @@tidb_partition_prune_mode = '%v'", mode))
for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} {
q := fmt.Sprintf("select count(*) from %v t1 join %v t2 on t1.a=t2.a where %v", tbl, tbl, cond)
if res == nil {
res = tk.MustQuery(q).Sort().Rows()
} else {
tk.MustQuery(q).Check(res)
}
}
}
}
}
7 changes: 5 additions & 2 deletions expression/expression.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -999,16 +999,19 @@ func scalarExprSupportedByFlash(function *ScalarFunction) bool {
case
ast.LogicOr, ast.LogicAnd, ast.UnaryNot, ast.BitNeg, ast.Xor, ast.And, ast.Or,
ast.GE, ast.LE, ast.EQ, ast.NE, ast.LT, ast.GT, ast.In, ast.IsNull, ast.Like,
ast.Plus, ast.Minus, ast.Div, ast.Mul, /*ast.Mod,*/
ast.Plus, ast.Minus, ast.Div, ast.Mul, ast.Abs, /*ast.Mod,*/
ast.If, ast.Ifnull, ast.Case,
ast.Concat, ast.ConcatWS,
ast.Year, ast.Month, ast.Day,
ast.DateDiff, ast.TimestampDiff, ast.DateFormat, ast.FromUnixTime,
ast.JSONLength:
return true
case ast.Substr, ast.Substring:
case ast.Substr, ast.Substring, ast.Left, ast.Right, ast.CharLength:
switch function.Function.PbCode() {
case
tipb.ScalarFuncSig_LeftUTF8,
tipb.ScalarFuncSig_RightUTF8,
tipb.ScalarFuncSig_CharLengthUTF8,
tipb.ScalarFuncSig_Substring2ArgsUTF8,
tipb.ScalarFuncSig_Substring3ArgsUTF8:
return true
Expand Down
Loading

0 comments on commit 4a4cbda

Please sign in to comment.