Skip to content

Commit 7a96df6

Browse files
SunRunAwaySunRunAway
authored
Merge branch 'release-3.1' into automated-cherry-pick-of-pingcap#12173-upstream-release-3.1
2 parents 8699305 + 7cee5b7 commit 7a96df6

15 files changed

+237
-39
lines changed

‎cmd/explaintest/r/tpch.result

+1-1
Original file line numberDiff line numberDiff line change
@@ -1223,7 +1223,7 @@ id count task operator info
12231223
Projection_25 1.00 root tpch.supplier.s_name, 17_col_0
12241224
└─TopN_28 1.00 root 17_col_0:desc, tpch.supplier.s_name:asc, offset:0, count:100
12251225
└─HashAgg_34 1.00 root group by:tpch.supplier.s_name, funcs:count(1), firstrow(tpch.supplier.s_name)
1226-
└─IndexJoin_40 7828961.66 root anti semi join, inner:IndexLookUp_39, outer key:tpch.l1.l_orderkey, inner key:tpch.l3.l_orderkey, other cond:ne(tpch.l3.l_suppkey, tpch.l1.l_suppkey), ne(tpch.l3.l_suppkey, tpch.supplier.s_suppkey)
1226+
└─IndexJoin_40 7828961.66 root anti semi join, inner:IndexLookUp_39, outer key:tpch.l1.l_orderkey, inner key:tpch.l3.l_orderkey, other cond:ne(tpch.l3.l_suppkey, tpch.l1.l_suppkey)
12271227
├─IndexJoin_56 9786202.08 root semi join, inner:IndexLookUp_55, outer key:tpch.l1.l_orderkey, inner key:tpch.l2.l_orderkey, other cond:ne(tpch.l2.l_suppkey, tpch.l1.l_suppkey), ne(tpch.l2.l_suppkey, tpch.supplier.s_suppkey)
12281228
│ ├─IndexJoin_62 12232752.60 root inner join, inner:TableReader_61, outer key:tpch.l1.l_orderkey, inner key:tpch.orders.o_orderkey
12291229
│ │ ├─HashRightJoin_66 12232752.60 root inner join, inner:HashRightJoin_72, equal:[eq(tpch.supplier.s_suppkey, tpch.l1.l_suppkey)]

‎executor/show.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@ func (e *ShowExec) fetchShowProcessList() error {
274274
if !hasProcessPriv && pi.User != loginUser.Username {
275275
continue
276276
}
277-
row := pi.ToRow(e.Full)
277+
row := pi.ToRowForShow(e.Full)
278278
e.appendRow(row)
279279
}
280280
return nil

‎expression/expression.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ func EvaluateExprWithNull(ctx sessionctx.Context, schema *Schema, expr Expressio
276276
for i, arg := range x.GetArgs() {
277277
args[i] = EvaluateExprWithNull(ctx, schema, arg)
278278
}
279-
return NewFunctionInternal(ctx, x.FuncName.L, types.NewFieldType(mysql.TypeTiny), args...)
279+
return NewFunctionInternal(ctx, x.FuncName.L, x.RetType, args...)
280280
case *Column:
281281
if !schema.Contains(x) {
282282
return x

‎infoschema/tables.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -543,6 +543,7 @@ var tableProcesslistCols = []columnInfo{
543543
{"TIME", mysql.TypeLong, 7, mysql.NotNullFlag, 0, nil},
544544
{"STATE", mysql.TypeVarchar, 7, 0, nil, nil},
545545
{"INFO", mysql.TypeString, 512, 0, nil, nil},
546+
{"MEM", mysql.TypeLonglong, 21, 0, nil, nil},
546547
}
547548

548549
var tableTiDBIndexesCols = []columnInfo{
@@ -862,7 +863,7 @@ func dataForProcesslist(ctx sessionctx.Context) [][]types.Datum {
862863
continue
863864
}
864865

865-
rows := pi.ToRow(true)
866+
rows := pi.ToRow()
866867
record := types.MakeDatums(rows...)
867868
records = append(records, record)
868869
}

‎infoschema/tables_test.go

+26-9
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ func (s *testTableSuite) TestInfoschemaFieldValue(c *C) {
131131
User: "root",
132132
Host: "127.0.0.1",
133133
Command: mysql.ComQuery,
134+
StmtCtx: tk.Se.GetSessionVars().StmtCtx,
134135
}
135136
tk.Se.SetSessionManager(sm)
136137
tk.MustQuery("SELECT user,host,command FROM information_schema.processlist;").Check(testkit.Rows("root 127.0.0.1 Query"))
@@ -342,19 +343,35 @@ func (s *testTableSuite) TestSomeTables(c *C) {
342343
DB: "information_schema",
343344
Command: byte(1),
344345
State: 1,
345-
Info: "do something"}
346+
Info: "do something",
347+
StmtCtx: tk.Se.GetSessionVars().StmtCtx,
348+
}
346349
sm.processInfoMap[2] = &util.ProcessInfo{
347350
ID: 2,
348351
User: "user-2",
349352
Host: "localhost",
350353
DB: "test",
351354
Command: byte(2),
352355
State: 2,
353-
Info: "do something"}
356+
Info: strings.Repeat("x", 101),
357+
StmtCtx: tk.Se.GetSessionVars().StmtCtx,
358+
}
354359
tk.Se.SetSessionManager(sm)
355-
tk.MustQuery("select * from information_schema.PROCESSLIST order by ID;").Check(
356-
testkit.Rows("1 user-1 localhost information_schema Quit 9223372036 1 do something",
357-
"2 user-2 localhost test Init DB 9223372036 2 do something"))
360+
tk.MustQuery("select * from information_schema.PROCESSLIST order by ID;").Sort().Check(
361+
testkit.Rows(
362+
fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 1 %s 0", "do something"),
363+
fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 2 %s 0", strings.Repeat("x", 101)),
364+
))
365+
tk.MustQuery("SHOW PROCESSLIST;").Sort().Check(
366+
testkit.Rows(
367+
fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 1 %s", "do something"),
368+
fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 2 %s", strings.Repeat("x", 100)),
369+
))
370+
tk.MustQuery("SHOW FULL PROCESSLIST;").Sort().Check(
371+
testkit.Rows(
372+
fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 1 %s", "do something"),
373+
fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 2 %s", strings.Repeat("x", 101)),
374+
))
358375

359376
sm = &mockSessionManager{make(map[uint64]*util.ProcessInfo, 2)}
360377
sm.processInfoMap[1] = &util.ProcessInfo{
@@ -380,8 +397,8 @@ func (s *testTableSuite) TestSomeTables(c *C) {
380397
tk.Se.SetSessionManager(sm)
381398
tk.MustQuery("select * from information_schema.PROCESSLIST order by ID;").Check(
382399
testkit.Rows(
383-
fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 1 %s", "<nil>"),
384-
fmt.Sprintf("2 user-2 localhost <nil> Init DB 9223372036 2 %s", strings.Repeat("x", 101)),
400+
fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 1 %s 0", "<nil>"),
401+
fmt.Sprintf("2 user-2 localhost <nil> Init DB 9223372036 2 %s 0", strings.Repeat("x", 101)),
385402
))
386403
tk.MustQuery("SHOW PROCESSLIST;").Sort().Check(
387404
testkit.Rows(
@@ -395,11 +412,11 @@ func (s *testTableSuite) TestSomeTables(c *C) {
395412
))
396413
tk.MustQuery("select * from information_schema.PROCESSLIST where db is null;").Check(
397414
testkit.Rows(
398-
fmt.Sprintf("2 user-2 localhost <nil> Init DB 9223372036 2 %s", strings.Repeat("x", 101)),
415+
fmt.Sprintf("2 user-2 localhost <nil> Init DB 9223372036 2 %s 0", strings.Repeat("x", 101)),
399416
))
400417
tk.MustQuery("select * from information_schema.PROCESSLIST where Info is null;").Check(
401418
testkit.Rows(
402-
fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 1 %s", "<nil>"),
419+
fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 1 %s 0", "<nil>"),
403420
))
404421
}
405422

‎planner/core/integration_test.go

+67
Original file line numberDiff line numberDiff line change
@@ -122,3 +122,70 @@ func (s *testIntegrationSuite) TestApplyNotNullFlag(c *C) {
122122

123123
tk.MustQuery("select IFNULL((select t1.x from t1 where t1.x = t2.x), 'xxx') as col1 from t2").Check(testkit.Rows("xxx"))
124124
}
125+
126+
func (s *testIntegrationSuite) TestSimplifyOuterJoinWithCast(c *C) {
127+
tk := testkit.NewTestKit(c, s.store)
128+
129+
tk.MustExec("use test")
130+
tk.MustExec("drop table if exists t")
131+
tk.MustExec("create table t(a int not null, b datetime default null)")
132+
133+
var input []string
134+
var output []struct {
135+
SQL string
136+
Plan []string
137+
}
138+
s.testData.GetTestCases(c, &input, &output)
139+
for i, tt := range input {
140+
s.testData.OnRecord(func() {
141+
output[i].SQL = tt
142+
output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows())
143+
})
144+
tk.MustQuery(tt).Check(testkit.Rows(output[i].Plan...))
145+
}
146+
}
147+
148+
func (s *testIntegrationSuite) TestAntiJoinConstProp(c *C) {
149+
store, dom, err := newStoreWithBootstrap()
150+
c.Assert(err, IsNil)
151+
tk := testkit.NewTestKit(c, store)
152+
defer func() {
153+
dom.Close()
154+
store.Close()
155+
}()
156+
tk.MustExec("use test")
157+
tk.MustExec("drop table if exists t1, t2")
158+
tk.MustExec("create table t1(a int not null, b int not null)")
159+
tk.MustExec("insert into t1 values (1,1)")
160+
tk.MustExec("create table t2(a int not null, b int not null)")
161+
tk.MustExec("insert into t2 values (2,2)")
162+
163+
tk.MustQuery("select * from t1 where t1.a not in (select a from t2 where t2.a = t1.a and t2.a > 1)").Check(testkit.Rows(
164+
"1 1",
165+
))
166+
tk.MustQuery("select * from t1 where t1.a not in (select a from t2 where t2.b = t1.b and t2.a > 1)").Check(testkit.Rows(
167+
"1 1",
168+
))
169+
tk.MustQuery("select * from t1 where t1.a not in (select a from t2 where t2.b = t1.b and t2.b > 1)").Check(testkit.Rows(
170+
"1 1",
171+
))
172+
tk.MustQuery("select q.a in (select count(*) from t1 s where not exists (select 1 from t1 p where q.a > 1 and p.a = s.a)) from t1 q").Check(testkit.Rows(
173+
"1",
174+
))
175+
tk.MustQuery("select q.a in (select not exists (select 1 from t1 p where q.a > 1 and p.a = s.a) from t1 s) from t1 q").Check(testkit.Rows(
176+
"1",
177+
))
178+
179+
tk.MustExec("drop table t1, t2")
180+
tk.MustExec("create table t1(a int not null, b int)")
181+
tk.MustExec("insert into t1 values (1,null)")
182+
tk.MustExec("create table t2(a int not null, b int)")
183+
tk.MustExec("insert into t2 values (2,2)")
184+
185+
tk.MustQuery("select * from t1 where t1.a not in (select a from t2 where t2.b > t1.b)").Check(testkit.Rows(
186+
"1 <nil>",
187+
))
188+
tk.MustQuery("select * from t1 where t1.a not in (select a from t2 where t1.a = 2)").Check(testkit.Rows(
189+
"1 <nil>",
190+
))
191+
}

‎planner/core/logical_plan_builder.go

+8-8
Original file line numberDiff line numberDiff line change
@@ -226,9 +226,14 @@ func (p *LogicalJoin) pushDownConstExpr(expr expression.Expression, leftCond []e
226226
} else {
227227
leftCond = append(leftCond, expr)
228228
}
229-
case SemiJoin, AntiSemiJoin, InnerJoin:
229+
case SemiJoin, InnerJoin:
230230
leftCond = append(leftCond, expr)
231231
rightCond = append(rightCond, expr)
232+
case AntiSemiJoin:
233+
if filterCond {
234+
leftCond = append(leftCond, expr)
235+
}
236+
rightCond = append(rightCond, expr)
232237
}
233238
return leftCond, rightCond
234239
}
@@ -255,18 +260,13 @@ func (p *LogicalJoin) extractOnCondition(conditions []expression.Expression, der
255260
arg0, arg1 = arg1, arg0
256261
}
257262
if leftCol != nil && rightCol != nil {
258-
// Do not derive `is not null` for anti join, since it may cause wrong results.
259-
// For example:
260-
// `select * from t t1 where t1.a not in (select b from t t2)` does not imply `t2.b is not null`,
261-
// `select * from t t1 where t1.a not in (select a from t t2 where t1.b = t2.b` does not imply `t1.b is not null`,
262-
// `select * from t t1 where not exists (select * from t t2 where t2.a = t1.a)` does not imply `t1.a is not null`,
263-
if deriveLeft && p.JoinType != AntiSemiJoin {
263+
if deriveLeft {
264264
if isNullRejected(ctx, left.Schema(), expr) && !mysql.HasNotNullFlag(leftCol.RetType.Flag) {
265265
notNullExpr := expression.BuildNotNullExpr(ctx, leftCol)
266266
leftCond = append(leftCond, notNullExpr)
267267
}
268268
}
269-
if deriveRight && p.JoinType != AntiSemiJoin {
269+
if deriveRight {
270270
if isNullRejected(ctx, right.Schema(), expr) && !mysql.HasNotNullFlag(rightCol.RetType.Flag) {
271271
notNullExpr := expression.BuildNotNullExpr(ctx, rightCol)
272272
rightCond = append(rightCond, notNullExpr)

‎planner/core/logical_plan_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -445,7 +445,7 @@ func (s *testPlanSuite) TestAntiSemiJoinConstFalse(c *C) {
445445
}{
446446
{
447447
sql: "select a from t t1 where not exists (select a from t t2 where t1.a = t2.a and t2.b = 1 and t2.b = 2)",
448-
best: "Join{DataScan(t1)->DataScan(t2)}->Projection",
448+
best: "Join{DataScan(t1)->DataScan(t2)}(test.t1.a,test.t2.a)->Projection",
449449
joinType: "anti semi join",
450450
},
451451
}

‎planner/core/logical_plans.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,7 @@ type LogicalUnionScan struct {
327327
conditions []expression.Expression
328328
}
329329

330-
// DataSource represents a tablescan without condition push down.
330+
// DataSource represents a tableScan without condition push down.
331331
type DataSource struct {
332332
logicalSchemaProducer
333333

‎planner/core/rule_predicate_push_down.go

+23-8
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret
149149
p.LeftConditions = nil
150150
ret = append(expression.ScalarFuncs2Exprs(equalCond), otherCond...)
151151
ret = append(ret, leftPushCond...)
152-
case SemiJoin, AntiSemiJoin, InnerJoin:
152+
case SemiJoin, InnerJoin:
153153
tempCond := make([]expression.Expression, 0, len(p.LeftConditions)+len(p.RightConditions)+len(p.EqualConditions)+len(p.OtherConditions)+len(predicates))
154154
tempCond = append(tempCond, p.LeftConditions...)
155155
tempCond = append(tempCond, p.RightConditions...)
@@ -158,13 +158,10 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret
158158
tempCond = append(tempCond, predicates...)
159159
tempCond = expression.ExtractFiltersFromDNFs(p.ctx, tempCond)
160160
tempCond = expression.PropagateConstant(p.ctx, tempCond)
161-
// Return table dual when filter is constant false or null. Not applicable to AntiSemiJoin.
162-
// TODO: For AntiSemiJoin, we can use outer plan to substitute LogicalJoin actually.
163-
if p.JoinType != AntiSemiJoin {
164-
dual := conds2TableDual(p, tempCond)
165-
if dual != nil {
166-
return ret, dual
167-
}
161+
// Return table dual when filter is constant false or null.
162+
dual := conds2TableDual(p, tempCond)
163+
if dual != nil {
164+
return ret, dual
168165
}
169166
equalCond, leftPushCond, rightPushCond, otherCond = p.extractOnCondition(tempCond, true, true)
170167
p.LeftConditions = nil
@@ -173,6 +170,24 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret
173170
p.OtherConditions = otherCond
174171
leftCond = leftPushCond
175172
rightCond = rightPushCond
173+
case AntiSemiJoin:
174+
predicates = expression.PropagateConstant(p.ctx, predicates)
175+
// Return table dual when filter is constant false or null.
176+
dual := conds2TableDual(p, predicates)
177+
if dual != nil {
178+
return ret, dual
179+
}
180+
// `predicates` should only contain left conditions or constant filters.
181+
_, leftPushCond, rightPushCond, _ = p.extractOnCondition(predicates, true, true)
182+
// Do not derive `is not null` for anti join, since it may cause wrong results.
183+
// For example:
184+
// `select * from t t1 where t1.a not in (select b from t t2)` does not imply `t2.b is not null`,
185+
// `select * from t t1 where t1.a not in (select a from t t2 where t1.b = t2.b` does not imply `t1.b is not null`,
186+
// `select * from t t1 where not exists (select * from t t2 where t2.a = t1.a)` does not imply `t1.a is not null`,
187+
leftCond = leftPushCond
188+
rightCond = append(p.RightConditions, rightPushCond...)
189+
p.RightConditions = nil
190+
176191
}
177192
leftCond = expression.RemoveDupExprs(p.ctx, leftCond)
178193
rightCond = expression.RemoveDupExprs(p.ctx, rightCond)

‎planner/core/testdata/integration_suite_in.json

+12-5
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44
"cases": [
55
// Limit should be pushed down into IndexLookUpReader, row count of IndexLookUpReader and TableScan should be 1.00.
66
"explain select * from tbl use index(idx_b_c) where b > 1 limit 2,1",
7-
// Projection atop IndexLookUpReader, Limit should be pushed down into IndexLookUpReader, and Projection should have row count 1.00 as well.
7+
// Projection atop IndexLookUpReader, Limit should be pushed down into IndexLookUpReader, and Projection should have row count 1.00 as well.
88
"explain select * from tbl use index(idx_b_c) where b > 1 order by b desc limit 2,1",
9-
// Limit should be pushed down into IndexLookUpReader when Selection on top of IndexScan.
10-
"explain select * from tbl use index(idx_b_c) where b > 1 and c > 1 limit 2,1",
11-
// Limit should NOT be pushed down into IndexLookUpReader when Selection on top of TableScan.
12-
"explain select * from tbl use index(idx_b_c) where b > 1 and a > 1 limit 2,1"
9+
// Limit should be pushed down into IndexLookUpReader when Selection on top of IndexScan.
10+
"explain select * from tbl use index(idx_b_c) where b > 1 and c > 1 limit 2,1",
11+
// Limit should NOT be pushed down into IndexLookUpReader when Selection on top of TableScan.
12+
"explain select * from tbl use index(idx_b_c) where b > 1 and a > 1 limit 2,1"
1313
]
1414
},
1515
{
@@ -18,5 +18,12 @@
1818
// fix #12385
1919
"explain select * from t t1 left join t t2 on t1.a=t2.a where from_unixtime(t2.b);"
2020
]
21+
},
22+
{
23+
"name": "TestSimplifyOuterJoinWithCast",
24+
"cases": [
25+
// LeftOuterJoin should no be simplified to InnerJoin.
26+
"explain select * from t t1 left join t t2 on t1.a = t2.a where cast(t1.b as date) >= '2019-01-01'"
27+
]
2128
}
2229
]

‎planner/core/testdata/integration_suite_out.json

+16
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,21 @@
2020
]
2121
}
2222
]
23+
},
24+
{
25+
"Name": "TestSimplifyOuterJoinWithCast",
26+
"Cases": [
27+
{
28+
"SQL": "explain select * from t t1 left join t t2 on t1.a = t2.a where cast(t1.b as date) >= '2019-01-01'",
29+
"Plan": [
30+
"HashLeftJoin_8 10000.00 root left outer join, inner:TableReader_13, equal:[eq(test.t1.a, test.t2.a)]",
31+
"├─Selection_9 8000.00 root ge(cast(test.t1.b), 2019-01-01 00:00:00.000000)",
32+
"│ └─TableReader_11 10000.00 root data:TableScan_10",
33+
"│ └─TableScan_10 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo",
34+
"└─TableReader_13 10000.00 root data:TableScan_12",
35+
" └─TableScan_12 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo"
36+
]
37+
}
38+
]
2339
}
2440
]

‎session/session.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ var (
9898
sessionExecuteParseDurationGeneral = metrics.SessionExecuteParseDuration.WithLabelValues(metrics.LblGeneral)
9999
)
100100

101-
// Session context
101+
// Session context, it is consistent with the lifecycle of a client connection.
102102
type Session interface {
103103
sessionctx.Context
104104
Status() uint16 // Flag of current status, such as autocommit.

0 commit comments

Comments
 (0)