Skip to content

Commit e6960cd

Browse files
eurekakaeurekaka
authored
Merge branch 'release-3.0' into automated-cherry-pick-of-pingcap#12173-upstream-release-3.0
2 parents 328cf89 + c6bbf0f commit e6960cd

9 files changed

+96
-26
lines changed

‎cmd/explaintest/r/tpch.result

+1-1
Original file line numberDiff line numberDiff line change
@@ -1223,7 +1223,7 @@ id count task operator info
12231223
Projection_25 1.00 root tpch.supplier.s_name, 17_col_0
12241224
└─TopN_28 1.00 root 17_col_0:desc, tpch.supplier.s_name:asc, offset:0, count:100
12251225
└─HashAgg_34 1.00 root group by:tpch.supplier.s_name, funcs:count(1), firstrow(tpch.supplier.s_name)
1226-
└─IndexJoin_40 7828961.66 root anti semi join, inner:IndexLookUp_39, outer key:tpch.l1.l_orderkey, inner key:tpch.l3.l_orderkey, other cond:ne(tpch.l3.l_suppkey, tpch.l1.l_suppkey), ne(tpch.l3.l_suppkey, tpch.supplier.s_suppkey)
1226+
└─IndexJoin_40 7828961.66 root anti semi join, inner:IndexLookUp_39, outer key:tpch.l1.l_orderkey, inner key:tpch.l3.l_orderkey, other cond:ne(tpch.l3.l_suppkey, tpch.l1.l_suppkey)
12271227
├─IndexJoin_56 9786202.08 root semi join, inner:IndexLookUp_55, outer key:tpch.l1.l_orderkey, inner key:tpch.l2.l_orderkey, other cond:ne(tpch.l2.l_suppkey, tpch.l1.l_suppkey), ne(tpch.l2.l_suppkey, tpch.supplier.s_suppkey)
12281228
│ ├─IndexJoin_62 12232752.60 root inner join, inner:TableReader_61, outer key:tpch.l1.l_orderkey, inner key:tpch.orders.o_orderkey
12291229
│ │ ├─HashRightJoin_66 12232752.60 root inner join, inner:HashRightJoin_72, equal:[eq(tpch.supplier.s_suppkey, tpch.l1.l_suppkey)]

‎planner/core/integration_test.go

+45
Original file line numberDiff line numberDiff line change
@@ -144,3 +144,48 @@ func (s *testIntegrationSuite) TestSimplifyOuterJoinWithCast(c *C) {
144144
tk.MustQuery(tt).Check(testkit.Rows(output[i].Plan...))
145145
}
146146
}
147+
148+
func (s *testIntegrationSuite) TestAntiJoinConstProp(c *C) {
149+
store, dom, err := newStoreWithBootstrap()
150+
c.Assert(err, IsNil)
151+
tk := testkit.NewTestKit(c, store)
152+
defer func() {
153+
dom.Close()
154+
store.Close()
155+
}()
156+
tk.MustExec("use test")
157+
tk.MustExec("drop table if exists t1, t2")
158+
tk.MustExec("create table t1(a int not null, b int not null)")
159+
tk.MustExec("insert into t1 values (1,1)")
160+
tk.MustExec("create table t2(a int not null, b int not null)")
161+
tk.MustExec("insert into t2 values (2,2)")
162+
163+
tk.MustQuery("select * from t1 where t1.a not in (select a from t2 where t2.a = t1.a and t2.a > 1)").Check(testkit.Rows(
164+
"1 1",
165+
))
166+
tk.MustQuery("select * from t1 where t1.a not in (select a from t2 where t2.b = t1.b and t2.a > 1)").Check(testkit.Rows(
167+
"1 1",
168+
))
169+
tk.MustQuery("select * from t1 where t1.a not in (select a from t2 where t2.b = t1.b and t2.b > 1)").Check(testkit.Rows(
170+
"1 1",
171+
))
172+
tk.MustQuery("select q.a in (select count(*) from t1 s where not exists (select 1 from t1 p where q.a > 1 and p.a = s.a)) from t1 q").Check(testkit.Rows(
173+
"1",
174+
))
175+
tk.MustQuery("select q.a in (select not exists (select 1 from t1 p where q.a > 1 and p.a = s.a) from t1 s) from t1 q").Check(testkit.Rows(
176+
"1",
177+
))
178+
179+
tk.MustExec("drop table t1, t2")
180+
tk.MustExec("create table t1(a int not null, b int)")
181+
tk.MustExec("insert into t1 values (1,null)")
182+
tk.MustExec("create table t2(a int not null, b int)")
183+
tk.MustExec("insert into t2 values (2,2)")
184+
185+
tk.MustQuery("select * from t1 where t1.a not in (select a from t2 where t2.b > t1.b)").Check(testkit.Rows(
186+
"1 <nil>",
187+
))
188+
tk.MustQuery("select * from t1 where t1.a not in (select a from t2 where t1.a = 2)").Check(testkit.Rows(
189+
"1 <nil>",
190+
))
191+
}

‎planner/core/logical_plan_builder.go

+8-8
Original file line numberDiff line numberDiff line change
@@ -211,9 +211,14 @@ func (p *LogicalJoin) pushDownConstExpr(expr expression.Expression, leftCond []e
211211
} else {
212212
leftCond = append(leftCond, expr)
213213
}
214-
case SemiJoin, AntiSemiJoin, InnerJoin:
214+
case SemiJoin, InnerJoin:
215215
leftCond = append(leftCond, expr)
216216
rightCond = append(rightCond, expr)
217+
case AntiSemiJoin:
218+
if filterCond {
219+
leftCond = append(leftCond, expr)
220+
}
221+
rightCond = append(rightCond, expr)
217222
}
218223
return leftCond, rightCond
219224
}
@@ -240,18 +245,13 @@ func (p *LogicalJoin) extractOnCondition(conditions []expression.Expression, der
240245
arg0, arg1 = arg1, arg0
241246
}
242247
if leftCol != nil && rightCol != nil {
243-
// Do not derive `is not null` for anti join, since it may cause wrong results.
244-
// For example:
245-
// `select * from t t1 where t1.a not in (select b from t t2)` does not imply `t2.b is not null`,
246-
// `select * from t t1 where t1.a not in (select a from t t2 where t1.b = t2.b` does not imply `t1.b is not null`,
247-
// `select * from t t1 where not exists (select * from t t2 where t2.a = t1.a)` does not imply `t1.a is not null`,
248-
if deriveLeft && p.JoinType != AntiSemiJoin {
248+
if deriveLeft {
249249
if isNullRejected(ctx, left.Schema(), expr) && !mysql.HasNotNullFlag(leftCol.RetType.Flag) {
250250
notNullExpr := expression.BuildNotNullExpr(ctx, leftCol)
251251
leftCond = append(leftCond, notNullExpr)
252252
}
253253
}
254-
if deriveRight && p.JoinType != AntiSemiJoin {
254+
if deriveRight {
255255
if isNullRejected(ctx, right.Schema(), expr) && !mysql.HasNotNullFlag(rightCol.RetType.Flag) {
256256
notNullExpr := expression.BuildNotNullExpr(ctx, rightCol)
257257
rightCond = append(rightCond, notNullExpr)

‎planner/core/logical_plan_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -445,7 +445,7 @@ func (s *testPlanSuite) TestAntiSemiJoinConstFalse(c *C) {
445445
}{
446446
{
447447
sql: "select a from t t1 where not exists (select a from t t2 where t1.a = t2.a and t2.b = 1 and t2.b = 2)",
448-
best: "Join{DataScan(t1)->DataScan(t2)}->Projection",
448+
best: "Join{DataScan(t1)->DataScan(t2)}(test.t1.a,test.t2.a)->Projection",
449449
joinType: "anti semi join",
450450
},
451451
}

‎planner/core/rule_predicate_push_down.go

+23-8
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret
149149
p.LeftConditions = nil
150150
ret = append(expression.ScalarFuncs2Exprs(equalCond), otherCond...)
151151
ret = append(ret, leftPushCond...)
152-
case SemiJoin, AntiSemiJoin, InnerJoin:
152+
case SemiJoin, InnerJoin:
153153
tempCond := make([]expression.Expression, 0, len(p.LeftConditions)+len(p.RightConditions)+len(p.EqualConditions)+len(p.OtherConditions)+len(predicates))
154154
tempCond = append(tempCond, p.LeftConditions...)
155155
tempCond = append(tempCond, p.RightConditions...)
@@ -158,13 +158,10 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret
158158
tempCond = append(tempCond, predicates...)
159159
tempCond = expression.ExtractFiltersFromDNFs(p.ctx, tempCond)
160160
tempCond = expression.PropagateConstant(p.ctx, tempCond)
161-
// Return table dual when filter is constant false or null. Not applicable to AntiSemiJoin.
162-
// TODO: For AntiSemiJoin, we can use outer plan to substitute LogicalJoin actually.
163-
if p.JoinType != AntiSemiJoin {
164-
dual := conds2TableDual(p, tempCond)
165-
if dual != nil {
166-
return ret, dual
167-
}
161+
// Return table dual when filter is constant false or null.
162+
dual := conds2TableDual(p, tempCond)
163+
if dual != nil {
164+
return ret, dual
168165
}
169166
equalCond, leftPushCond, rightPushCond, otherCond = p.extractOnCondition(tempCond, true, true)
170167
p.LeftConditions = nil
@@ -173,6 +170,24 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret
173170
p.OtherConditions = otherCond
174171
leftCond = leftPushCond
175172
rightCond = rightPushCond
173+
case AntiSemiJoin:
174+
predicates = expression.PropagateConstant(p.ctx, predicates)
175+
// Return table dual when filter is constant false or null.
176+
dual := conds2TableDual(p, predicates)
177+
if dual != nil {
178+
return ret, dual
179+
}
180+
// `predicates` should only contain left conditions or constant filters.
181+
_, leftPushCond, rightPushCond, _ = p.extractOnCondition(predicates, true, true)
182+
// Do not derive `is not null` for anti join, since it may cause wrong results.
183+
// For example:
184+
// `select * from t t1 where t1.a not in (select b from t t2)` does not imply `t2.b is not null`,
185+
// `select * from t t1 where t1.a not in (select a from t t2 where t1.b = t2.b` does not imply `t1.b is not null`,
186+
// `select * from t t1 where not exists (select * from t t2 where t2.a = t1.a)` does not imply `t1.a is not null`,
187+
leftCond = leftPushCond
188+
rightCond = append(p.RightConditions, rightPushCond...)
189+
p.RightConditions = nil
190+
176191
}
177192
leftCond = expression.RemoveDupExprs(p.ctx, leftCond)
178193
rightCond = expression.RemoveDupExprs(p.ctx, rightCond)

‎statistics/cmsketch.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -445,7 +445,9 @@ func LoadCMSketchWithTopN(exec sqlexec.RestrictedSQLExecutor, tableID, isIndex,
445445
}
446446
topN := make([]*TopNMeta, 0, len(topNRows))
447447
for _, row := range topNRows {
448-
topN = append(topN, &TopNMeta{Data: row.GetBytes(0), Count: row.GetUint64(1)})
448+
data := make([]byte, len(row.GetBytes(0)))
449+
copy(data, row.GetBytes(0))
450+
topN = append(topN, &TopNMeta{Data: data, Count: row.GetUint64(1)})
449451
}
450452
return decodeCMSketch(cms, topN)
451453
}

‎statistics/handle/bootstrap.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ func (h *Handle) initStatsHistograms4Chunk(is infoschema.InfoSchema, tables Stat
9191
continue
9292
}
9393
id, ndv, nullCount, version, totColSize := row.GetInt64(2), row.GetInt64(3), row.GetInt64(5), row.GetUint64(4), row.GetInt64(7)
94+
lastAnalyzePos := row.GetDatum(11, types.NewFieldType(mysql.TypeBlob))
9495
tbl, _ := h.getTableByPhysicalID(is, table.PhysicalID)
9596
if row.GetInt64(1) > 0 {
9697
var idxInfo *model.IndexInfo
@@ -109,7 +110,7 @@ func (h *Handle) initStatsHistograms4Chunk(is infoschema.InfoSchema, tables Stat
109110
terror.Log(errors.Trace(err))
110111
}
111112
hist := statistics.NewHistogram(id, ndv, nullCount, version, types.NewFieldType(mysql.TypeBlob), chunk.InitialCapacity, 0)
112-
table.Indices[hist.ID] = &statistics.Index{Histogram: *hist, CMSketch: cms, Info: idxInfo, StatsVer: row.GetInt64(8), Flag: row.GetInt64(10), LastAnalyzePos: row.GetDatum(11, types.NewFieldType(mysql.TypeBlob))}
113+
table.Indices[hist.ID] = &statistics.Index{Histogram: *hist, CMSketch: cms, Info: idxInfo, StatsVer: row.GetInt64(8), Flag: row.GetInt64(10), LastAnalyzePos: *lastAnalyzePos.Copy()}
113114
} else {
114115
var colInfo *model.ColumnInfo
115116
for _, col := range tbl.Meta().Columns {
@@ -130,7 +131,7 @@ func (h *Handle) initStatsHistograms4Chunk(is infoschema.InfoSchema, tables Stat
130131
Count: nullCount,
131132
IsHandle: tbl.Meta().PKIsHandle && mysql.HasPriKeyFlag(colInfo.Flag),
132133
Flag: row.GetInt64(10),
133-
LastAnalyzePos: row.GetDatum(11, types.NewFieldType(mysql.TypeBlob)),
134+
LastAnalyzePos: *lastAnalyzePos.Copy(),
134135
}
135136
}
136137
}

‎statistics/handle/handle.go

+7-4
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,8 @@ func (h *Handle) Clear() {
8383
}
8484
h.feedback = h.feedback[:0]
8585
h.mu.ctx.GetSessionVars().InitChunkSize = 1
86-
h.mu.ctx.GetSessionVars().MaxChunkSize = 32
86+
h.mu.ctx.GetSessionVars().MaxChunkSize = 1
87+
h.mu.ctx.GetSessionVars().ProjectionConcurrency = 0
8788
h.listHead = &SessionStatsCollector{mapper: make(tableDeltaMap), rateMap: make(errorRateDeltaMap)}
8889
h.globalMap = make(tableDeltaMap)
8990
h.mu.rateMap = make(errorRateDeltaMap)
@@ -353,6 +354,7 @@ func (h *Handle) indexStatsFromStorage(row chunk.Row, table *statistics.Table, t
353354
idx := table.Indices[histID]
354355
errorRate := statistics.ErrorRate{}
355356
flag := row.GetInt64(8)
357+
lastAnalyzePos := row.GetDatum(10, types.NewFieldType(mysql.TypeBlob))
356358
if statistics.IsAnalyzed(flag) {
357359
h.mu.Lock()
358360
h.mu.rateMap.clear(table.PhysicalID, histID, true)
@@ -373,7 +375,7 @@ func (h *Handle) indexStatsFromStorage(row chunk.Row, table *statistics.Table, t
373375
if err != nil {
374376
return errors.Trace(err)
375377
}
376-
idx = &statistics.Index{Histogram: *hg, CMSketch: cms, Info: idxInfo, ErrorRate: errorRate, StatsVer: row.GetInt64(7), Flag: flag, LastAnalyzePos: row.GetDatum(10, types.NewFieldType(mysql.TypeBlob))}
378+
idx = &statistics.Index{Histogram: *hg, CMSketch: cms, Info: idxInfo, ErrorRate: errorRate, StatsVer: row.GetInt64(7), Flag: flag, LastAnalyzePos: *lastAnalyzePos.Copy()}
377379
}
378380
break
379381
}
@@ -392,6 +394,7 @@ func (h *Handle) columnStatsFromStorage(row chunk.Row, table *statistics.Table,
392394
nullCount := row.GetInt64(5)
393395
totColSize := row.GetInt64(6)
394396
correlation := row.GetFloat64(9)
397+
lastAnalyzePos := row.GetDatum(10, types.NewFieldType(mysql.TypeBlob))
395398
col := table.Columns[histID]
396399
errorRate := statistics.ErrorRate{}
397400
flag := row.GetInt64(8)
@@ -429,7 +432,7 @@ func (h *Handle) columnStatsFromStorage(row chunk.Row, table *statistics.Table,
429432
ErrorRate: errorRate,
430433
IsHandle: tableInfo.PKIsHandle && mysql.HasPriKeyFlag(colInfo.Flag),
431434
Flag: flag,
432-
LastAnalyzePos: row.GetDatum(10, types.NewFieldType(mysql.TypeBlob)),
435+
LastAnalyzePos: *lastAnalyzePos.Copy(),
433436
}
434437
col.Histogram.Correlation = correlation
435438
break
@@ -452,7 +455,7 @@ func (h *Handle) columnStatsFromStorage(row chunk.Row, table *statistics.Table,
452455
ErrorRate: errorRate,
453456
IsHandle: tableInfo.PKIsHandle && mysql.HasPriKeyFlag(colInfo.Flag),
454457
Flag: flag,
455-
LastAnalyzePos: row.GetDatum(10, types.NewFieldType(mysql.TypeBlob)),
458+
LastAnalyzePos: *lastAnalyzePos.Copy(),
456459
}
457460
break
458461
}

‎statistics/handle/handle_test.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -371,7 +371,7 @@ func (s *testStatsSuite) TestInitStats(c *C) {
371371
testKit := testkit.NewTestKit(c, s.store)
372372
testKit.MustExec("use test")
373373
testKit.MustExec("create table t(a int, b int, c int, primary key(a), key idx(b))")
374-
testKit.MustExec("insert into t values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6)")
374+
testKit.MustExec("insert into t values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,7,8)")
375375
testKit.MustExec("analyze table t")
376376
h := s.do.StatsHandle()
377377
is := s.do.InfoSchema()
@@ -384,6 +384,10 @@ func (s *testStatsSuite) TestInitStats(c *C) {
384384
h.Clear()
385385
c.Assert(h.InitStats(is), IsNil)
386386
table0 := h.GetTableStats(tbl.Meta())
387+
cols := table0.Columns
388+
c.Assert(cols[1].LastAnalyzePos.GetBytes()[0], Equals, uint8(0x36))
389+
c.Assert(cols[2].LastAnalyzePos.GetBytes()[0], Equals, uint8(0x37))
390+
c.Assert(cols[3].LastAnalyzePos.GetBytes()[0], Equals, uint8(0x38))
387391
h.Clear()
388392
c.Assert(h.Update(is), IsNil)
389393
table1 := h.GetTableStats(tbl.Meta())

0 commit comments

Comments
 (0)