From 02c86ef59c5ffd65658bde64ffb580b3d41e7373 Mon Sep 17 00:00:00 2001 From: ti-srebot <66930949+ti-srebot@users.noreply.github.com> Date: Mon, 6 Sep 2021 18:30:58 +0800 Subject: [PATCH 1/2] expression: do not derive filters containing null sensitive functions from outer join (#27067) (#27193) --- expression/constant_propagation.go | 2 +- expression/integration_test.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/expression/constant_propagation.go b/expression/constant_propagation.go index 7c521d7ea37ae..b383168d5fcf4 100644 --- a/expression/constant_propagation.go +++ b/expression/constant_propagation.go @@ -146,7 +146,7 @@ func tryToReplaceCond(ctx sessionctx.Context, src *Column, tgt *Column, cond Exp sf.FuncName.L == ast.If || sf.FuncName.L == ast.Case || sf.FuncName.L == ast.NullEQ) { - return false, false, cond + return false, true, cond } for idx, expr := range sf.GetArgs() { if src.Equal(nil, expr) { diff --git a/expression/integration_test.go b/expression/integration_test.go index 021745a0dfbea..83b84ab890304 100755 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -8251,3 +8251,21 @@ func (s *testIntegrationSuite) TestJiraSetInnoDBDefaultRowFormat(c *C) { tk.MustQuery("SHOW VARIABLES LIKE 'innodb_file_format'").Check(testkit.Rows("innodb_file_format Barracuda")) tk.MustQuery("SHOW VARIABLES LIKE 'innodb_large_prefix'").Check(testkit.Rows("innodb_large_prefix ON")) } + +func (s *testIntegrationSuite) TestConstPropNullFunctions(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("create table t1 (a integer)") + tk.MustExec("insert into t1 values (0), (1), (2), (3)") + tk.MustExec("create table t2 (a integer, b integer)") + tk.MustExec("insert into t2 values (0,1), (1,1), (2,1), (3,1)") + tk.MustQuery("select t1.* from t1 left join t2 on t2.a = t1.a where t1.a = ifnull(t2.b, 0)").Check(testkit.Rows("1")) + + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("create table t1 (i1 integer, c1 char)") + tk.MustExec("insert into t1 values (2, 'a'), (1, 'b'), (3, 'c'), (0, null);") + tk.MustExec("create table t2 (i2 integer, c2 char, f2 float)") + tk.MustExec("insert into t2 values (0, 'c', null), (1, null, 0.1), (3, 'b', 0.01), (2, 'q', 0.12), (null, 'a', -0.1), (null, null, null)") + tk.MustQuery("select * from t2 where t2.i2=((select count(1) from t1 where t1.i1=t2.i2))").Check(testkit.Rows("1 0.1")) +} From 1881af706e2ac07117b717feebd7c4ea03c888d7 Mon Sep 17 00:00:00 2001 From: ti-srebot <66930949+ti-srebot@users.noreply.github.com> Date: Wed, 8 Sep 2021 19:32:58 +0800 Subject: [PATCH 2/2] executor: make `group_concat` function consider the collation (#27490) (#27835) --- executor/aggfuncs/func_group_concat.go | 21 ++++++++++++++++----- executor/analyze_test.go | 13 +++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/executor/aggfuncs/func_group_concat.go b/executor/aggfuncs/func_group_concat.go index c2d3b745d12fc..7d892adb54a5c 100644 --- a/executor/aggfuncs/func_group_concat.go +++ b/executor/aggfuncs/func_group_concat.go @@ -26,8 +26,8 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/codec" + "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/dbterror" - "github.com/pingcap/tidb/util/hack" "github.com/pingcap/tidb/util/set" ) @@ -175,10 +175,16 @@ func (e *groupConcatDistinct) ResetPartialResult(pr PartialResult) { func (e *groupConcatDistinct) UpdatePartialResult(sctx sessionctx.Context, rowsInGroup []chunk.Row, pr PartialResult) (err error) { p := (*partialResult4GroupConcatDistinct)(pr) v, isNull := "", false + + collators := make([]collate.Collator, 0, len(e.args)) + for _, arg := range e.args { + collators = append(collators, collate.GetCollator(arg.GetType().Collate)) + } + for _, row := range rowsInGroup { p.valsBuf.Reset() p.encodeBytesBuffer = p.encodeBytesBuffer[:0] - for _, arg := range e.args { + for i, arg := range e.args { v, isNull, err = arg.EvalString(sctx, row) if err != nil { return err @@ -186,7 +192,7 @@ func (e *groupConcatDistinct) UpdatePartialResult(sctx sessionctx.Context, rowsI if isNull { break } - p.encodeBytesBuffer = codec.EncodeBytes(p.encodeBytesBuffer, hack.Slice(v)) + p.encodeBytesBuffer = codec.EncodeBytes(p.encodeBytesBuffer, collators[i].Key(v)) p.valsBuf.WriteString(v) } if isNull { @@ -478,10 +484,15 @@ func (e *groupConcatDistinctOrder) UpdatePartialResult(sctx sessionctx.Context, p := (*partialResult4GroupConcatOrderDistinct)(pr) p.topN.sctx = sctx v, isNull := "", false + collators := make([]collate.Collator, 0, len(e.args)) + for _, arg := range e.args { + collators = append(collators, collate.GetCollator(arg.GetType().Collate)) + } + for _, row := range rowsInGroup { buffer := new(bytes.Buffer) p.encodeBytesBuffer = p.encodeBytesBuffer[:0] - for _, arg := range e.args { + for i, arg := range e.args { v, isNull, err = arg.EvalString(sctx, row) if err != nil { return err @@ -489,7 +500,7 @@ func (e *groupConcatDistinctOrder) UpdatePartialResult(sctx sessionctx.Context, if isNull { break } - p.encodeBytesBuffer = codec.EncodeBytes(p.encodeBytesBuffer, hack.Slice(v)) + p.encodeBytesBuffer = codec.EncodeBytes(p.encodeBytesBuffer, collators[i].Key(v)) buffer.WriteString(v) } if isNull { diff --git a/executor/analyze_test.go b/executor/analyze_test.go index 764b1c38ce692..4573394690d36 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -648,6 +648,19 @@ func (s *testSuite1) TestDefaultValForAnalyze(c *C) { "└─IndexRangeScan_5 1.00 cop[tikv] table:t, index:a(a) range:[1,1], keep order:false")) } +func (s *testSerialSuite2) TestIssue27429(c *C) { + collate.SetNewCollationEnabledForTest(true) + defer collate.SetNewCollationEnabledForTest(false) + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table test.t(id int, value varchar(20) charset utf8mb4 collate utf8mb4_general_ci, value1 varchar(20) charset utf8mb4 collate utf8mb4_bin)") + tk.MustExec("insert into test.t values (1, 'abc', 'abc '),(4, 'Abc', 'abc'),(3,'def', 'def ');") + + tk.MustQuery("select upper(group_concat(distinct value order by 1)) from test.t;").Check(testkit.Rows("ABC,DEF")) + tk.MustQuery("select upper(group_concat(distinct value)) from test.t;").Check(testkit.Rows("ABC,DEF")) +} + func (s *testSerialSuite2) TestIssue20874(c *C) { collate.SetNewCollationEnabledForTest(true) defer collate.SetNewCollationEnabledForTest(false)