diff --git a/Makefile b/Makefile index d09ed06f2c528..3b3c42b9332e8 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ dev: checklist check test # Install the check tools. check-setup:tools/bin/revive tools/bin/goword tools/bin/gometalinter tools/bin/gosec -check: fmt errcheck unconvert lint tidy testSuite check-static vet staticcheck errdoc +check: fmt errcheck unconvert lint tidy testSuite check-static vet errdoc # These need to be fixed before they can be ran regularly check-fail: goword check-slow @@ -89,6 +89,7 @@ vet: @echo "vet" $(GO) vet -all $(PACKAGES) 2>&1 | $(FAIL_ON_STDOUT) +# staticcheck seems not introduced in 5.2/5.3 and blocked the ci staticcheck: $(GO) get honnef.co/go/tools/cmd/staticcheck $(STATICCHECK) ./... diff --git a/cmd/explaintest/r/collation_agg_func.result b/cmd/explaintest/r/collation_agg_func.result new file mode 100644 index 0000000000000..75ba58783482b --- /dev/null +++ b/cmd/explaintest/r/collation_agg_func.result @@ -0,0 +1,302 @@ +create database collation_agg_func; +use collation_agg_func; +create table t(id int, value varchar(20) charset utf8mb4 collate utf8mb4_general_ci, value1 varchar(20) charset utf8mb4 collate utf8mb4_bin); +insert into t values (1, 'abc', 'abc '),(4, 'Abc', 'abc'),(3,'def', 'def '), (5, 'abc', 'ABC'); +desc format='brief' select group_concat(value order by 1) from t; +id estRows task access object operator info +HashAgg 1.00 root funcs:group_concat(collation_agg_func.t.value order by collation_agg_func.t.value separator ",")->Column#5 +└─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select group_concat(value order by 1) from t; +group_concat(value order by 1) +Abc,abc,abc,def +desc format='brief' select group_concat(value) from t; +id estRows task access object operator info +HashAgg 1.00 root funcs:group_concat(collation_agg_func.t.value separator ",")->Column#5 +└─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select group_concat(value) from t; +group_concat(value) +abc,Abc,def,abc +desc format='brief' select group_concat(value collate utf8mb4_bin) from t; +id estRows task access object operator info +HashAgg 1.00 root funcs:group_concat(Column#6 separator ",")->Column#5 +└─Projection 10000.00 root cast(collation_agg_func.t.value, varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#6 + └─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select group_concat(value collate utf8mb4_bin) from t; +group_concat(value collate utf8mb4_bin) +abc,Abc,def,abc +desc format='brief' select group_concat(distinct value order by 1) from t; +id estRows task access object operator info +StreamAgg 1.00 root funcs:group_concat(distinct collation_agg_func.t.value order by collation_agg_func.t.value separator ",")->Column#5 +└─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select upper(group_concat(distinct value order by 1)) from t; +upper(group_concat(distinct value order by 1)) +ABC,ABC,DEF +desc format='brief' select group_concat(distinct value collate utf8mb4_bin order by 1) from t; +id estRows task access object operator info +StreamAgg 1.00 root funcs:group_concat(distinct Column#6 order by Column#7 separator ",")->Column#5 +└─Projection 10000.00 root cast(collation_agg_func.t.value, varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#6, cast(collation_agg_func.t.value, varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#7 + └─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select upper(group_concat(distinct value collate utf8mb4_bin order by 1)) from t; +upper(group_concat(distinct value collate utf8mb4_bin order by 1)) +ABC,ABC,DEF +desc format='brief' select group_concat(distinct value) from t; +id estRows task access object operator info +StreamAgg 1.00 root funcs:group_concat(distinct collation_agg_func.t.value separator ",")->Column#5 +└─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select upper(group_concat(distinct value)) from t; +upper(group_concat(distinct value)) +ABC,ABC,DEF +desc format='brief' select group_concat(distinct value collate utf8mb4_bin) from t; +id estRows task access object operator info +StreamAgg 1.00 root funcs:group_concat(distinct Column#6 separator ",")->Column#5 +└─Projection 10000.00 root cast(collation_agg_func.t.value, varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#6 + └─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select upper(group_concat(distinct value collate utf8mb4_bin)) from t; +upper(group_concat(distinct value collate utf8mb4_bin)) +ABC,ABC,DEF +desc format='brief' select count(distinct value) from t; +id estRows task access object operator info +StreamAgg 1.00 root funcs:count(distinct collation_agg_func.t.value)->Column#5 +└─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select count(distinct value) from t; +count(distinct value) +3 +desc format='brief' select count(distinct value collate utf8mb4_bin) from t; +id estRows task access object operator info +StreamAgg 1.00 root funcs:count(distinct Column#6)->Column#5 +└─Projection 10000.00 root cast(collation_agg_func.t.value, varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#6 + └─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select count(distinct value collate utf8mb4_bin) from t; +count(distinct value collate utf8mb4_bin) +3 +desc format='brief' select count(distinct value, value1) from t; +id estRows task access object operator info +StreamAgg 1.00 root funcs:count(distinct collation_agg_func.t.value, collation_agg_func.t.value1)->Column#5 +└─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select count(distinct value, value1) from t; +count(distinct value, value1) +4 +desc format='brief' select count(distinct value collate utf8mb4_bin, value1) from t; +id estRows task access object operator info +StreamAgg 1.00 root funcs:count(distinct Column#6, Column#7)->Column#5 +└─Projection 10000.00 root cast(collation_agg_func.t.value, varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#6, collation_agg_func.t.value1 + └─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select count(distinct value collate utf8mb4_bin, value1) from t; +count(distinct value collate utf8mb4_bin, value1) +4 +desc format='brief' select approx_count_distinct(value) from t; +id estRows task access object operator info +HashAgg 1.00 root funcs:approx_count_distinct(collation_agg_func.t.value)->Column#5 +└─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select approx_count_distinct(value) from t; +approx_count_distinct(value) +3 +desc format='brief' select approx_count_distinct(value collate utf8mb4_bin) from t; +id estRows task access object operator info +HashAgg 1.00 root funcs:approx_count_distinct(Column#6)->Column#5 +└─Projection 10000.00 root cast(collation_agg_func.t.value, varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#6 + └─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select approx_count_distinct(value collate utf8mb4_bin) from t; +approx_count_distinct(value collate utf8mb4_bin) +3 +desc format='brief' select approx_count_distinct(value, value1) from t; +id estRows task access object operator info +HashAgg 1.00 root funcs:approx_count_distinct(collation_agg_func.t.value, collation_agg_func.t.value1)->Column#5 +└─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select approx_count_distinct(value, value1) from t; +approx_count_distinct(value, value1) +4 +desc format='brief' select approx_count_distinct(value collate utf8mb4_bin, value1) from t; +id estRows task access object operator info +HashAgg 1.00 root funcs:approx_count_distinct(Column#6, Column#7)->Column#5 +└─Projection 10000.00 root cast(collation_agg_func.t.value, varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#6, collation_agg_func.t.value1 + └─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select approx_count_distinct(value collate utf8mb4_bin, value1) from t; +approx_count_distinct(value collate utf8mb4_bin, value1) +4 +create table tt(a char(10), b enum('a', 'B', 'c'), c set('a', 'B', 'c'), d json) collate utf8mb4_general_ci; +insert into tt values ("a", "a", "a", JSON_OBJECT("a", "a")); +insert into tt values ("A", "A", "A", JSON_OBJECT("A", "A")); +Error 1265: Data truncated for column 'b' at row 1 +insert into tt values ("b", "b", "b", JSON_OBJECT("b", "b")); +Error 1265: Data truncated for column 'b' at row 1 +insert into tt values ("B", "B", "B", JSON_OBJECT("B", "B")); +insert into tt values ("c", "c", "c", JSON_OBJECT("c", "c")); +insert into tt values ("C", "C", "C", JSON_OBJECT("C", "C")); +Error 1265: Data truncated for column 'b' at row 1 +split table tt by (0), (1), (2), (3), (4), (5); +desc format='brief' select min(a) from tt; +id estRows task access object operator info +StreamAgg 1.00 root funcs:min(collation_agg_func.tt.a)->Column#6 +└─TopN 1.00 root collation_agg_func.tt.a, offset:0, count:1 + └─TableReader 1.00 root data:TopN + └─TopN 1.00 cop[tikv] collation_agg_func.tt.a, offset:0, count:1 + └─Selection 9990.00 cop[tikv] not(isnull(collation_agg_func.tt.a)) + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +select min(a) from tt; +min(a) +B +desc format='brief' select min(a collate utf8mb4_bin) from tt; +id estRows task access object operator info +StreamAgg 1.00 root funcs:min(Column#8)->Column#6 +└─Projection 1.00 root cast(collation_agg_func.tt.a, char(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#8 + └─Projection 1.00 root collation_agg_func.tt.a + └─TopN 1.00 root Column#7, offset:0, count:1 + └─Projection 1.00 root collation_agg_func.tt.a, cast(collation_agg_func.tt.a, char(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#7 + └─TableReader 1.00 root data:TopN + └─TopN 1.00 cop[tikv] cast(collation_agg_func.tt.a, char(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin), offset:0, count:1 + └─Selection 8000.00 cop[tikv] not(isnull(cast(collation_agg_func.tt.a, char(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin))) + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +select min(a collate utf8mb4_bin) from tt; +min(a collate utf8mb4_bin) +B +desc format='brief' select max(a) from tt; +id estRows task access object operator info +StreamAgg 1.00 root funcs:max(collation_agg_func.tt.a)->Column#6 +└─TopN 1.00 root collation_agg_func.tt.a:desc, offset:0, count:1 + └─TableReader 1.00 root data:TopN + └─TopN 1.00 cop[tikv] collation_agg_func.tt.a:desc, offset:0, count:1 + └─Selection 9990.00 cop[tikv] not(isnull(collation_agg_func.tt.a)) + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +select max(a) from tt; +max(a) +c +desc format='brief' select max(a collate utf8mb4_bin) from tt; +id estRows task access object operator info +StreamAgg 1.00 root funcs:max(Column#8)->Column#6 +└─Projection 1.00 root cast(collation_agg_func.tt.a, char(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#8 + └─Projection 1.00 root collation_agg_func.tt.a + └─TopN 1.00 root Column#7:desc, offset:0, count:1 + └─Projection 1.00 root collation_agg_func.tt.a, cast(collation_agg_func.tt.a, char(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#7 + └─TableReader 1.00 root data:TopN + └─TopN 1.00 cop[tikv] cast(collation_agg_func.tt.a, char(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin):desc, offset:0, count:1 + └─Selection 8000.00 cop[tikv] not(isnull(cast(collation_agg_func.tt.a, char(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin))) + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +select max(a collate utf8mb4_bin) from tt; +max(a collate utf8mb4_bin) +c +desc format='brief' select min(b) from tt; +id estRows task access object operator info +StreamAgg 1.00 root funcs:min(Column#8)->Column#6 +└─TableReader 1.00 root data:StreamAgg + └─StreamAgg 1.00 cop[tikv] funcs:min(collation_agg_func.tt.b)->Column#8 + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +select min(b) from tt; +min(b) +B +desc format='brief' select min(b collate utf8mb4_bin) from tt; +id estRows task access object operator info +StreamAgg 1.00 root funcs:min(Column#8)->Column#6 +└─TableReader 1.00 root data:StreamAgg + └─StreamAgg 1.00 cop[tikv] funcs:min(cast(collation_agg_func.tt.b, enum('a','B','c')))->Column#8 + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +desc format='brief' select max(b) from tt; +id estRows task access object operator info +StreamAgg 1.00 root funcs:max(Column#8)->Column#6 +└─TableReader 1.00 root data:StreamAgg + └─StreamAgg 1.00 cop[tikv] funcs:max(collation_agg_func.tt.b)->Column#8 + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +select max(b) from tt; +max(b) +c +desc format='brief' select max(b collate utf8mb4_bin) from tt; +id estRows task access object operator info +StreamAgg 1.00 root funcs:max(Column#8)->Column#6 +└─TableReader 1.00 root data:StreamAgg + └─StreamAgg 1.00 cop[tikv] funcs:max(cast(collation_agg_func.tt.b, enum('a','B','c')))->Column#8 + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +desc format='brief' select min(c) from tt; +id estRows task access object operator info +HashAgg 1.00 root funcs:min(collation_agg_func.tt.c)->Column#6 +└─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +select min(c) from tt; +min(c) +B +desc format='brief' select min(c collate utf8mb4_bin) from tt; +id estRows task access object operator info +HashAgg 1.00 root funcs:min(Column#7)->Column#6 +└─Projection 10000.00 root cast(collation_agg_func.tt.c, set('a','B','c'))->Column#7 + └─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +desc format='brief' select max(c) from tt; +id estRows task access object operator info +HashAgg 1.00 root funcs:max(collation_agg_func.tt.c)->Column#6 +└─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +select max(c) from tt; +max(c) +c +desc format='brief' select max(c collate utf8mb4_bin) from tt; +id estRows task access object operator info +HashAgg 1.00 root funcs:max(Column#7)->Column#6 +└─Projection 10000.00 root cast(collation_agg_func.tt.c, set('a','B','c'))->Column#7 + └─TableReader 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +desc format='brief' select min(d) from tt; +id estRows task access object operator info +StreamAgg 1.00 root funcs:min(collation_agg_func.tt.d)->Column#6 +└─TopN 1.00 root collation_agg_func.tt.d, offset:0, count:1 + └─TableReader 1.00 root data:TopN + └─TopN 1.00 cop[tikv] collation_agg_func.tt.d, offset:0, count:1 + └─Selection 8000.00 cop[tikv] not(isnull(cast(collation_agg_func.tt.d, var_string(4294967295)))) + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +select min(d) from tt; +min(d) +{"B": "B"} +desc format='brief' select min(d collate utf8mb4_bin) from tt; +id estRows task access object operator info +StreamAgg 1.00 root funcs:min(Column#8)->Column#6 +└─Projection 1.00 root cast(collation_agg_func.tt.d, json BINARY)->Column#8 + └─Projection 1.00 root collation_agg_func.tt.d + └─TopN 1.00 root Column#7, offset:0, count:1 + └─Projection 1.00 root collation_agg_func.tt.d, cast(collation_agg_func.tt.d, json BINARY)->Column#7 + └─TableReader 1.00 root data:TopN + └─TopN 1.00 cop[tikv] cast(collation_agg_func.tt.d, json BINARY), offset:0, count:1 + └─Selection 8000.00 cop[tikv] not(isnull(cast(cast(collation_agg_func.tt.d, json BINARY), var_string(4294967295)))) + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +select min(d collate utf8mb4_bin) from tt; +min(d collate utf8mb4_bin) +{"B": "B"} +desc format='brief' select max(d) from tt; +id estRows task access object operator info +StreamAgg 1.00 root funcs:max(collation_agg_func.tt.d)->Column#6 +└─TopN 1.00 root collation_agg_func.tt.d:desc, offset:0, count:1 + └─TableReader 1.00 root data:TopN + └─TopN 1.00 cop[tikv] collation_agg_func.tt.d:desc, offset:0, count:1 + └─Selection 8000.00 cop[tikv] not(isnull(cast(collation_agg_func.tt.d, var_string(4294967295)))) + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +select max(d) from tt; +max(d) +{"c": "c"} +desc format='brief' select max(d collate utf8mb4_bin) from tt; +id estRows task access object operator info +StreamAgg 1.00 root funcs:max(Column#8)->Column#6 +└─Projection 1.00 root cast(collation_agg_func.tt.d, json BINARY)->Column#8 + └─Projection 1.00 root collation_agg_func.tt.d + └─TopN 1.00 root Column#7:desc, offset:0, count:1 + └─Projection 1.00 root collation_agg_func.tt.d, cast(collation_agg_func.tt.d, json BINARY)->Column#7 + └─TableReader 1.00 root data:TopN + └─TopN 1.00 cop[tikv] cast(collation_agg_func.tt.d, json BINARY):desc, offset:0, count:1 + └─Selection 8000.00 cop[tikv] not(isnull(cast(cast(collation_agg_func.tt.d, json BINARY), var_string(4294967295)))) + └─TableFullScan 10000.00 cop[tikv] table:tt keep order:false, stats:pseudo +select max(d collate utf8mb4_bin) from tt; +max(d collate utf8mb4_bin) +{"c": "c"} +drop database collation_agg_func; +use test diff --git a/cmd/explaintest/r/subquery.result b/cmd/explaintest/r/subquery.result index 84bac87bb1d23..2bf71b91088cb 100644 --- a/cmd/explaintest/r/subquery.result +++ b/cmd/explaintest/r/subquery.result @@ -22,9 +22,9 @@ Projection 5.00 root Column#22 ├─TableReader(Build) 5.00 root data:TableFullScan │ └─TableFullScan 5.00 cop[tikv] table:t keep order:false └─StreamAgg(Probe) 1.00 root funcs:count(1)->Column#21 - └─IndexJoin 0.22 root inner join, inner:TableReader, outer key:test.t.a, inner key:test.t.a, equal cond:eq(test.t.a, test.t.a) - ├─IndexReader(Build) 0.45 root index:IndexRangeScan - │ └─IndexRangeScan 0.45 cop[tikv] table:s, index:idx(b, c, d) range: decided by [eq(test.t.b, 1) eq(test.t.c, 1) eq(test.t.d, test.t.a)], keep order:false + └─IndexJoin 0.50 root inner join, inner:TableReader, outer key:test.t.a, inner key:test.t.a, equal cond:eq(test.t.a, test.t.a) + ├─IndexReader(Build) 1.00 root index:IndexRangeScan + │ └─IndexRangeScan 1.00 cop[tikv] table:s, index:idx(b, c, d) range: decided by [eq(test.t.b, 1) eq(test.t.c, 1) eq(test.t.d, test.t.a)], keep order:false └─TableReader(Probe) 1.00 root data:TableRangeScan └─TableRangeScan 1.00 cop[tikv] table:t1 range: decided by [test.t.a], keep order:false drop table if exists t; diff --git a/cmd/explaintest/t/collation_agg_func.test b/cmd/explaintest/t/collation_agg_func.test new file mode 100644 index 0000000000000..eb7ada2209981 --- /dev/null +++ b/cmd/explaintest/t/collation_agg_func.test @@ -0,0 +1,102 @@ +# These tests test the aggregate function's behavior according to collation. +# The result of min/max of enum/set is wrong, please fix them soon. + +# prepare database +create database collation_agg_func; +use collation_agg_func; + +create table t(id int, value varchar(20) charset utf8mb4 collate utf8mb4_general_ci, value1 varchar(20) charset utf8mb4 collate utf8mb4_bin); +insert into t values (1, 'abc', 'abc '),(4, 'Abc', 'abc'),(3,'def', 'def '), (5, 'abc', 'ABC'); + +# group_concat +desc format='brief' select group_concat(value order by 1) from t; +select group_concat(value order by 1) from t; +desc format='brief' select group_concat(value) from t; +select group_concat(value) from t; +desc format='brief' select group_concat(value collate utf8mb4_bin) from t; +select group_concat(value collate utf8mb4_bin) from t; +desc format='brief' select group_concat(distinct value order by 1) from t; +select upper(group_concat(distinct value order by 1)) from t; +desc format='brief' select group_concat(distinct value collate utf8mb4_bin order by 1) from t; +select upper(group_concat(distinct value collate utf8mb4_bin order by 1)) from t; +desc format='brief' select group_concat(distinct value) from t; +select upper(group_concat(distinct value)) from t; +desc format='brief' select group_concat(distinct value collate utf8mb4_bin) from t; +select upper(group_concat(distinct value collate utf8mb4_bin)) from t; + +# count(distinct) +desc format='brief' select count(distinct value) from t; +select count(distinct value) from t; +desc format='brief' select count(distinct value collate utf8mb4_bin) from t; +select count(distinct value collate utf8mb4_bin) from t; +desc format='brief' select count(distinct value, value1) from t; +select count(distinct value, value1) from t; +desc format='brief' select count(distinct value collate utf8mb4_bin, value1) from t; +select count(distinct value collate utf8mb4_bin, value1) from t; + +# approxCountDistinct +desc format='brief' select approx_count_distinct(value) from t; +select approx_count_distinct(value) from t; +desc format='brief' select approx_count_distinct(value collate utf8mb4_bin) from t; +select approx_count_distinct(value collate utf8mb4_bin) from t; +desc format='brief' select approx_count_distinct(value, value1) from t; +select approx_count_distinct(value, value1) from t; +desc format='brief' select approx_count_distinct(value collate utf8mb4_bin, value1) from t; +select approx_count_distinct(value collate utf8mb4_bin, value1) from t; + +# minMax +create table tt(a char(10), b enum('a', 'B', 'c'), c set('a', 'B', 'c'), d json) collate utf8mb4_general_ci; +insert into tt values ("a", "a", "a", JSON_OBJECT("a", "a")); +--error 1265 +insert into tt values ("A", "A", "A", JSON_OBJECT("A", "A")); +--error 1265 +insert into tt values ("b", "b", "b", JSON_OBJECT("b", "b")); +insert into tt values ("B", "B", "B", JSON_OBJECT("B", "B")); +insert into tt values ("c", "c", "c", JSON_OBJECT("c", "c")); +--error 1265 +insert into tt values ("C", "C", "C", JSON_OBJECT("C", "C")); +split table tt by (0), (1), (2), (3), (4), (5); +desc format='brief' select min(a) from tt; +select min(a) from tt; +desc format='brief' select min(a collate utf8mb4_bin) from tt; +select min(a collate utf8mb4_bin) from tt; +desc format='brief' select max(a) from tt; +select max(a) from tt; +desc format='brief' select max(a collate utf8mb4_bin) from tt; +select max(a collate utf8mb4_bin) from tt; +desc format='brief' select min(b) from tt; +select min(b) from tt; +desc format='brief' select min(b collate utf8mb4_bin) from tt; +# Fix me later. +# select min(b collate utf8mb4_bin) from tt; +desc format='brief' select max(b) from tt; +select max(b) from tt; +desc format='brief' select max(b collate utf8mb4_bin) from tt; +# Fix me later. +# select max(b collate utf8mb4_bin) from tt; +desc format='brief' select min(c) from tt; +select min(c) from tt; +desc format='brief' select min(c collate utf8mb4_bin) from tt; +# Fix me later. +# select min(c collate utf8mb4_bin) from tt; +desc format='brief' select max(c) from tt; +select max(c) from tt; +desc format='brief' select max(c collate utf8mb4_bin) from tt; +# Fix me later. +# select max(c collate utf8mb4_bin) from tt; +desc format='brief' select min(d) from tt; +select min(d) from tt; +--error 1253 +desc format='brief' select min(d collate utf8mb4_bin) from tt; +--error 1253 +select min(d collate utf8mb4_bin) from tt; +desc format='brief' select max(d) from tt; +select max(d) from tt; +--error 1253 +desc format='brief' select max(d collate utf8mb4_bin) from tt; +--error 1253 +select max(d collate utf8mb4_bin) from tt; + +# cleanup environment +drop database collation_agg_func; +use test diff --git a/ddl/backfilling.go b/ddl/backfilling.go index 8a5b7a837f552..c6f1b28c42f22 100644 --- a/ddl/backfilling.go +++ b/ddl/backfilling.go @@ -423,6 +423,13 @@ func (w *worker) handleReorgTasks(reorgInfo *reorgInfo, totalAddedCount *int64, } func tryDecodeToHandleString(key kv.Key) string { + defer func() { + if r := recover(); r != nil { + logutil.BgLogger().Warn("tryDecodeToHandleString panic", + zap.Any("recover()", r), + zap.Binary("key", key)) + } + }() handle, err := tablecodec.DecodeRowKey(key) if err != nil { recordPrefixIdx := bytes.Index(key, []byte("_r")) diff --git a/ddl/column.go b/ddl/column.go index 082962ec6baa5..d58f5cd5ae44e 100644 --- a/ddl/column.go +++ b/ddl/column.go @@ -840,7 +840,7 @@ func (w *worker) onModifyColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in if job.IsRollingback() { // For those column-type-change jobs which don't reorg the data. if !needChangeColumnData(oldCol, jobParam.newCol) { - return rollbackModifyColumnJob(t, tblInfo, job, oldCol, jobParam.modifyColumnTp) + return rollbackModifyColumnJob(t, tblInfo, job, jobParam.newCol, oldCol, jobParam.modifyColumnTp) } // For those column-type-change jobs which reorg the data. return rollbackModifyColumnJobWithData(t, tblInfo, job, oldCol, jobParam) @@ -1454,6 +1454,10 @@ func updateChangingInfo(changingCol *model.ColumnInfo, changingIdxs []*model.Ind func (w *worker) doModifyColumn( d *ddlCtx, t *meta.Meta, job *model.Job, dbInfo *model.DBInfo, tblInfo *model.TableInfo, newCol, oldCol *model.ColumnInfo, pos *ast.ColumnPosition) (ver int64, _ error) { + if oldCol.ID != newCol.ID { + job.State = model.JobStateRollingback + return ver, errKeyColumnDoesNotExits.GenWithStack("column %s id %d does not exist, this column may have been updated by other DDL ran in parallel", oldCol.Name, newCol.ID) + } // Column from null to not null. if !mysql.HasNotNullFlag(oldCol.Flag) && mysql.HasNotNullFlag(newCol.Flag) { noPreventNullFlag := !mysql.HasPreventNullInsertFlag(oldCol.Flag) @@ -1762,9 +1766,9 @@ func checkAddColumnTooManyColumns(colNum int) error { } // rollbackModifyColumnJob rollbacks the job when an error occurs. -func rollbackModifyColumnJob(t *meta.Meta, tblInfo *model.TableInfo, job *model.Job, oldCol *model.ColumnInfo, modifyColumnTp byte) (ver int64, _ error) { +func rollbackModifyColumnJob(t *meta.Meta, tblInfo *model.TableInfo, job *model.Job, newCol, oldCol *model.ColumnInfo, modifyColumnTp byte) (ver int64, _ error) { var err error - if modifyColumnTp == mysql.TypeNull { + if oldCol.ID == newCol.ID && modifyColumnTp == mysql.TypeNull { // field NotNullFlag flag reset. tblInfo.Columns[oldCol.Offset].Flag = oldCol.Flag &^ mysql.NotNullFlag // field PreventNullInsertFlag flag reset. diff --git a/ddl/db_change_test.go b/ddl/db_change_test.go index a0534c4a505c4..786f31bf27681 100644 --- a/ddl/db_change_test.go +++ b/ddl/db_change_test.go @@ -1046,6 +1046,54 @@ func (s *testStateChangeSuite) TestParallelAlterModifyColumn(c *C) { s.testControlParallelExecSQL(c, sql, sql, f) } +func (s *testStateChangeSuite) TestParallelAlterModifyColumnWithData(c *C) { + sql := "ALTER TABLE t MODIFY COLUMN c int;" + f := func(c *C, err1, err2 error) { + c.Assert(err1, IsNil) + c.Assert(err2.Error(), Equals, "[ddl:1072]column c id 3 does not exist, this column may have been updated by other DDL ran in parallel") + rs, err := s.se.Execute(context.Background(), "select * from t") + c.Assert(err, IsNil) + sRows, err := session.ResultSetToStringSlice(context.Background(), s.se, rs[0]) + c.Assert(err, IsNil) + c.Assert(sRows[0][2], Equals, "3") + c.Assert(rs[0].Close(), IsNil) + _, err = s.se.Execute(context.Background(), "insert into t values(11, 22, 33.3, 44, 55)") + c.Assert(err, IsNil) + rs, err = s.se.Execute(context.Background(), "select * from t") + c.Assert(err, IsNil) + sRows, err = session.ResultSetToStringSlice(context.Background(), s.se, rs[0]) + c.Assert(err, IsNil) + c.Assert(sRows[1][2], Equals, "33") + c.Assert(rs[0].Close(), IsNil) + } + s.testControlParallelExecSQL(c, sql, sql, f) +} + +func (s *testStateChangeSuite) TestParallelAlterModifyColumnToNotNullWithData(c *C) { + sql := "ALTER TABLE t MODIFY COLUMN c int not null;" + f := func(c *C, err1, err2 error) { + c.Assert(err1, IsNil) + c.Assert(err2.Error(), Equals, "[ddl:1072]column c id 3 does not exist, this column may have been updated by other DDL ran in parallel") + rs, err := s.se.Execute(context.Background(), "select * from t") + c.Assert(err, IsNil) + sRows, err := session.ResultSetToStringSlice(context.Background(), s.se, rs[0]) + c.Assert(err, IsNil) + c.Assert(sRows[0][2], Equals, "3") + c.Assert(rs[0].Close(), IsNil) + _, err = s.se.Execute(context.Background(), "insert into t values(11, 22, null, 44, 55)") + c.Assert(err, NotNil) + _, err = s.se.Execute(context.Background(), "insert into t values(11, 22, 33.3, 44, 55)") + c.Assert(err, IsNil) + rs, err = s.se.Execute(context.Background(), "select * from t") + c.Assert(err, IsNil) + sRows, err = session.ResultSetToStringSlice(context.Background(), s.se, rs[0]) + c.Assert(err, IsNil) + c.Assert(sRows[1][2], Equals, "33") + c.Assert(rs[0].Close(), IsNil) + } + s.testControlParallelExecSQL(c, sql, sql, f) +} + func (s *testStateChangeSuite) TestParallelAddGeneratedColumnAndAlterModifyColumn(c *C) { sql1 := "ALTER TABLE t ADD COLUMN f INT GENERATED ALWAYS AS(a+1);" sql2 := "ALTER TABLE t MODIFY COLUMN a tinyint;" @@ -1322,12 +1370,15 @@ func (s *testStateChangeSuiteBase) prepareTestControlParallelExecSQL(c *C) (sess func (s *testStateChangeSuiteBase) testControlParallelExecSQL(c *C, sql1, sql2 string, f checkRet) { _, err := s.se.Execute(context.Background(), "use test_db_state") c.Assert(err, IsNil) - _, err = s.se.Execute(context.Background(), "create table t(a int, b int, c int, d int auto_increment,e int, index idx1(d), index idx2(d,e))") + _, err = s.se.Execute(context.Background(), "create table t(a int, b int, c double default null, d int auto_increment,e int, index idx1(d), index idx2(d,e))") c.Assert(err, IsNil) if len(s.preSQL) != 0 { _, err := s.se.Execute(context.Background(), s.preSQL) c.Assert(err, IsNil) } + _, err = s.se.Execute(context.Background(), "insert into t values(1, 2, 3.1234, 4, 5)") + c.Assert(err, IsNil) + defer func() { _, err := s.se.Execute(context.Background(), "drop table t") c.Assert(err, IsNil) diff --git a/ddl/serial_test.go b/ddl/serial_test.go index 915e9d66f8cef..833fe5bd59e05 100644 --- a/ddl/serial_test.go +++ b/ddl/serial_test.go @@ -1044,7 +1044,10 @@ func (s *testSerialSuite) TestTableLocksEnable(c *C) { }) tk.MustExec("lock tables t1 write") + tk.MustQuery("SHOW WARNINGS").Check(testkit.Rows("Warning 1235 LOCK TABLES is not supported. To enable this experimental feature, set 'enable-table-lock' in the configuration file.")) checkTableLock(c, tk.Se, "test", "t1", model.TableLockNone) + tk.MustExec("unlock tables") + tk.MustQuery("SHOW WARNINGS").Check(testkit.Rows("Warning 1235 UNLOCK TABLES is not supported. To enable this experimental feature, set 'enable-table-lock' in the configuration file.")) } func (s *testSerialDBSuite) TestAutoRandomOnTemporaryTable(c *C) { diff --git a/distsql/request_builder.go b/distsql/request_builder.go index ed97014d6154b..c0ad23066055e 100644 --- a/distsql/request_builder.go +++ b/distsql/request_builder.go @@ -17,6 +17,7 @@ import ( "fmt" "math" "sort" + "sync/atomic" "github.com/pingcap/errors" "github.com/pingcap/failpoint" @@ -553,13 +554,25 @@ func PartitionHandlesToKVRanges(handles []kv.Handle) []kv.KeyRange { // IndexRangesToKVRanges converts index ranges to "KeyRange". func IndexRangesToKVRanges(sc *stmtctx.StatementContext, tid, idxID int64, ranges []*ranger.Range, fb *statistics.QueryFeedback) ([]kv.KeyRange, error) { - return IndexRangesToKVRangesForTables(sc, []int64{tid}, idxID, ranges, fb) + return IndexRangesToKVRangesWithInterruptSignal(sc, tid, idxID, ranges, fb, nil, nil) +} + +// IndexRangesToKVRangesWithInterruptSignal converts index ranges to "KeyRange". +// The process can be interrupted by set `interruptSignal` to true. +func IndexRangesToKVRangesWithInterruptSignal(sc *stmtctx.StatementContext, tid, idxID int64, ranges []*ranger.Range, fb *statistics.QueryFeedback, memTracker *memory.Tracker, interruptSignal *atomic.Value) ([]kv.KeyRange, error) { + return indexRangesToKVRangesForTablesWithInterruptSignal(sc, []int64{tid}, idxID, ranges, fb, memTracker, interruptSignal) } // IndexRangesToKVRangesForTables converts indexes ranges to "KeyRange". func IndexRangesToKVRangesForTables(sc *stmtctx.StatementContext, tids []int64, idxID int64, ranges []*ranger.Range, fb *statistics.QueryFeedback) ([]kv.KeyRange, error) { + return indexRangesToKVRangesForTablesWithInterruptSignal(sc, tids, idxID, ranges, fb, nil, nil) +} + +// IndexRangesToKVRangesForTablesWithInterruptSignal converts indexes ranges to "KeyRange". +// The process can be interrupted by set `interruptSignal` to true. +func indexRangesToKVRangesForTablesWithInterruptSignal(sc *stmtctx.StatementContext, tids []int64, idxID int64, ranges []*ranger.Range, fb *statistics.QueryFeedback, memTracker *memory.Tracker, interruptSignal *atomic.Value) ([]kv.KeyRange, error) { if fb == nil || fb.Hist == nil { - return indexRangesToKVWithoutSplit(sc, tids, idxID, ranges) + return indexRangesToKVWithoutSplit(sc, tids, idxID, ranges, memTracker, interruptSignal) } feedbackRanges := make([]*ranger.Range, 0, len(ranges)) for _, ran := range ranges { @@ -644,18 +657,37 @@ func VerifyTxnScope(txnScope string, physicalTableID int64, is infoschema.InfoSc return true } -func indexRangesToKVWithoutSplit(sc *stmtctx.StatementContext, tids []int64, idxID int64, ranges []*ranger.Range) ([]kv.KeyRange, error) { +func indexRangesToKVWithoutSplit(sc *stmtctx.StatementContext, tids []int64, idxID int64, ranges []*ranger.Range, memTracker *memory.Tracker, interruptSignal *atomic.Value) ([]kv.KeyRange, error) { krs := make([]kv.KeyRange, 0, len(ranges)) - for _, ran := range ranges { + const CheckSignalStep = 8 + var estimatedMemUsage int64 + // encodeIndexKey and EncodeIndexSeekKey is time-consuming, thus we need to + // check the interrupt signal periodically. + for i, ran := range ranges { low, high, err := encodeIndexKey(sc, ran) if err != nil { return nil, err } + if i == 0 { + estimatedMemUsage += int64(cap(low) + cap(high)) + } for _, tid := range tids { startKey := tablecodec.EncodeIndexSeekKey(tid, idxID, low) endKey := tablecodec.EncodeIndexSeekKey(tid, idxID, high) + if i == 0 { + estimatedMemUsage += int64(cap(startKey)) + int64(cap(endKey)) + } krs = append(krs, kv.KeyRange{StartKey: startKey, EndKey: endKey}) } + if i%CheckSignalStep == 0 { + if i == 0 && memTracker != nil { + estimatedMemUsage *= int64(len(ranges)) + memTracker.Consume(estimatedMemUsage) + } + if interruptSignal != nil && interruptSignal.Load().(bool) { + return nil, nil + } + } } return krs, nil } diff --git a/distsql/select_result.go b/distsql/select_result.go index ef1a99e193e57..0eef83e65b757 100644 --- a/distsql/select_result.go +++ b/distsql/select_result.go @@ -23,6 +23,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/errno" @@ -272,6 +273,12 @@ func (r *selectResult) Next(ctx context.Context, chk *chunk.Chunk) error { // NextRaw returns the next raw partial result. func (r *selectResult) NextRaw(ctx context.Context) (data []byte, err error) { + failpoint.Inject("mockNextRawError", func(val failpoint.Value) { + if val.(bool) { + failpoint.Return(nil, errors.New("mockNextRawError")) + } + }) + resultSubset, err := r.resp.Next(ctx) r.partialCount++ r.feedback.Invalidate() @@ -394,9 +401,14 @@ func (r *selectResult) updateCopRuntimeStats(ctx context.Context, copStats *copr } else { // For cop task cases, we still need this protection. if len(r.selectResp.GetExecutionSummaries()) != len(r.copPlanIDs) { - logutil.Logger(ctx).Error("invalid cop task execution summaries length", - zap.Int("expected", len(r.copPlanIDs)), - zap.Int("received", len(r.selectResp.GetExecutionSummaries()))) + // for TiFlash streaming call(BatchCop and MPP), it is by design that only the last response will + // carry the execution summaries, so it is ok if some responses have no execution summaries, should + // not trigger an error log in this case. + if !(r.storeType == kv.TiFlash && len(r.selectResp.GetExecutionSummaries()) == 0) { + logutil.Logger(ctx).Error("invalid cop task execution summaries length", + zap.Int("expected", len(r.copPlanIDs)), + zap.Int("received", len(r.selectResp.GetExecutionSummaries()))) + } return } for i, detail := range r.selectResp.GetExecutionSummaries() { diff --git a/errors.toml b/errors.toml index 6482bf991e149..423d96c6cef87 100644 --- a/errors.toml +++ b/errors.toml @@ -511,6 +511,11 @@ error = ''' Incorrect usage of %s and %s ''' +["executor:1235"] +error = ''' +%-.32s is not supported. To enable this experimental feature, set '%-.32s' in the configuration file. +''' + ["executor:1242"] error = ''' Subquery returns more than 1 row diff --git a/executor/adapter.go b/executor/adapter.go index f94091026321c..5459c985ff883 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -1014,13 +1014,12 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) { if _, ok := a.StmtNode.(*ast.CommitStmt); ok { slowItems.PrevStmt = sessVars.PrevStmt.String() } + slowLog := sessVars.SlowLogFormat(slowItems) if trace.IsEnabled() { - trace.Log(a.GoCtx, "details", sessVars.SlowLogFormat(slowItems)) + trace.Log(a.GoCtx, "details", slowLog) } - if costTime < threshold { - logutil.SlowQueryLogger.Debug(sessVars.SlowLogFormat(slowItems)) - } else { - logutil.SlowQueryLogger.Warn(sessVars.SlowLogFormat(slowItems)) + logutil.SlowQueryLogger.Warn(slowLog) + if costTime >= threshold { if sessVars.InRestrictedSQL { totalQueryProcHistogramInternal.Observe(costTime.Seconds()) totalCopProcHistogramInternal.Observe(execDetail.TimeDetail.ProcessTime.Seconds()) diff --git a/executor/aggfuncs/builder.go b/executor/aggfuncs/builder.go index c914ea4838f2d..ce6659af3ee45 100644 --- a/executor/aggfuncs/builder.go +++ b/executor/aggfuncs/builder.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/logutil" "go.uber.org/zap" ) @@ -390,7 +391,8 @@ func buildMaxMin(aggFuncDesc *aggregation.AggFuncDesc, ordinal int, isMax bool) args: aggFuncDesc.Args, ordinal: ordinal, }, - isMax: isMax, + isMax: isMax, + collator: collate.GetCollator(aggFuncDesc.RetTp.Collate), } frac := base.args[0].GetType().Decimal if frac == -1 { diff --git a/executor/aggfuncs/func_max_min.go b/executor/aggfuncs/func_max_min.go index d3662aff258d5..51055c6e24b5e 100644 --- a/executor/aggfuncs/func_max_min.go +++ b/executor/aggfuncs/func_max_min.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/json" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/stringutil" ) @@ -232,7 +233,8 @@ type partialResult4MaxMinSet struct { type baseMaxMinAggFunc struct { baseAggFunc - isMax bool + isMax bool + collator collate.Collator } type maxMin4Int struct { @@ -1485,7 +1487,7 @@ func (e *maxMin4Enum) UpdatePartialResult(sctx sessionctx.Context, rowsInGroup [ continue } en := d.GetMysqlEnum() - if e.isMax && en.Name > p.val.Name || !e.isMax && en.Name < p.val.Name { + if e.isMax && e.collator.Compare(en.Name, p.val.Name) > 0 || !e.isMax && e.collator.Compare(en.Name, p.val.Name) < 0 { oldMem := len(p.val.Name) newMem := len(en.Name) memDelta += int64(newMem - oldMem) @@ -1504,7 +1506,7 @@ func (e *maxMin4Enum) MergePartialResult(sctx sessionctx.Context, src, dst Parti *p2 = *p1 return 0, nil } - if e.isMax && p1.val.Name > p2.val.Name || !e.isMax && p1.val.Name < p2.val.Name { + if e.isMax && e.collator.Compare(p1.val.Name, p2.val.Name) > 0 || !e.isMax && e.collator.Compare(p1.val.Name, p2.val.Name) < 0 { p2.val, p2.isNull = p1.val, false } return 0, nil @@ -1552,7 +1554,7 @@ func (e *maxMin4Set) UpdatePartialResult(sctx sessionctx.Context, rowsInGroup [] continue } s := d.GetMysqlSet() - if e.isMax && s.Name > p.val.Name || !e.isMax && s.Name < p.val.Name { + if e.isMax && e.collator.Compare(s.Name, p.val.Name) > 0 || !e.isMax && e.collator.Compare(s.Name, p.val.Name) < 0 { oldMem := len(p.val.Name) newMem := len(s.Name) memDelta += int64(newMem - oldMem) @@ -1571,7 +1573,7 @@ func (e *maxMin4Set) MergePartialResult(sctx sessionctx.Context, src, dst Partia *p2 = *p1 return 0, nil } - if e.isMax && p1.val.Name > p2.val.Name || !e.isMax && p1.val.Name < p2.val.Name { + if e.isMax && e.collator.Compare(p1.val.Name, p2.val.Name) > 0 || !e.isMax && e.collator.Compare(p1.val.Name, p2.val.Name) < 0 { p2.val, p2.isNull = p1.val, false } return 0, nil diff --git a/executor/aggregate_test.go b/executor/aggregate_test.go index 81848934ea075..3e84256d79138 100644 --- a/executor/aggregate_test.go +++ b/executor/aggregate_test.go @@ -1374,6 +1374,26 @@ func (s *testSuiteAgg) TestIssue20658(c *C) { } } +func (s *testSuiteAgg) TestAvgDecimal(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("use test;") + tk.MustExec("drop table if exists td;") + tk.MustExec("create table td (col_bigint bigint(20), col_smallint smallint(6));") + tk.MustExec("insert into td values (null, 22876);") + tk.MustExec("insert into td values (9220557287087669248, 32767);") + tk.MustExec("insert into td values (28030, 32767);") + tk.MustExec("insert into td values (-3309864251140603904,32767);") + tk.MustExec("insert into td values (4,0);") + tk.MustExec("insert into td values (null,0);") + tk.MustExec("insert into td values (4,-23828);") + tk.MustExec("insert into td values (54720,32767);") + tk.MustExec("insert into td values (0,29815);") + tk.MustExec("insert into td values (10017,-32661);") + tk.MustQuery(" SELECT AVG( col_bigint / col_smallint) AS field1 FROM td;").Sort().Check(testkit.Rows("25769363061037.62077260")) + tk.MustQuery(" SELECT AVG(col_bigint) OVER (PARTITION BY col_smallint) as field2 FROM td where col_smallint = -23828;").Sort().Check(testkit.Rows("4.0000")) + tk.MustExec("drop table td;") +} + func (s *testSerialSuite) TestRandomPanicAggConsume(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("set @@tidb_max_chunk_size=32") diff --git a/executor/analyze.go b/executor/analyze.go index a3bc12c355aff..342828471cbca 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -808,6 +808,21 @@ func (e AnalyzeColumnsExec) decodeSampleDataWithVirtualColumn( return nil } +func readDataAndSendTask(handler *tableResultHandler, mergeTaskCh chan []byte) error { + defer close(mergeTaskCh) + for { + data, err := handler.nextRaw(context.TODO()) + if err != nil { + return errors.Trace(err) + } + if data == nil { + break + } + mergeTaskCh <- data + } + return nil +} + func (e *AnalyzeColumnsExec) buildSamplingStats( ranges []*ranger.Range, needExtStats bool, @@ -853,17 +868,10 @@ func (e *AnalyzeColumnsExec) buildSamplingStats( for i := 0; i < statsConcurrency; i++ { go e.subMergeWorker(mergeResultCh, mergeTaskCh, l, i == 0) } - for { - data, err1 := e.resultHandler.nextRaw(context.TODO()) - if err1 != nil { - return 0, nil, nil, nil, nil, err1 - } - if data == nil { - break - } - mergeTaskCh <- data + if err = readDataAndSendTask(e.resultHandler, mergeTaskCh); err != nil { + return 0, nil, nil, nil, nil, err } - close(mergeTaskCh) + mergeWorkerPanicCnt := 0 for mergeWorkerPanicCnt < statsConcurrency { mergeResult, ok := <-mergeResultCh diff --git a/executor/analyze_test.go b/executor/analyze_test.go index 21d36544c0deb..f2c461b3ee756 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -413,8 +413,8 @@ func (s *testFastAnalyze) TestFastAnalyze(c *C) { tk.MustExec("use test") tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int primary key, b int, c char(10), index index_b(b))") - tk.MustExec("set @@session.tidb_enable_fast_analyze=1") - tk.MustExec("set @@session.tidb_build_stats_concurrency=1") + tk.MustExec("set @@session.tidb_enable_fast_analyze = 1") + tk.MustExec("set @@session.tidb_build_stats_concurrency = 1") tk.MustExec("set @@tidb_analyze_version = 1") // Should not panic. tk.MustExec("analyze table t") @@ -564,18 +564,18 @@ func (s *testSerialSuite2) TestAnalyzeIndex(c *C) { tk.MustExec("drop table if exists t1") tk.MustExec("create table t1 (id int, v int, primary key(id), index k(v))") tk.MustExec("insert into t1(id, v) values(1, 2), (2, 2), (3, 2), (4, 2), (5, 1), (6, 3), (7, 4)") - tk.MustExec("set @@tidb_analyze_version=1") + tk.MustExec("set @@tidb_analyze_version = 1") tk.MustExec("analyze table t1 index k") c.Assert(len(tk.MustQuery("show stats_buckets where table_name = 't1' and column_name = 'k' and is_index = 1").Rows()), Greater, 0) tk.MustExec("set @@tidb_analyze_version=default") tk.MustExec("analyze table t1") - c.Assert(len(tk.MustQuery("show stats_topn where table_name = 't1' and column_name = 'k' and is_index = 1").Rows()), Greater, 0) + c.Assert(len(tk.MustQuery("show stats_topn where table_name = 't1' and column_name = 'k' and is_index = 1").Rows()), Equals, 0) func() { defer tk.MustExec("set @@session.tidb_enable_fast_analyze=0") tk.MustExec("drop stats t1") - tk.MustExec("set @@session.tidb_enable_fast_analyze=1") - tk.MustExec("set @@tidb_analyze_version=1") + tk.MustExec("set @@session.tidb_enable_fast_analyze = 1") + tk.MustExec("set @@tidb_analyze_version = 1") tk.MustExec("analyze table t1 index k") c.Assert(len(tk.MustQuery("show stats_buckets where table_name = 't1' and column_name = 'k' and is_index = 1").Rows()), Greater, 1) }() @@ -863,7 +863,7 @@ func (s *testSuite1) TestNormalAnalyzeOnCommonHandle(c *C) { tk.MustExec("insert into t3 values(1,1,1), (2,2,2), (3,3,3)") // Version2 is tested in TestStatsVer2. - tk.MustExec("set@@tidb_analyze_version=1") + tk.MustExec("set@@tidb_analyze_version = 1") tk.MustExec("analyze table t1, t2, t3") tk.MustQuery(`show stats_buckets where table_name in ("t1", "t2", "t3")`).Sort().Check(testkit.Rows( @@ -913,6 +913,8 @@ func (s *testSuite1) TestDefaultValForAnalyze(c *C) { for i := 1; i < 4; i++ { tk.MustExec("insert into t values (?)", i) } + tk.MustQuery("select @@tidb_analyze_version").Check(testkit.Rows("1")) + tk.MustQuery("select @@session.tidb_analyze_version").Check(testkit.Rows("1")) tk.MustQuery("select @@tidb_enable_fast_analyze").Check(testkit.Rows("0")) tk.MustQuery("select @@session.tidb_enable_fast_analyze").Check(testkit.Rows("0")) tk.MustExec("analyze table t with 0 topn;") @@ -956,7 +958,7 @@ func (s *testSerialSuite2) TestIssue20874(c *C) { tk.MustExec("delete from mysql.stats_histograms") tk.MustExec("create table t (a char(10) collate utf8mb4_unicode_ci not null, b char(20) collate utf8mb4_general_ci not null, key idxa(a), key idxb(b))") tk.MustExec("insert into t values ('#', 'C'), ('$', 'c'), ('a', 'a')") - tk.MustExec("set @@tidb_analyze_version=1") + tk.MustExec("set @@tidb_analyze_version = 1") tk.MustExec("analyze table t") tk.MustQuery("show stats_buckets where db_name = 'test' and table_name = 't'").Sort().Check(testkit.Rows( "test t a 0 0 1 1 \x02\xd2 \x02\xd2 0", diff --git a/executor/builder.go b/executor/builder.go index 2c04de8638624..85eb1e63ce26f 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -20,6 +20,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "unsafe" @@ -54,6 +55,7 @@ import ( "github.com/pingcap/tidb/util/dbterror" "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/logutil" + "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/ranger" "github.com/pingcap/tidb/util/rowcodec" "github.com/pingcap/tidb/util/timeutil" @@ -87,6 +89,9 @@ type executorBuilder struct { // ExplicitStaleness means whether the 'SELECT' clause are using 'AS OF TIMESTAMP' to perform stale read explicitly. explicitStaleness bool txnScope string + inUpdateStmt bool + inDeleteStmt bool + inInsertStmt bool } // CTEStorages stores resTbl and iterInTbl for CTEExec. @@ -779,6 +784,7 @@ func (b *executorBuilder) buildSetConfig(v *plannercore.SetConfig) Executor { } func (b *executorBuilder) buildInsert(v *plannercore.Insert) Executor { + b.inInsertStmt = true if v.SelectPlan != nil { // Try to update the forUpdateTS for insert/replace into select statements. // Set the selectPlan parameter to nil to make it always update the forUpdateTS. @@ -1176,15 +1182,6 @@ func (b *executorBuilder) buildHashJoin(v *plannercore.PhysicalHashJoin) Executo } } - // consider collations - leftTypes := make([]*types.FieldType, 0, len(retTypes(leftExec))) - for _, tp := range retTypes(leftExec) { - leftTypes = append(leftTypes, tp.Clone()) - } - rightTypes := make([]*types.FieldType, 0, len(retTypes(rightExec))) - for _, tp := range retTypes(rightExec) { - rightTypes = append(rightTypes, tp.Clone()) - } leftIsBuildSide := true e.isNullEQ = v.IsNullEQ @@ -1227,24 +1224,32 @@ func (b *executorBuilder) buildHashJoin(v *plannercore.PhysicalHashJoin) Executo } executorCountHashJoinExec.Inc() + // We should use JoinKey to construct the type information using by hashing, instead of using the child's schema directly. + // When a hybrid type column is hashed multiple times, we need to distinguish what field types are used. + // For example, the condition `enum = int and enum = string`, we should use ETInt to hash the first column, + // and use ETString to hash the second column, although they may be the same column. + leftExecTypes, rightExecTypes := retTypes(leftExec), retTypes(rightExec) + leftTypes, rightTypes := make([]*types.FieldType, 0, len(v.LeftJoinKeys)), make([]*types.FieldType, 0, len(v.RightJoinKeys)) + for i, col := range v.LeftJoinKeys { + leftTypes = append(leftTypes, leftExecTypes[col.Index].Clone()) + leftTypes[i].Flag = col.RetType.Flag + } + for i, col := range v.RightJoinKeys { + rightTypes = append(rightTypes, rightExecTypes[col.Index].Clone()) + rightTypes[i].Flag = col.RetType.Flag + } + + // consider collations for i := range v.EqualConditions { chs, coll := v.EqualConditions[i].CharsetAndCollation(e.ctx) - bt := leftTypes[v.LeftJoinKeys[i].Index] - bt.Charset, bt.Collate = chs, coll - pt := rightTypes[v.RightJoinKeys[i].Index] - pt.Charset, pt.Collate = chs, coll + leftTypes[i].Charset, leftTypes[i].Collate = chs, coll + rightTypes[i].Charset, rightTypes[i].Collate = chs, coll } if leftIsBuildSide { e.buildTypes, e.probeTypes = leftTypes, rightTypes } else { e.buildTypes, e.probeTypes = rightTypes, leftTypes } - for _, key := range e.buildKeys { - e.buildTypes[key.Index].Flag = key.RetType.Flag - } - for _, key := range e.probeKeys { - e.probeTypes[key.Index].Flag = key.RetType.Flag - } return e } @@ -1386,6 +1391,12 @@ func (b *executorBuilder) buildProjection(v *plannercore.PhysicalProjection) Exe if int64(v.StatsCount()) < int64(b.ctx.GetSessionVars().MaxChunkSize) { e.numWorkers = 0 } + + // Use un-parallel projection for query that write on memdb to avoid data race. + // See also https://github.com/pingcap/tidb/issues/26832 + if b.inUpdateStmt || b.inDeleteStmt || b.inInsertStmt || b.hasLock { + e.numWorkers = 0 + } return e } @@ -1859,6 +1870,7 @@ func (b *executorBuilder) buildSplitRegion(v *plannercore.SplitRegion) Executor } func (b *executorBuilder) buildUpdate(v *plannercore.Update) Executor { + b.inUpdateStmt = true tblID2table := make(map[int64]table.Table, len(v.TblColPosInfos)) multiUpdateOnSameTable := make(map[int64]bool) for _, info := range v.TblColPosInfos { @@ -1930,6 +1942,7 @@ func getAssignFlag(ctx sessionctx.Context, v *plannercore.Update, schemaLen int) } func (b *executorBuilder) buildDelete(v *plannercore.Delete) Executor { + b.inDeleteStmt = true tblID2table := make(map[int64]table.Table, len(v.TblColPosInfos)) for _, info := range v.TblColPosInfos { tblID2table[info.TblID], _ = b.is.TableByID(info.TblID) @@ -2476,6 +2489,21 @@ func (b *executorBuilder) buildIndexLookUpJoin(v *plannercore.PhysicalIndexJoin) outerTypes[col.Index].Flag = col.RetType.Flag } + // We should use JoinKey to construct the type information using by hashing, instead of using the child's schema directly. + // When a hybrid type column is hashed multiple times, we need to distinguish what field types are used. + // For example, the condition `enum = int and enum = string`, we should use ETInt to hash the first column, + // and use ETString to hash the second column, although they may be the same column. + innerHashTypes := make([]*types.FieldType, len(v.InnerHashKeys)) + outerHashTypes := make([]*types.FieldType, len(v.OuterHashKeys)) + for i, col := range v.InnerHashKeys { + innerHashTypes[i] = innerTypes[col.Index].Clone() + innerHashTypes[i].Flag = col.RetType.Flag + } + for i, col := range v.OuterHashKeys { + outerHashTypes[i] = outerTypes[col.Index].Clone() + outerHashTypes[i].Flag = col.RetType.Flag + } + var ( outerFilter []expression.Expression leftTypes, rightTypes []*types.FieldType @@ -2510,12 +2538,14 @@ func (b *executorBuilder) buildIndexLookUpJoin(v *plannercore.PhysicalIndexJoin) e := &IndexLookUpJoin{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID(), outerExec), outerCtx: outerCtx{ - rowTypes: outerTypes, - filter: outerFilter, + rowTypes: outerTypes, + hashTypes: outerHashTypes, + filter: outerFilter, }, innerCtx: innerCtx{ readerBuilder: &dataReaderBuilder{Plan: innerPlan, executorBuilder: b}, rowTypes: innerTypes, + hashTypes: innerHashTypes, colLens: v.IdxColLens, hasPrefixCol: hasPrefixCol, }, @@ -2524,6 +2554,7 @@ func (b *executorBuilder) buildIndexLookUpJoin(v *plannercore.PhysicalIndexJoin) indexRanges: v.Ranges, keyOff2IdxOff: v.KeyOff2IdxOff, lastColHelper: v.CompareFilters, + finished: &atomic.Value{}, } childrenUsedSchema := markChildrenUsedCols(v.Schema(), v.Children()[0].Schema(), v.Children()[1].Schema()) e.joiner = newJoiner(b.ctx, v.JoinType, v.InnerChildIdx == 0, defaultValues, v.OtherConditions, leftTypes, rightTypes, childrenUsedSchema) @@ -3353,21 +3384,21 @@ type mockPhysicalIndexReader struct { } func (builder *dataReaderBuilder) buildExecutorForIndexJoin(ctx context.Context, lookUpContents []*indexJoinLookUpContent, - IndexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager, canReorderHandles bool) (Executor, error) { - return builder.buildExecutorForIndexJoinInternal(ctx, builder.Plan, lookUpContents, IndexRanges, keyOff2IdxOff, cwc, canReorderHandles) + IndexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager, canReorderHandles bool, memTracker *memory.Tracker, interruptSignal *atomic.Value) (Executor, error) { + return builder.buildExecutorForIndexJoinInternal(ctx, builder.Plan, lookUpContents, IndexRanges, keyOff2IdxOff, cwc, canReorderHandles, memTracker, interruptSignal) } func (builder *dataReaderBuilder) buildExecutorForIndexJoinInternal(ctx context.Context, plan plannercore.Plan, lookUpContents []*indexJoinLookUpContent, - IndexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager, canReorderHandles bool) (Executor, error) { + IndexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager, canReorderHandles bool, memTracker *memory.Tracker, interruptSignal *atomic.Value) (Executor, error) { switch v := plan.(type) { case *plannercore.PhysicalTableReader: - return builder.buildTableReaderForIndexJoin(ctx, v, lookUpContents, IndexRanges, keyOff2IdxOff, cwc, canReorderHandles) + return builder.buildTableReaderForIndexJoin(ctx, v, lookUpContents, IndexRanges, keyOff2IdxOff, cwc, canReorderHandles, memTracker, interruptSignal) case *plannercore.PhysicalIndexReader: - return builder.buildIndexReaderForIndexJoin(ctx, v, lookUpContents, IndexRanges, keyOff2IdxOff, cwc) + return builder.buildIndexReaderForIndexJoin(ctx, v, lookUpContents, IndexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) case *plannercore.PhysicalIndexLookUpReader: - return builder.buildIndexLookUpReaderForIndexJoin(ctx, v, lookUpContents, IndexRanges, keyOff2IdxOff, cwc) + return builder.buildIndexLookUpReaderForIndexJoin(ctx, v, lookUpContents, IndexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) case *plannercore.PhysicalUnionScan: - return builder.buildUnionScanForIndexJoin(ctx, v, lookUpContents, IndexRanges, keyOff2IdxOff, cwc, canReorderHandles) + return builder.buildUnionScanForIndexJoin(ctx, v, lookUpContents, IndexRanges, keyOff2IdxOff, cwc, canReorderHandles, memTracker, interruptSignal) // The inner child of IndexJoin might be Projection when a combination of the following conditions is true: // 1. The inner child fetch data using indexLookupReader // 2. PK is not handle @@ -3375,11 +3406,11 @@ func (builder *dataReaderBuilder) buildExecutorForIndexJoinInternal(ctx context. // In this case, an extra column tidb_rowid will be appended in the output result of IndexLookupReader(see copTask.doubleReadNeedProj). // Then we need a Projection upon IndexLookupReader to prune the redundant column. case *plannercore.PhysicalProjection: - return builder.buildProjectionForIndexJoin(ctx, v, lookUpContents, IndexRanges, keyOff2IdxOff, cwc) + return builder.buildProjectionForIndexJoin(ctx, v, lookUpContents, IndexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) // Need to support physical selection because after PR 16389, TiDB will push down all the expr supported by TiKV or TiFlash // in predicate push down stage, so if there is an expr which only supported by TiFlash, a physical selection will be added after index read case *plannercore.PhysicalSelection: - childExec, err := builder.buildExecutorForIndexJoinInternal(ctx, v.Children()[0], lookUpContents, IndexRanges, keyOff2IdxOff, cwc, canReorderHandles) + childExec, err := builder.buildExecutorForIndexJoinInternal(ctx, v.Children()[0], lookUpContents, IndexRanges, keyOff2IdxOff, cwc, canReorderHandles, memTracker, interruptSignal) if err != nil { return nil, err } @@ -3397,9 +3428,9 @@ func (builder *dataReaderBuilder) buildExecutorForIndexJoinInternal(ctx context. func (builder *dataReaderBuilder) buildUnionScanForIndexJoin(ctx context.Context, v *plannercore.PhysicalUnionScan, values []*indexJoinLookUpContent, indexRanges []*ranger.Range, keyOff2IdxOff []int, - cwc *plannercore.ColWithCmpFuncManager, canReorderHandles bool) (Executor, error) { + cwc *plannercore.ColWithCmpFuncManager, canReorderHandles bool, memTracker *memory.Tracker, interruptSignal *atomic.Value) (Executor, error) { childBuilder := &dataReaderBuilder{Plan: v.Children()[0], executorBuilder: builder.executorBuilder} - reader, err := childBuilder.buildExecutorForIndexJoin(ctx, values, indexRanges, keyOff2IdxOff, cwc, canReorderHandles) + reader, err := childBuilder.buildExecutorForIndexJoin(ctx, values, indexRanges, keyOff2IdxOff, cwc, canReorderHandles, memTracker, interruptSignal) if err != nil { return nil, err } @@ -3413,7 +3444,7 @@ func (builder *dataReaderBuilder) buildUnionScanForIndexJoin(ctx context.Context func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Context, v *plannercore.PhysicalTableReader, lookUpContents []*indexJoinLookUpContent, indexRanges []*ranger.Range, keyOff2IdxOff []int, - cwc *plannercore.ColWithCmpFuncManager, canReorderHandles bool) (Executor, error) { + cwc *plannercore.ColWithCmpFuncManager, canReorderHandles bool, memTracker *memory.Tracker, interruptSignal *atomic.Value) (Executor, error) { e, err := buildNoRangeTableReader(builder.executorBuilder, v) if err != nil { return nil, err @@ -3421,7 +3452,7 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte tbInfo := e.table.Meta() if v.IsCommonHandle { if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { - kvRanges, err := buildKvRangesForIndexJoin(e.ctx, getPhysicalTableID(e.table), -1, lookUpContents, indexRanges, keyOff2IdxOff, cwc) + kvRanges, err := buildKvRangesForIndexJoin(e.ctx, getPhysicalTableID(e.table), -1, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) if err != nil { return nil, err } @@ -3450,7 +3481,7 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte return nil, err } pid := p.GetPhysicalID() - tmp, err := buildKvRangesForIndexJoin(e.ctx, pid, -1, []*indexJoinLookUpContent{content}, indexRanges, keyOff2IdxOff, cwc) + tmp, err := buildKvRangesForIndexJoin(e.ctx, pid, -1, []*indexJoinLookUpContent{content}, indexRanges, keyOff2IdxOff, cwc, nil, interruptSignal) if err != nil { return nil, err } @@ -3465,7 +3496,7 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte kvRanges = make([]kv.KeyRange, 0, len(partitions)*len(lookUpContents)) for _, p := range partitions { pid := p.GetPhysicalID() - tmp, err := buildKvRangesForIndexJoin(e.ctx, pid, -1, lookUpContents, indexRanges, keyOff2IdxOff, cwc) + tmp, err := buildKvRangesForIndexJoin(e.ctx, pid, -1, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) if err != nil { return nil, err } @@ -3631,14 +3662,14 @@ func (builder *dataReaderBuilder) buildTableReaderFromKvRanges(ctx context.Conte } func (builder *dataReaderBuilder) buildIndexReaderForIndexJoin(ctx context.Context, v *plannercore.PhysicalIndexReader, - lookUpContents []*indexJoinLookUpContent, indexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager) (Executor, error) { + lookUpContents []*indexJoinLookUpContent, indexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager, memoryTracker *memory.Tracker, interruptSignal *atomic.Value) (Executor, error) { e, err := buildNoRangeIndexReader(builder.executorBuilder, v) if err != nil { return nil, err } tbInfo := e.table.Meta() if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { - kvRanges, err := buildKvRangesForIndexJoin(e.ctx, e.physicalTableID, e.index.ID, lookUpContents, indexRanges, keyOff2IdxOff, cwc) + kvRanges, err := buildKvRangesForIndexJoin(e.ctx, e.physicalTableID, e.index.ID, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memoryTracker, interruptSignal) if err != nil { return nil, err } @@ -3677,7 +3708,7 @@ func (builder *dataReaderBuilder) buildIndexReaderForIndexJoin(ctx context.Conte } func (builder *dataReaderBuilder) buildIndexLookUpReaderForIndexJoin(ctx context.Context, v *plannercore.PhysicalIndexLookUpReader, - lookUpContents []*indexJoinLookUpContent, indexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager) (Executor, error) { + lookUpContents []*indexJoinLookUpContent, indexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager, memTracker *memory.Tracker, interruptSignal *atomic.Value) (Executor, error) { e, err := buildNoRangeIndexLookUpReader(builder.executorBuilder, v) if err != nil { return nil, err @@ -3685,7 +3716,7 @@ func (builder *dataReaderBuilder) buildIndexLookUpReaderForIndexJoin(ctx context tbInfo := e.table.Meta() if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { - e.kvRanges, err = buildKvRangesForIndexJoin(e.ctx, getPhysicalTableID(e.table), e.index.ID, lookUpContents, indexRanges, keyOff2IdxOff, cwc) + e.kvRanges, err = buildKvRangesForIndexJoin(e.ctx, getPhysicalTableID(e.table), e.index.ID, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) if err != nil { return nil, err } @@ -3725,12 +3756,12 @@ func (builder *dataReaderBuilder) buildIndexLookUpReaderForIndexJoin(ctx context } func (builder *dataReaderBuilder) buildProjectionForIndexJoin(ctx context.Context, v *plannercore.PhysicalProjection, - lookUpContents []*indexJoinLookUpContent, indexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager) (Executor, error) { + lookUpContents []*indexJoinLookUpContent, indexRanges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager, memTracker *memory.Tracker, interruptSignal *atomic.Value) (Executor, error) { physicalIndexLookUp, isDoubleRead := v.Children()[0].(*plannercore.PhysicalIndexLookUpReader) if !isDoubleRead { return nil, errors.Errorf("inner child of Projection should be IndexLookupReader, but got %T", v) } - childExec, err := builder.buildIndexLookUpReaderForIndexJoin(ctx, physicalIndexLookUp, lookUpContents, indexRanges, keyOff2IdxOff, cwc) + childExec, err := builder.buildIndexLookUpReaderForIndexJoin(ctx, physicalIndexLookUp, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) if err != nil { return nil, err } @@ -3797,7 +3828,7 @@ func buildRangesForIndexJoin(ctx sessionctx.Context, lookUpContents []*indexJoin // buildKvRangesForIndexJoin builds kv ranges for index join when the inner plan is index scan plan. func buildKvRangesForIndexJoin(ctx sessionctx.Context, tableID, indexID int64, lookUpContents []*indexJoinLookUpContent, - ranges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager) (_ []kv.KeyRange, err error) { + ranges []*ranger.Range, keyOff2IdxOff []int, cwc *plannercore.ColWithCmpFuncManager, memTracker *memory.Tracker, interruptSignal *atomic.Value) (_ []kv.KeyRange, err error) { kvRanges := make([]kv.KeyRange, 0, len(ranges)*len(lookUpContents)) lastPos := len(ranges[0].LowVal) - 1 sc := ctx.GetSessionVars().StmtCtx @@ -3816,7 +3847,7 @@ func buildKvRangesForIndexJoin(ctx sessionctx.Context, tableID, indexID int64, l if indexID == -1 { tmpKvRanges, err = distsql.CommonHandleRangesToKVRanges(sc, []int64{tableID}, ranges) } else { - tmpKvRanges, err = distsql.IndexRangesToKVRanges(sc, tableID, indexID, ranges, nil) + tmpKvRanges, err = distsql.IndexRangesToKVRangesWithInterruptSignal(sc, tableID, indexID, ranges, nil, memTracker, interruptSignal) } if err != nil { return nil, err @@ -3838,7 +3869,12 @@ func buildKvRangesForIndexJoin(ctx sessionctx.Context, tableID, indexID int64, l } } } - + if len(kvRanges) != 0 && memTracker != nil { + memTracker.Consume(int64(2 * cap(kvRanges[0].StartKey) * len(kvRanges))) + } + if len(tmpDatumRanges) != 0 && memTracker != nil { + memTracker.Consume(2 * int64(len(tmpDatumRanges)) * types.EstimatedMemUsage(tmpDatumRanges[0].LowVal, len(tmpDatumRanges))) + } if cwc == nil { sort.Slice(kvRanges, func(i, j int) bool { return bytes.Compare(kvRanges[i].StartKey, kvRanges[j].StartKey) < 0 @@ -3854,7 +3890,7 @@ func buildKvRangesForIndexJoin(ctx sessionctx.Context, tableID, indexID int64, l if indexID == -1 { return distsql.CommonHandleRangesToKVRanges(ctx.GetSessionVars().StmtCtx, []int64{tableID}, tmpDatumRanges) } - return distsql.IndexRangesToKVRanges(ctx.GetSessionVars().StmtCtx, tableID, indexID, tmpDatumRanges, nil) + return distsql.IndexRangesToKVRangesWithInterruptSignal(ctx.GetSessionVars().StmtCtx, tableID, indexID, tmpDatumRanges, nil, memTracker, interruptSignal) } func (b *executorBuilder) buildWindow(v *plannercore.PhysicalWindow) Executor { @@ -3875,7 +3911,7 @@ func (b *executorBuilder) buildWindow(v *plannercore.PhysicalWindow) Executor { partialResults := make([]aggfuncs.PartialResult, 0, len(v.WindowFuncDescs)) resultColIdx := v.Schema().Len() - len(v.WindowFuncDescs) for _, desc := range v.WindowFuncDescs { - aggDesc, err := aggregation.NewAggFuncDesc(b.ctx, desc.Name, desc.Args, false) + aggDesc, err := aggregation.NewAggFuncDescForWindowFunc(b.ctx, desc, false) if err != nil { b.err = err return nil diff --git a/executor/ddl.go b/executor/ddl.go index 13cc0b4f5aef8..77b24ddfbd7fd 100644 --- a/executor/ddl.go +++ b/executor/ddl.go @@ -612,13 +612,16 @@ func (e *DDLExec) executeFlashbackTable(s *ast.FlashBackTableStmt) error { func (e *DDLExec) executeLockTables(s *ast.LockTablesStmt) error { if !config.TableLockEnabled() { + e.ctx.GetSessionVars().StmtCtx.AppendWarning(ErrFuncNotEnabled.GenWithStackByArgs("LOCK TABLES", "enable-table-lock")) return nil } + return domain.GetDomain(e.ctx).DDL().LockTables(e.ctx, s) } func (e *DDLExec) executeUnlockTables(_ *ast.UnlockTablesStmt) error { if !config.TableLockEnabled() { + e.ctx.GetSessionVars().StmtCtx.AppendWarning(ErrFuncNotEnabled.GenWithStackByArgs("UNLOCK TABLES", "enable-table-lock")) return nil } lockedTables := e.ctx.GetAllTableLocks() diff --git a/executor/delete.go b/executor/delete.go index 16f0e9c421b19..d2840ae65bfee 100644 --- a/executor/delete.go +++ b/executor/delete.go @@ -138,6 +138,9 @@ func (e *DeleteExec) doBatchDelete(ctx context.Context) error { func (e *DeleteExec) composeTblRowMap(tblRowMap tableRowMapType, colPosInfos []plannercore.TblColPosInfo, joinedRow []types.Datum) error { // iterate all the joined tables, and got the copresonding rows in joinedRow. for _, info := range colPosInfos { + if unmatchedOuterRow(info, joinedRow) { + continue + } if tblRowMap[info.TblID] == nil { tblRowMap[info.TblID] = kv.NewHandleMap() } diff --git a/executor/errors.go b/executor/errors.go index 99066ecba9f19..7ad61b02f01af 100644 --- a/executor/errors.go +++ b/executor/errors.go @@ -59,6 +59,7 @@ var ( ErrCTEMaxRecursionDepth = dbterror.ClassExecutor.NewStd(mysql.ErrCTEMaxRecursionDepth) ErrDataInConsistentExtraIndex = dbterror.ClassExecutor.NewStd(mysql.ErrDataInConsistentExtraIndex) ErrDataInConsistentMisMatchIndex = dbterror.ClassExecutor.NewStd(mysql.ErrDataInConsistentMisMatchIndex) + ErrFuncNotEnabled = dbterror.ClassExecutor.NewStdErr(mysql.ErrNotSupportedYet, parser_mysql.Message("%-.32s is not supported. To enable this experimental feature, set '%-.32s' in the configuration file.", nil)) errUnsupportedFlashbackTmpTable = dbterror.ClassDDL.NewStdErr(mysql.ErrUnsupportedDDLOperation, parser_mysql.Message("Recover/flashback table is not supported on temporary tables", nil)) errTruncateWrongInsertValue = dbterror.ClassTable.NewStdErr(mysql.ErrTruncatedWrongValue, parser_mysql.Message("Incorrect %-.32s value: '%-.128s' for column '%.192s' at row %d", nil)) diff --git a/executor/executor_pkg_test.go b/executor/executor_pkg_test.go index 5b81b1afce095..0c8505703fb2a 100644 --- a/executor/executor_pkg_test.go +++ b/executor/executor_pkg_test.go @@ -195,7 +195,7 @@ func (s *testExecSuite) TestBuildKvRangesForIndexJoinWithoutCwc(c *C) { keyOff2IdxOff := []int{1, 3} ctx := mock.NewContext() - kvRanges, err := buildKvRangesForIndexJoin(ctx, 0, 0, joinKeyRows, indexRanges, keyOff2IdxOff, nil) + kvRanges, err := buildKvRangesForIndexJoin(ctx, 0, 0, joinKeyRows, indexRanges, keyOff2IdxOff, nil, nil, nil) c.Assert(err, IsNil) // Check the kvRanges is in order. for i, kvRange := range kvRanges { diff --git a/executor/executor_test.go b/executor/executor_test.go index d78ae6bdf55b3..fd583d7d23f8f 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -18,6 +18,7 @@ import ( "flag" "fmt" "math" + "math/rand" "net" "os" "strconv" @@ -8754,3 +8755,131 @@ func (s *testSuite) TestIssue26532(c *C) { tk.MustQuery("select greatest(\"2020-01-01 01:01:01\" ,\"2019-01-01 01:01:01\" )union select null;").Sort().Check(testkit.Rows("2020-01-01 01:01:01", "")) tk.MustQuery("select least(\"2020-01-01 01:01:01\" , \"2019-01-01 01:01:01\" )union select null;").Sort().Check(testkit.Rows("2019-01-01 01:01:01", "")) } + +func (s *testSerialSuite) TestIssue28650(c *C) { + defer testleak.AfterTest(c)() + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1, t2;") + tk.MustExec("create table t1(a int, index(a));") + tk.MustExec("create table t2(a int, c int, b char(50), index(a,c,b));") + tk.MustExec("set tidb_enable_rate_limit_action=off;") + + wg := &sync.WaitGroup{} + sql := `explain analyze + select /*+ stream_agg(@sel_1) stream_agg(@sel_3) %s(@sel_2 t2)*/ count(1) from + ( + SELECT t2.a AS t2_external_user_ext_id, t2.b AS t2_t1_ext_id FROM t2 INNER JOIN (SELECT t1.a AS d_t1_ext_id FROM t1 GROUP BY t1.a) AS anon_1 ON anon_1.d_t1_ext_id = t2.a WHERE t2.c = 123 AND t2.b + IN ("%s") ) tmp` + + wg.Add(1) + sqls := make([]string, 2) + go func() { + defer wg.Done() + inElems := make([]string, 1000) + for i := 0; i < len(inElems); i++ { + inElems[i] = fmt.Sprintf("wm_%dbDgAAwCD-v1QB%dxky-g_dxxQCw", rand.Intn(100), rand.Intn(100)) + } + sqls[0] = fmt.Sprintf(sql, "inl_join", strings.Join(inElems, "\",\"")) + sqls[1] = fmt.Sprintf(sql, "inl_hash_join", strings.Join(inElems, "\",\"")) + }() + + tk.MustExec("insert into t1 select rand()*400;") + for i := 0; i < 10; i++ { + tk.MustExec("insert into t1 select rand()*400 from t1;") + } + config.UpdateGlobal(func(conf *config.Config) { + conf.OOMAction = config.OOMActionCancel + }) + defer func() { + config.UpdateGlobal(func(conf *config.Config) { + conf.OOMAction = config.OOMActionLog + }) + }() + wg.Wait() + for _, sql := range sqls { + tk.MustExec("set @@tidb_mem_quota_query = 1073741824") // 1GB + c.Assert(tk.QueryToErr(sql), IsNil) + tk.MustExec("set @@tidb_mem_quota_query = 33554432") // 32MB, out of memory during executing + c.Assert(strings.Contains(tk.QueryToErr(sql).Error(), "Out Of Memory Quota!"), IsTrue) + tk.MustExec("set @@tidb_mem_quota_query = 65536") // 64KB, out of memory during building the plan + func() { + defer func() { + r := recover() + c.Assert(r, NotNil) + err := errors.Errorf("%v", r) + c.Assert(strings.Contains(err.Error(), "Out Of Memory Quota!"), IsTrue) + }() + tk.MustExec(sql) + }() + } +} + +func (s *testSerialSuite) TestIndexJoin31494(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + tk.MustExec("drop table if exists t1, t2;") + tk.MustExec("create table t1(a int(11) default null, b int(11) default null, key(b));") + insertStr := "insert into t1 values(1, 1)" + for i := 1; i < 32768; i++ { + insertStr += fmt.Sprintf(", (%d, %d)", i, i) + } + tk.MustExec(insertStr) + tk.MustExec("create table t2(a int(11) default null, b int(11) default null, c int(11) default null)") + insertStr = "insert into t2 values(1, 1, 1)" + for i := 1; i < 32768; i++ { + insertStr += fmt.Sprintf(", (%d, %d, %d)", i, i, i) + } + tk.MustExec(insertStr) + sm := &mockSessionManager1{ + PS: make([]*util.ProcessInfo, 0), + } + tk.Se.SetSessionManager(sm) + s.domain.ExpensiveQueryHandle().SetSessionManager(sm) + defer config.RestoreFunc()() + config.UpdateGlobal(func(conf *config.Config) { + conf.OOMAction = config.OOMActionCancel + }) + c.Assert(tk.Se.Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil), IsTrue) + tk.MustExec("set @@tidb_mem_quota_query=2097152;") + // This bug will be reproduced in 10 times. + for i := 0; i < 10; i++ { + err := tk.QueryToErr("select /*+ inl_join(t1) */ * from t1 right join t2 on t1.b=t2.b;") + c.Assert(err, NotNil) + c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") + err = tk.QueryToErr("select /*+ inl_hash_join(t1) */ * from t1 right join t2 on t1.b=t2.b;") + c.Assert(err, NotNil) + c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") + } +} + +func (s *testSuite) TestDeleteWithMulTbl(c *C) { + tk := testkit.NewTestKit(c, s.store) + + // Delete multiple tables from left joined table. + // The result of left join is (3, null, null). + // Because rows in t2 are not matched, so no row will be deleted in t2. + // But row in t1 is matched, so it should be deleted. + tk.MustExec("use test;") + tk.MustExec("drop table if exists t1, t2;") + tk.MustExec("create table t1 (c1 int);") + tk.MustExec("create table t2 (c1 int primary key, c2 int);") + tk.MustExec("insert into t1 values(3);") + tk.MustExec("insert into t2 values(2, 2);") + tk.MustExec("insert into t2 values(0, 0);") + tk.MustExec("delete from t1, t2 using t1 left join t2 on t1.c1 = t2.c2;") + tk.MustQuery("select * from t1 order by c1;").Check(testkit.Rows()) + tk.MustQuery("select * from t2 order by c1;").Check(testkit.Rows("0 0", "2 2")) + + // Rows in both t1 and t2 are matched, so will be deleted even if it's null. + // NOTE: The null values are not generated by join. + tk.MustExec("drop table if exists t1, t2;") + tk.MustExec("create table t1 (c1 int);") + tk.MustExec("create table t2 (c2 int);") + tk.MustExec("insert into t1 values(null);") + tk.MustExec("insert into t2 values(null);") + tk.MustExec("delete from t1, t2 using t1 join t2 where t1.c1 is null;") + tk.MustQuery("select * from t1;").Check(testkit.Rows()) + tk.MustQuery("select * from t2;").Check(testkit.Rows()) +} diff --git a/executor/explain_test.go b/executor/explain_test.go index a0ce1a1eb2e0c..ca7923e11049b 100644 --- a/executor/explain_test.go +++ b/executor/explain_test.go @@ -351,3 +351,24 @@ func (s *testSuite2) TestExplainAnalyzeCTEMemoryAndDiskInfo(c *C) { c.Assert(rows[4][7].(string), Not(Equals), "N/A") c.Assert(rows[4][8].(string), Not(Equals), "N/A") } + +func (s *testSuite) TestFix29401(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists tt123;") + tk.MustExec(`CREATE TABLE tt123 ( + id int(11) NOT NULL, + a bigint(20) DEFAULT NULL, + b char(20) DEFAULT NULL, + c datetime DEFAULT NULL, + d double DEFAULT NULL, + e json DEFAULT NULL, + f decimal(40,6) DEFAULT NULL, + PRIMARY KEY (id) /*T![clustered_index] CLUSTERED */, + KEY a (a), + KEY b (b), + KEY c (c), + KEY d (d), + KEY f (f) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;`) + tk.MustExec(" explain select /*+ inl_hash_join(t1) */ * from tt123 t1 join tt123 t2 on t1.b=t2.e;") +} diff --git a/executor/hash_table.go b/executor/hash_table.go index cd356d0549a5f..8d59da9855b8b 100644 --- a/executor/hash_table.go +++ b/executor/hash_table.go @@ -33,6 +33,7 @@ import ( // hashContext keeps the needed hash context of a db table in hash join. type hashContext struct { + // allTypes one-to-one correspondence with keyColIdx allTypes []*types.FieldType keyColIdx []int buf []byte @@ -80,9 +81,9 @@ type hashRowContainer struct { rowContainer *chunk.RowContainer } -func newHashRowContainer(sCtx sessionctx.Context, estCount int, hCtx *hashContext) *hashRowContainer { +func newHashRowContainer(sCtx sessionctx.Context, estCount int, hCtx *hashContext, allTypes []*types.FieldType) *hashRowContainer { maxChunkSize := sCtx.GetSessionVars().MaxChunkSize - rc := chunk.NewRowContainer(hCtx.allTypes, maxChunkSize) + rc := chunk.NewRowContainer(allTypes, maxChunkSize) c := &hashRowContainer{ sc: sCtx.GetSessionVars().StmtCtx, hCtx: hCtx, @@ -160,7 +161,7 @@ func (c *hashRowContainer) PutChunkSelected(chk *chunk.Chunk, selected, ignoreNu hCtx := c.hCtx for keyIdx, colIdx := range c.hCtx.keyColIdx { ignoreNull := len(ignoreNulls) > keyIdx && ignoreNulls[keyIdx] - err := codec.HashChunkSelected(c.sc, hCtx.hashVals, chk, hCtx.allTypes[colIdx], colIdx, hCtx.buf, hCtx.hasNull, selected, ignoreNull) + err := codec.HashChunkSelected(c.sc, hCtx.hashVals, chk, hCtx.allTypes[keyIdx], colIdx, hCtx.buf, hCtx.hasNull, selected, ignoreNull) if err != nil { return errors.Trace(err) } diff --git a/executor/hash_table_test.go b/executor/hash_table_test.go index 3458002987f1e..fbd4d1e6d2381 100644 --- a/executor/hash_table_test.go +++ b/executor/hash_table_test.go @@ -118,7 +118,7 @@ func (s *pkgTestSerialSuite) testHashRowContainer(c *C, hashFunc func() hash.Has for i := 0; i < numRows; i++ { hCtx.hashVals = append(hCtx.hashVals, hashFunc()) } - rowContainer := newHashRowContainer(sctx, 0, hCtx) + rowContainer := newHashRowContainer(sctx, 0, hCtx, hCtx.allTypes) tracker := rowContainer.GetMemTracker() tracker.SetLabel(memory.LabelForBuildSideResult) if spill { diff --git a/executor/index_lookup_hash_join.go b/executor/index_lookup_hash_join.go index 62fa460c39111..0332e3186a2d8 100644 --- a/executor/index_lookup_hash_join.go +++ b/executor/index_lookup_hash_join.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/expression" plannercore "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/codec" @@ -151,6 +152,7 @@ func (e *IndexNestedLoopHashJoin) Open(ctx context.Context) error { e.stats = &indexLookUpJoinRuntimeStats{} e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats) } + e.finished.Store(false) e.startWorkers(ctx) return nil } @@ -200,6 +202,7 @@ func (e *IndexNestedLoopHashJoin) startWorkers(ctx context.Context) { func (e *IndexNestedLoopHashJoin) finishJoinWorkers(r interface{}) { if r != nil { + e.IndexLookUpJoin.finished.Store(true) err := errors.New(fmt.Sprintf("%v", r)) if !e.keepOuterOrder { e.resultCh <- &indexHashJoinResult{err: err} @@ -318,6 +321,7 @@ func (e *IndexNestedLoopHashJoin) Close() error { close(e.joinChkResourceCh[i]) } e.joinChkResourceCh = nil + e.finished.Store(false) return e.baseExecutor.Close() } @@ -325,6 +329,7 @@ func (ow *indexHashJoinOuterWorker) run(ctx context.Context) { defer trace.StartRegion(ctx, "IndexHashJoinOuterWorker").End() defer close(ow.innerCh) for { + failpoint.Inject("TestIssue30211", nil) task, err := ow.buildTask(ctx) failpoint.Inject("testIndexHashJoinOuterWorkerErr", func() { err = errors.New("mockIndexHashJoinOuterWorkerErr") @@ -332,7 +337,10 @@ func (ow *indexHashJoinOuterWorker) run(ctx context.Context) { if err != nil { task = &indexHashJoinTask{err: err} if ow.keepOuterOrder { - task.keepOuterOrder, task.resultCh = true, make(chan *indexHashJoinResult, 1) + // The outerBuilder and innerFetcher run concurrently, we may + // get 2 errors at simultaneously. Thus the capacity of task.resultCh + // needs to be initialized to 2 to avoid waiting. + task.keepOuterOrder, task.resultCh = true, make(chan *indexHashJoinResult, 2) ow.pushToChan(ctx, task, ow.taskCh) } ow.pushToChan(ctx, task, ow.innerCh) @@ -431,6 +439,8 @@ func (e *IndexNestedLoopHashJoin) newInnerWorker(taskCh chan *indexHashJoinTask, indexRanges: copiedRanges, keyOff2IdxOff: e.keyOff2IdxOff, stats: innerStats, + lookup: &e.IndexLookUpJoin, + memTracker: memory.NewTracker(memory.LabelForIndexJoinInnerWorker, -1), }, taskCh: taskCh, joiner: e.joiners[workerID], @@ -440,6 +450,14 @@ func (e *IndexNestedLoopHashJoin) newInnerWorker(taskCh chan *indexHashJoinTask, joinKeyBuf: make([]byte, 1), outerRowStatus: make([]outerRowStatusFlag, 0, e.maxChunkSize), } + iw.memTracker.AttachTo(e.memTracker) + if len(copiedRanges) != 0 { + // We should not consume this memory usage in `iw.memTracker`. The + // memory usage of inner worker will be reset the end of iw.handleTask. + // While the life cycle of this memory consumption exists throughout the + // whole active period of inner worker. + e.ctx.GetSessionVars().StmtCtx.MemTracker.Consume(2 * types.EstimatedMemUsage(copiedRanges[0].LowVal, len(copiedRanges))) + } if e.lastColHelper != nil { // nextCwf.TmpConstant needs to be reset for every individual // inner worker to avoid data race when the inner workers is running @@ -499,19 +517,26 @@ func (iw *indexHashJoinInnerWorker) run(ctx context.Context, cancelFunc context. failpoint.Inject("testIndexHashJoinInnerWorkerErr", func() { joinResult.err = errors.New("mockIndexHashJoinInnerWorkerErr") }) - if joinResult.err != nil { - resultCh <- joinResult - return - } - // When task.keepOuterOrder is TRUE(resultCh != iw.resultCh), the last - // joinResult will be checked when the a task has been processed, thus we do - // not need to check it here again. - if resultCh == iw.resultCh && joinResult.chk != nil && joinResult.chk.NumRows() > 0 { - select { - case resultCh <- joinResult: - case <-ctx.Done(): + // When task.keepOuterOrder is TRUE (resultCh != iw.resultCh): + // - the last joinResult will be handled when the task has been processed, + // thus we DO NOT need to check it here again. + // - we DO NOT check the error here neither, because: + // - if the error is from task.err, the main thread will check the error of each task + // - if the error is from handleTask, the error will be handled in handleTask + // We should not check `task != nil && !task.keepOuterOrder` here since it's + // possible that `join.chk.NumRows > 0` is true even if task == nil. + if resultCh == iw.resultCh { + if joinResult.err != nil { + resultCh <- joinResult return } + if joinResult.chk != nil && joinResult.chk.NumRows() > 0 { + select { + case resultCh <- joinResult: + case <-ctx.Done(): + return + } + } } } @@ -529,6 +554,7 @@ func (iw *indexHashJoinInnerWorker) getNewJoinResult(ctx context.Context) (*inde } func (iw *indexHashJoinInnerWorker) buildHashTableForOuterResult(ctx context.Context, task *indexHashJoinTask, h hash.Hash64) { + failpoint.Inject("IndexHashJoinBuildHashTablePanic", nil) if iw.stats != nil { start := time.Now() defer func() { @@ -540,6 +566,9 @@ func (iw *indexHashJoinInnerWorker) buildHashTableForOuterResult(ctx context.Con for chkIdx := 0; chkIdx < numChks; chkIdx++ { chk := task.outerResult.GetChunk(chkIdx) numRows := chk.NumRows() + if iw.lookup.finished.Load().(bool) { + return + } OUTER: for rowIdx := 0; rowIdx < numRows; rowIdx++ { if task.outerMatch != nil && !task.outerMatch[chkIdx][rowIdx] { @@ -553,7 +582,7 @@ func (iw *indexHashJoinInnerWorker) buildHashTableForOuterResult(ctx context.Con } } h.Reset() - err := codec.HashChunkRow(iw.ctx.GetSessionVars().StmtCtx, h, row, iw.outerCtx.rowTypes, hashColIdx, buf) + err := codec.HashChunkRow(iw.ctx.GetSessionVars().StmtCtx, h, row, iw.outerCtx.hashTypes, hashColIdx, buf) failpoint.Inject("testIndexHashJoinBuildErr", func() { err = errors.New("mockIndexHashJoinBuildErr") }) @@ -575,14 +604,27 @@ func (iw *indexHashJoinInnerWorker) fetchInnerResults(ctx context.Context, task return iw.innerWorker.fetchInnerResults(ctx, task, lookUpContents) } -func (iw *indexHashJoinInnerWorker) handleHashJoinInnerWorkerPanic(r interface{}) { - if r != nil { - iw.resultCh <- &indexHashJoinResult{err: errors.Errorf("%v", r)} +func (iw *indexHashJoinInnerWorker) handleHashJoinInnerWorkerPanic(resultCh chan *indexHashJoinResult, err error) { + defer func() { + iw.wg.Done() + iw.lookup.workerWg.Done() + }() + if err != nil { + resultCh <- &indexHashJoinResult{err: err} } - iw.wg.Done() } -func (iw *indexHashJoinInnerWorker) handleTask(ctx context.Context, task *indexHashJoinTask, joinResult *indexHashJoinResult, h hash.Hash64, resultCh chan *indexHashJoinResult) error { +func (iw *indexHashJoinInnerWorker) handleTask(ctx context.Context, task *indexHashJoinTask, joinResult *indexHashJoinResult, h hash.Hash64, resultCh chan *indexHashJoinResult) (err error) { + defer func() { + iw.memTracker.Consume(-iw.memTracker.BytesConsumed()) + if task.keepOuterOrder { + if err != nil { + joinResult.err = err + resultCh <- joinResult + } + close(resultCh) + } + }() var joinStartTime time.Time if iw.stats != nil { start := time.Now() @@ -596,12 +638,29 @@ func (iw *indexHashJoinInnerWorker) handleTask(ctx context.Context, task *indexH iw.wg = &sync.WaitGroup{} iw.wg.Add(1) // TODO(XuHuaiyu): we may always use the smaller side to build the hashtable. - go util.WithRecovery(func() { iw.buildHashTableForOuterResult(ctx, task, h) }, iw.handleHashJoinInnerWorkerPanic) - err := iw.fetchInnerResults(ctx, task.lookUpJoinTask) + go util.WithRecovery( + func() { + iw.lookup.workerWg.Add(1) + iw.buildHashTableForOuterResult(ctx, task, h) + }, + func(r interface{}) { + var err error + if r != nil { + err = errors.Errorf("%v", r) + } + iw.handleHashJoinInnerWorkerPanic(resultCh, err) + }, + ) + err = iw.fetchInnerResults(ctx, task.lookUpJoinTask) + iw.wg.Wait() + failpoint.Inject("IndexHashJoinFetchInnerResultsErr", func() { + err = errors.New("IndexHashJoinFetchInnerResultsErr") + }) + // check error after wg.Wait to make sure error message can be sent to + // resultCh even if panic happen in buildHashTableForOuterResult. if err != nil { return err } - iw.wg.Wait() joinStartTime = time.Now() if !task.keepOuterOrder { @@ -644,7 +703,7 @@ func (iw *indexHashJoinInnerWorker) doJoinUnordered(ctx context.Context, task *i func (iw *indexHashJoinInnerWorker) getMatchedOuterRows(innerRow chunk.Row, task *indexHashJoinTask, h hash.Hash64, buf []byte) (matchedRows []chunk.Row, matchedRowPtr []chunk.RowPtr, err error) { h.Reset() - err = codec.HashChunkRow(iw.ctx.GetSessionVars().StmtCtx, h, innerRow, iw.rowTypes, iw.hashCols, buf) + err = codec.HashChunkRow(iw.ctx.GetSessionVars().StmtCtx, h, innerRow, iw.hashTypes, iw.hashCols, buf) if err != nil { return nil, nil, err } @@ -658,7 +717,7 @@ func (iw *indexHashJoinInnerWorker) getMatchedOuterRows(innerRow chunk.Row, task matchedRowPtr = make([]chunk.RowPtr, 0, len(iw.matchedOuterPtrs)) for _, ptr := range iw.matchedOuterPtrs { outerRow := task.outerResult.GetRow(ptr) - ok, err := codec.EqualChunkRow(iw.ctx.GetSessionVars().StmtCtx, innerRow, iw.rowTypes, iw.keyCols, outerRow, iw.outerCtx.rowTypes, iw.outerCtx.hashCols) + ok, err := codec.EqualChunkRow(iw.ctx.GetSessionVars().StmtCtx, innerRow, iw.hashTypes, iw.hashCols, outerRow, iw.outerCtx.hashTypes, iw.outerCtx.hashCols) if err != nil { return nil, nil, err } @@ -745,13 +804,15 @@ func (iw *indexHashJoinInnerWorker) doJoinInOrder(ctx context.Context, task *ind joinResult.src <- joinResult.chk } } - close(resultCh) }() for i, numChunks := 0, task.innerResult.NumChunks(); i < numChunks; i++ { for j, chk := 0, task.innerResult.GetChunk(i); j < chk.NumRows(); j++ { row := chk.GetRow(j) ptr := chunk.RowPtr{ChkIdx: uint32(i), RowIdx: uint32(j)} err = iw.collectMatchedInnerPtrs4OuterRows(ctx, row, ptr, task, h, iw.joinKeyBuf) + failpoint.Inject("TestIssue31129", func() { + err = errors.New("TestIssue31129") + }) if err != nil { return err } diff --git a/executor/index_lookup_join.go b/executor/index_lookup_join.go index 6f62fcff339cb..c494da48d5d48 100644 --- a/executor/index_lookup_join.go +++ b/executor/index_lookup_join.go @@ -26,6 +26,7 @@ import ( "unsafe" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/expression" @@ -81,14 +82,16 @@ type IndexLookUpJoin struct { memTracker *memory.Tracker // track memory usage. - stats *indexLookUpJoinRuntimeStats + stats *indexLookUpJoinRuntimeStats + finished *atomic.Value } type outerCtx struct { - rowTypes []*types.FieldType - keyCols []int - hashCols []int - filter expression.CNFExprs + rowTypes []*types.FieldType + keyCols []int + hashTypes []*types.FieldType + hashCols []int + filter expression.CNFExprs } type innerCtx struct { @@ -96,6 +99,7 @@ type innerCtx struct { rowTypes []*types.FieldType keyCols []int keyColIDs []int64 // the original ID in its table, used by dynamic partition pruning + hashTypes []*types.FieldType hashCols []int colLens []int hasPrefixCol bool @@ -142,11 +146,13 @@ type innerWorker struct { outerCtx outerCtx ctx sessionctx.Context executorChk *chunk.Chunk + lookup *IndexLookUpJoin indexRanges []*ranger.Range nextColCompareFilters *plannercore.ColWithCmpFuncManager keyOff2IdxOff []int stats *innerWorkerRuntimeStats + memTracker *memory.Tracker } // Open implements the Executor interface. @@ -158,6 +164,7 @@ func (e *IndexLookUpJoin) Open(ctx context.Context) error { e.memTracker = memory.NewTracker(e.id, -1) e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) e.innerPtrBytes = make([][]byte, 0, 8) + e.finished.Store(false) if e.runtimeStats != nil { e.stats = &indexLookUpJoinRuntimeStats{} e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats) @@ -219,6 +226,16 @@ func (e *IndexLookUpJoin) newInnerWorker(taskCh chan *lookUpJoinTask) *innerWork indexRanges: copiedRanges, keyOff2IdxOff: e.keyOff2IdxOff, stats: innerStats, + lookup: e, + memTracker: memory.NewTracker(memory.LabelForIndexJoinInnerWorker, -1), + } + iw.memTracker.AttachTo(e.memTracker) + if len(copiedRanges) != 0 { + // We should not consume this memory usage in `iw.memTracker`. The + // memory usage of inner worker will be reset the end of iw.handleTask. + // While the life cycle of this memory consumption exists throughout the + // whole active period of inner worker. + e.ctx.GetSessionVars().StmtCtx.MemTracker.Consume(2 * types.EstimatedMemUsage(copiedRanges[0].LowVal, len(copiedRanges))) } if e.lastColHelper != nil { // nextCwf.TmpConstant needs to be reset for every individual @@ -330,12 +347,14 @@ func (ow *outerWorker) run(ctx context.Context, wg *sync.WaitGroup) { defer trace.StartRegion(ctx, "IndexLookupJoinOuterWorker").End() defer func() { if r := recover(); r != nil { + ow.lookup.finished.Store(true) buf := make([]byte, 4096) stackSize := runtime.Stack(buf, false) buf = buf[:stackSize] logutil.Logger(ctx).Error("outerWorker panicked", zap.String("stack", string(buf))) task := &lookUpJoinTask{doneCh: make(chan error, 1)} - task.doneCh <- errors.Errorf("%v", r) + err := errors.Errorf("%v", r) + task.doneCh <- err ow.pushToChan(ctx, task, ow.resultCh) } close(ow.resultCh) @@ -343,6 +362,7 @@ func (ow *outerWorker) run(ctx context.Context, wg *sync.WaitGroup) { wg.Done() }() for { + failpoint.Inject("TestIssue30211", nil) task, err := ow.buildTask(ctx) if err != nil { task.doneCh <- err @@ -447,12 +467,14 @@ func (iw *innerWorker) run(ctx context.Context, wg *sync.WaitGroup) { var task *lookUpJoinTask defer func() { if r := recover(); r != nil { + iw.lookup.finished.Store(true) buf := make([]byte, 4096) stackSize := runtime.Stack(buf, false) buf = buf[:stackSize] logutil.Logger(ctx).Error("innerWorker panicked", zap.String("stack", string(buf))) + err := errors.Errorf("%v", r) // "task != nil" is guaranteed when panic happened. - task.doneCh <- errors.Errorf("%v", r) + task.doneCh <- err } wg.Done() }() @@ -486,6 +508,9 @@ func (iw *innerWorker) handleTask(ctx context.Context, task *lookUpJoinTask) err atomic.AddInt64(&iw.stats.totalTime, int64(time.Since(start))) }() } + defer func() { + iw.memTracker.Consume(-iw.memTracker.BytesConsumed()) + }() lookUpContents, err := iw.constructLookupContent(task) if err != nil { return err @@ -519,6 +544,9 @@ func (iw *innerWorker) constructLookupContent(task *lookUpJoinTask) ([]*indexJoi if err != nil { return nil, err } + if rowIdx == 0 { + iw.lookup.memTracker.Consume(types.EstimatedMemUsage(dLookUpKey, numRows)) + } if dHashKey == nil { // Append null to make looUpKeys the same length as outer Result. task.encodedLookUpKeys[chkIdx].AppendNull(0) @@ -643,7 +671,7 @@ func (iw *innerWorker) fetchInnerResults(ctx context.Context, task *lookUpJoinTa atomic.AddInt64(&iw.stats.fetch, int64(time.Since(start))) }() } - innerExec, err := iw.readerBuilder.buildExecutorForIndexJoin(ctx, lookUpContent, iw.indexRanges, iw.keyOff2IdxOff, iw.nextColCompareFilters, true) + innerExec, err := iw.readerBuilder.buildExecutorForIndexJoin(ctx, lookUpContent, iw.indexRanges, iw.keyOff2IdxOff, iw.nextColCompareFilters, true, iw.memTracker, iw.lookup.finished) if innerExec != nil { defer terror.Call(innerExec.Close) } @@ -725,6 +753,7 @@ func (e *IndexLookUpJoin) Close() error { e.workerWg.Wait() e.memTracker = nil e.task = nil + e.finished.Store(false) return e.baseExecutor.Close() } diff --git a/executor/index_lookup_merge_join.go b/executor/index_lookup_merge_join.go index a16ffca0a771d..83133eb1f6a00 100644 --- a/executor/index_lookup_merge_join.go +++ b/executor/index_lookup_merge_join.go @@ -501,7 +501,7 @@ func (imw *innerMergeWorker) handleTask(ctx context.Context, task *lookUpMergeJo dLookUpKeys[i], dLookUpKeys[lenKeys-i-1] = dLookUpKeys[lenKeys-i-1], dLookUpKeys[i] } } - imw.innerExec, err = imw.readerBuilder.buildExecutorForIndexJoin(ctx, dLookUpKeys, imw.indexRanges, imw.keyOff2IdxOff, imw.nextColCompareFilters, false) + imw.innerExec, err = imw.readerBuilder.buildExecutorForIndexJoin(ctx, dLookUpKeys, imw.indexRanges, imw.keyOff2IdxOff, imw.nextColCompareFilters, false, nil, nil) if imw.innerExec != nil { defer terror.Call(imw.innerExec.Close) } diff --git a/executor/insert.go b/executor/insert.go index c6195ccef34c9..7c7b18f70de21 100644 --- a/executor/insert.go +++ b/executor/insert.go @@ -346,7 +346,7 @@ func (e *InsertExec) initEvalBuffer4Dup() { evalBufferTypes = append(evalBufferTypes, &col.FieldType) } if extraLen > 0 { - evalBufferTypes = append(evalBufferTypes, e.SelectExec.base().retFieldTypes[numWritableCols:]...) + evalBufferTypes = append(evalBufferTypes, e.SelectExec.base().retFieldTypes[e.rowLen:]...) } for _, col := range e.Table.Cols() { evalBufferTypes = append(evalBufferTypes, &col.FieldType) @@ -355,7 +355,7 @@ func (e *InsertExec) initEvalBuffer4Dup() { evalBufferTypes = append(evalBufferTypes, types.NewFieldType(mysql.TypeLonglong)) } e.evalBuffer4Dup = chunk.MutRowFromTypes(evalBufferTypes) - e.curInsertVals = chunk.MutRowFromTypes(evalBufferTypes[numWritableCols:]) + e.curInsertVals = chunk.MutRowFromTypes(evalBufferTypes[numWritableCols+extraLen:]) e.row4Update = make([]types.Datum, 0, len(evalBufferTypes)) } diff --git a/executor/insert_test.go b/executor/insert_test.go index 24344903ca52f..da9b2063a8816 100644 --- a/executor/insert_test.go +++ b/executor/insert_test.go @@ -208,6 +208,36 @@ func (s *testSuite8) TestInsertOnDuplicateKey(c *C) { c.Assert(tk.Se.AffectedRows(), Equals, uint64(2)) tk.MustQuery("select * from a").Check(testkit.Rows("2")) + // Test issue 28078. + // Use different types of columns so that there's likely to be error if the types mismatches. + tk.MustExec("drop table if exists a, b") + tk.MustExec("create table a(id int, a1 timestamp, a2 varchar(10), a3 float, unique(id))") + tk.MustExec("create table b(id int, b1 time, b2 varchar(10), b3 int)") + tk.MustExec("insert into a values (1, '2022-01-04 07:02:04', 'a', 1.1), (2, '2022-01-04 07:02:05', 'b', 2.2)") + tk.MustExec("insert into b values (2, '12:34:56', 'c', 10), (3, '01:23:45', 'd', 20)") + tk.MustExec("insert into a (id) select id from b on duplicate key update a.a2 = b.b2, a.a3 = 3.3") + c.Assert(tk.Se.AffectedRows(), Equals, uint64(3)) + tk.MustQuery("select * from a").Check(testutil.RowsWithSep("/", + "1/2022-01-04 07:02:04/a/1.1", + "2/2022-01-04 07:02:05/c/3.3", + "3///")) + tk.MustExec("insert into a (id) select 4 from b where b3 = 20 on duplicate key update a.a3 = b.b3") + c.Assert(tk.Se.AffectedRows(), Equals, uint64(1)) + tk.MustQuery("select * from a").Check(testutil.RowsWithSep("/", + "1/2022-01-04 07:02:04/a/1.1", + "2/2022-01-04 07:02:05/c/3.3", + "3///", + "4///")) + tk.MustExec("insert into a (a2, a3) select 'x', 1.2 from b on duplicate key update a.a2 = b.b3") + c.Assert(tk.Se.AffectedRows(), Equals, uint64(2)) + tk.MustQuery("select * from a").Check(testutil.RowsWithSep("/", + "1/2022-01-04 07:02:04/a/1.1", + "2/2022-01-04 07:02:05/c/3.3", + "3///", + "4///", + "//x/1.2", + "//x/1.2")) + // reproduce insert on duplicate key update bug under new row format. tk.MustExec(`drop table if exists t1`) tk.MustExec(`create table t1(c1 decimal(6,4), primary key(c1))`) diff --git a/executor/join.go b/executor/join.go index 1a3f62de47ac1..01e0ef3653f8f 100644 --- a/executor/join.go +++ b/executor/join.go @@ -571,7 +571,7 @@ func (e *HashJoinExec) join2Chunk(workerID uint, probeSideChk *chunk.Chunk, hCtx hCtx.initHash(probeSideChk.NumRows()) for keyIdx, i := range hCtx.keyColIdx { ignoreNull := len(e.isNullEQ) > keyIdx && e.isNullEQ[keyIdx] - err = codec.HashChunkSelected(e.rowContainer.sc, hCtx.hashVals, probeSideChk, hCtx.allTypes[i], i, hCtx.buf, hCtx.hasNull, selected, ignoreNull) + err = codec.HashChunkSelected(e.rowContainer.sc, hCtx.hashVals, probeSideChk, hCtx.allTypes[keyIdx], i, hCtx.buf, hCtx.hasNull, selected, ignoreNull) if err != nil { joinResult.err = err return false, joinResult @@ -612,8 +612,8 @@ func (e *HashJoinExec) join2Chunk(workerID uint, probeSideChk *chunk.Chunk, hCtx // join2ChunkForOuterHashJoin joins chunks when using the outer to build a hash table (refer to outer hash join) func (e *HashJoinExec) join2ChunkForOuterHashJoin(workerID uint, probeSideChk *chunk.Chunk, hCtx *hashContext, joinResult *hashjoinWorkerResult) (ok bool, _ *hashjoinWorkerResult) { hCtx.initHash(probeSideChk.NumRows()) - for _, i := range hCtx.keyColIdx { - err := codec.HashChunkColumns(e.rowContainer.sc, hCtx.hashVals, probeSideChk, hCtx.allTypes[i], i, hCtx.buf, hCtx.hasNull) + for keyIdx, i := range hCtx.keyColIdx { + err := codec.HashChunkColumns(e.rowContainer.sc, hCtx.hashVals, probeSideChk, hCtx.allTypes[keyIdx], i, hCtx.buf, hCtx.hasNull) if err != nil { joinResult.err = err return false, joinResult @@ -740,7 +740,7 @@ func (e *HashJoinExec) buildHashTableForList(buildSideResultCh <-chan *chunk.Chu } var err error var selected []bool - e.rowContainer = newHashRowContainer(e.ctx, int(e.buildSideEstCount), hCtx) + e.rowContainer = newHashRowContainer(e.ctx, int(e.buildSideEstCount), hCtx, retTypes(e.buildSideExec)) e.rowContainer.GetMemTracker().AttachTo(e.memTracker) e.rowContainer.GetMemTracker().SetLabel(memory.LabelForBuildSideResult) e.rowContainer.GetDiskTracker().AttachTo(e.diskTracker) diff --git a/executor/join_test.go b/executor/join_test.go index a5ca48e7b1976..f27af19c80b72 100644 --- a/executor/join_test.go +++ b/executor/join_test.go @@ -655,6 +655,39 @@ func (s *testSuiteJoin1) TestUsing(c *C) { tk.MustQuery("select t1.t0, t2.t0 from t1 join t2 using(t0) having t1.t0 > 0").Check(testkit.Rows("1 1")) } +func (s *testSuiteWithData) TestUsingAndNaturalJoinSchema(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1, t2, t3, t4") + tk.MustExec("create table t1 (c int, b int);") + tk.MustExec("create table t2 (a int, b int);") + tk.MustExec("create table t3 (b int, c int);") + tk.MustExec("create table t4 (y int, c int);") + + tk.MustExec("insert into t1 values (10,1);") + tk.MustExec("insert into t1 values (3 ,1);") + tk.MustExec("insert into t1 values (3 ,2);") + tk.MustExec("insert into t2 values (2, 1);") + tk.MustExec("insert into t3 values (1, 3);") + tk.MustExec("insert into t3 values (1,10);") + tk.MustExec("insert into t4 values (11,3);") + tk.MustExec("insert into t4 values (2, 3);") + + var input []string + var output []struct { + SQL string + Res []string + } + s.testData.GetTestCases(c, &input, &output) + for i, tt := range input { + s.testData.OnRecord(func() { + output[i].SQL = tt + output[i].Res = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Sort().Rows()) + }) + tk.MustQuery(tt).Sort().Check(testkit.Rows(output[i].Res...)) + } +} + func (s *testSuiteWithData) TestNaturalJoin(c *C) { tk := testkit.NewTestKit(c, s.store) @@ -2609,3 +2642,88 @@ func (s *testSuiteJoinSerial) TestIssue25902(c *C) { tk.MustQuery("select * from tt1 where ts in (select ts from tt2);").Check(testkit.Rows()) tk.MustExec("set @@session.time_zone = @tmp;") } + +func (s *testSuiteJoinSerial) TestIssue30211(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1, t2;") + tk.MustExec("create table t1(a int, index(a));") + tk.MustExec("create table t2(a int, index(a));") + func() { + fpName := "github.com/pingcap/tidb/executor/TestIssue30211" + c.Assert(failpoint.Enable(fpName, `panic("TestIssue30211 IndexJoinPanic")`), IsNil) + defer func() { + c.Assert(failpoint.Disable(fpName), IsNil) + }() + err := tk.QueryToErr("select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.a;").Error() + c.Assert(err, Matches, "failpoint panic: TestIssue30211 IndexJoinPanic") + + err = tk.QueryToErr("select /*+ inl_hash_join(t1) */ * from t1 join t2 on t1.a = t2.a;").Error() + c.Assert(err, Matches, "failpoint panic: TestIssue30211 IndexJoinPanic") + }() + tk.MustExec("insert into t1 values(1),(2);") + tk.MustExec("insert into t2 values(1),(1),(2),(2);") + tk.MustExec("set @@tidb_mem_quota_query=8000;") + tk.MustExec("set tidb_index_join_batch_size = 1;") + config.UpdateGlobal(func(conf *config.Config) { + conf.OOMAction = config.OOMActionCancel + }) + defer func() { + config.UpdateGlobal(func(conf *config.Config) { + conf.OOMAction = config.OOMActionLog + }) + }() + err := tk.QueryToErr("select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.a;").Error() + c.Assert(strings.Contains(err, "Out Of Memory Quota"), IsTrue) + err = tk.QueryToErr("select /*+ inl_hash_join(t1) */ * from t1 join t2 on t1.a = t2.a;").Error() + c.Assert(strings.Contains(err, "Out Of Memory Quota"), IsTrue) +} + +func (s *testSuiteJoinSerial) TestIssue31129(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("set @@tidb_init_chunk_size=2") + tk.MustExec("set @@tidb_index_join_batch_size=10") + tk.MustExec("DROP TABLE IF EXISTS t, s") + tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeIntOnly + tk.MustExec("create table t(pk int primary key, a int)") + for i := 0; i < 100; i++ { + tk.MustExec(fmt.Sprintf("insert into t values(%d, %d)", i, i)) + } + tk.MustExec("create table s(a int primary key)") + for i := 0; i < 100; i++ { + tk.MustExec(fmt.Sprintf("insert into s values(%d)", i)) + } + tk.MustExec("analyze table t") + tk.MustExec("analyze table s") + + // Test IndexNestedLoopHashJoin keepOrder. + fpName := "github.com/pingcap/tidb/executor/TestIssue31129" + c.Assert(failpoint.Enable(fpName, "return"), IsNil) + err := tk.QueryToErr("select /*+ INL_HASH_JOIN(s) */ * from t left join s on t.a=s.a order by t.pk") + c.Assert(strings.Contains(err.Error(), "TestIssue31129"), IsTrue) + c.Assert(failpoint.Disable(fpName), IsNil) + + // Test IndexNestedLoopHashJoin build hash table panic. + fpName = "github.com/pingcap/tidb/executor/IndexHashJoinBuildHashTablePanic" + c.Assert(failpoint.Enable(fpName, `panic("IndexHashJoinBuildHashTablePanic")`), IsNil) + err = tk.QueryToErr("select /*+ INL_HASH_JOIN(s) */ * from t left join s on t.a=s.a order by t.pk") + c.Assert(strings.Contains(err.Error(), "IndexHashJoinBuildHashTablePanic"), IsTrue) + c.Assert(failpoint.Disable(fpName), IsNil) + + // Test IndexNestedLoopHashJoin fetch inner fail. + fpName = "github.com/pingcap/tidb/executor/IndexHashJoinFetchInnerResultsErr" + c.Assert(failpoint.Enable(fpName, "return"), IsNil) + err = tk.QueryToErr("select /*+ INL_HASH_JOIN(s) */ * from t left join s on t.a=s.a order by t.pk") + c.Assert(strings.Contains(err.Error(), "IndexHashJoinFetchInnerResultsErr"), IsTrue) + c.Assert(failpoint.Disable(fpName), IsNil) + + // Test IndexNestedLoopHashJoin build hash table panic and IndexNestedLoopHashJoin fetch inner fail at the same time. + fpName1, fpName2 := "github.com/pingcap/tidb/executor/IndexHashJoinBuildHashTablePanic", "github.com/pingcap/tidb/executor/IndexHashJoinFetchInnerResultsErr" + c.Assert(failpoint.Enable(fpName1, `panic("IndexHashJoinBuildHashTablePanic")`), IsNil) + c.Assert(failpoint.Enable(fpName2, "return"), IsNil) + err = tk.QueryToErr("select /*+ INL_HASH_JOIN(s) */ * from t left join s on t.a=s.a order by t.pk") + c.Assert(strings.Contains(err.Error(), "IndexHashJoinBuildHashTablePanic"), IsTrue) + c.Assert(failpoint.Disable(fpName1), IsNil) + c.Assert(failpoint.Disable(fpName2), IsNil) +} diff --git a/executor/prepared_test.go b/executor/prepared_test.go index e0e2c19ee0f22..7e2536de534d3 100644 --- a/executor/prepared_test.go +++ b/executor/prepared_test.go @@ -404,3 +404,46 @@ func (s *testPrepareSuite) TestPlanCacheWithDifferentVariableTypes(c *C) { } } } + +func (s *testSerialSuite) TestIssue28087And28162(c *C) { + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + tk := testkit.NewTestKit(c, store) + defer func() { + dom.Close() + store.Close() + }() + orgEnable := plannercore.PreparedPlanCacheEnabled() + defer func() { + plannercore.SetPreparedPlanCache(orgEnable) + }() + plannercore.SetPreparedPlanCache(true) + + // issue 28087 + tk.MustExec(`use test`) + tk.MustExec(`drop table if exists IDT_26207`) + tk.MustExec(`CREATE TABLE IDT_26207 (col1 bit(1))`) + tk.MustExec(`insert into IDT_26207 values(0x0), (0x1)`) + tk.MustExec(`prepare stmt from 'select t1.col1 from IDT_26207 as t1 left join IDT_26207 as t2 on t1.col1 = t2.col1 where t1.col1 in (?, ?, ?)'`) + tk.MustExec(`set @a=0x01, @b=0x01, @c=0x01`) + tk.MustQuery(`execute stmt using @a,@b,@c`).Check(testkit.Rows("\x01")) + tk.MustExec(`set @a=0x00, @b=0x00, @c=0x01`) + tk.MustQuery(`execute stmt using @a,@b,@c`).Check(testkit.Rows("\x00", "\x01")) + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) + + // issue 28162 + tk.MustExec(`drop table if exists IDT_MC21780`) + tk.MustExec(`CREATE TABLE IDT_MC21780 ( + COL1 timestamp NULL DEFAULT NULL, + COL2 timestamp NULL DEFAULT NULL, + COL3 timestamp NULL DEFAULT NULL, + KEY U_M_COL (COL1,COL2) + )`) + tk.MustExec(`insert into IDT_MC21780 values("1970-12-18 10:53:28", "1970-12-18 10:53:28", "1970-12-18 10:53:28")`) + tk.MustExec(`prepare stmt from 'select/*+ hash_join(t1) */ * from IDT_MC21780 t1 join IDT_MC21780 t2 on t1.col1 = t2.col1 where t1. col1 < ? and t2. col1 in (?, ?, ?);'`) + tk.MustExec(`set @a="2038-01-19 03:14:07", @b="2038-01-19 03:14:07", @c="2038-01-19 03:14:07", @d="2038-01-19 03:14:07"`) + tk.MustQuery(`execute stmt using @a,@b,@c,@d`).Check(testkit.Rows()) + tk.MustExec(`set @a="1976-09-09 20:21:11", @b="2021-07-14 09:28:16", @c="1982-01-09 03:36:39", @d="1970-12-18 10:53:28"`) + tk.MustQuery(`execute stmt using @a,@b,@c,@d`).Check(testkit.Rows("1970-12-18 10:53:28 1970-12-18 10:53:28 1970-12-18 10:53:28 1970-12-18 10:53:28 1970-12-18 10:53:28 1970-12-18 10:53:28")) + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) +} diff --git a/executor/seqtest/seq_executor_test.go b/executor/seqtest/seq_executor_test.go index bcecfc8d52ad4..fc1d58d4755ae 100644 --- a/executor/seqtest/seq_executor_test.go +++ b/executor/seqtest/seq_executor_test.go @@ -1578,3 +1578,20 @@ func (s *seqTestSuite) TestIssue19410(c *C) { tk.MustQuery("select /*+ INL_HASH_JOIN(t3) */ * from t join t3 on t.b = t3.b1;").Check(testkit.Rows("1 A 1 A")) tk.MustQuery("select /*+ INL_JOIN(t3) */ * from t join t3 on t.b = t3.b1;").Check(testkit.Rows("1 A 1 A")) } + +func (s *seqTestSuite) TestAnalyzeNextRawErrorNoLeak(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1(id int, c varchar(32))") + tk.MustExec("set @@session.tidb_analyze_version = 2") + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/distsql/mockNextRawError", `return(true)`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/distsql/mockNextRawError"), IsNil) + }() + + err := tk.ExecToErr("analyze table t1") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "mockNextRawError") +} diff --git a/executor/set.go b/executor/set.go index bc85bbbbc47ca..4e768aee9c3e6 100644 --- a/executor/set.go +++ b/executor/set.go @@ -180,7 +180,7 @@ func (e *SetExecutor) setSysVariable(ctx context.Context, name string, v *expres } } - err = e.loadSnapshotInfoSchemaIfNeeded(newSnapshotTS) + err = e.loadSnapshotInfoSchemaIfNeeded(name, newSnapshotTS) if err != nil { fallbackOldSnapshotTS() return err @@ -253,7 +253,10 @@ func (e *SetExecutor) getVarValue(v *expression.VarAssignment, sysVar *variable. return nativeVal.ToString() } -func (e *SetExecutor) loadSnapshotInfoSchemaIfNeeded(snapshotTS uint64) error { +func (e *SetExecutor) loadSnapshotInfoSchemaIfNeeded(name string, snapshotTS uint64) error { + if name != variable.TiDBSnapshot && name != variable.TiDBTxnReadTS { + return nil + } vars := e.ctx.GetSessionVars() if snapshotTS == 0 { vars.SnapshotInfoschema = nil diff --git a/executor/testdata/executor_suite_in.json b/executor/testdata/executor_suite_in.json index cd8fa234c0117..c8d609cdf8e31 100644 --- a/executor/testdata/executor_suite_in.json +++ b/executor/testdata/executor_suite_in.json @@ -40,6 +40,25 @@ "SELECT * FROM t1 NATURAL LEFT JOIN t2 WHERE not(t1.a <=> t2.a)" ] }, + { + "name": "TestUsingAndNaturalJoinSchema", + "cases": [ + "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) natural join (t3 natural join t4);", + "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) join (t3 natural join t4) using (b);", + "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) join (t3 natural join t4) using (c);", + "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) join (t3 natural join t4) using (c,b);", + "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) left outer join (t3 natural join t4) using (b);", + "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) left outer join (t3 natural join t4) using (c);", + "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) left outer join (t3 natural join t4) using (c,b);", + "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) right outer join (t3 natural join t4) using (b);", + "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) right outer join (t3 natural join t4) using (c);", + "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) right outer join (t3 natural join t4) using (c,b);", + "select * from (t1 natural join t2) natural join (t3 natural join t4);", + "select * from (t1 natural join t2) join (t3 natural join t4) using (b);", + "select * from (t1 natural join t2) left outer join (t3 natural join t4) using (b);", + "select * from (t1 natural join t2) right outer join (t3 natural join t4) using (c,b);" + ] + }, { "name": "TestIndexScanWithYearCol", "cases": [ diff --git a/executor/testdata/executor_suite_out.json b/executor/testdata/executor_suite_out.json index eab098751f2ba..d2ad1621f6049 100644 --- a/executor/testdata/executor_suite_out.json +++ b/executor/testdata/executor_suite_out.json @@ -519,6 +519,121 @@ } ] }, + { + "Name": "TestUsingAndNaturalJoinSchema", + "Cases": [ + { + "SQL": "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) natural join (t3 natural join t4);", + "Res": [ + "3 1 2 1 1 3 11 3", + "3 1 2 1 1 3 2 3" + ] + }, + { + "SQL": "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) join (t3 natural join t4) using (b);", + "Res": [ + "10 1 2 1 1 3 11 3", + "10 1 2 1 1 3 2 3", + "3 1 2 1 1 3 11 3", + "3 1 2 1 1 3 2 3" + ] + }, + { + "SQL": "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) join (t3 natural join t4) using (c);", + "Res": [ + "3 1 2 1 1 3 11 3", + "3 1 2 1 1 3 2 3" + ] + }, + { + "SQL": "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) join (t3 natural join t4) using (c,b);", + "Res": [ + "3 1 2 1 1 3 11 3", + "3 1 2 1 1 3 2 3" + ] + }, + { + "SQL": "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) left outer join (t3 natural join t4) using (b);", + "Res": [ + "10 1 2 1 1 3 11 3", + "10 1 2 1 1 3 2 3", + "3 1 2 1 1 3 11 3", + "3 1 2 1 1 3 2 3" + ] + }, + { + "SQL": "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) left outer join (t3 natural join t4) using (c);", + "Res": [ + "10 1 2 1 ", + "3 1 2 1 1 3 11 3", + "3 1 2 1 1 3 2 3" + ] + }, + { + "SQL": "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) left outer join (t3 natural join t4) using (c,b);", + "Res": [ + "10 1 2 1 ", + "3 1 2 1 1 3 11 3", + "3 1 2 1 1 3 2 3" + ] + }, + { + "SQL": "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) right outer join (t3 natural join t4) using (b);", + "Res": [ + "10 1 2 1 1 3 11 3", + "10 1 2 1 1 3 2 3", + "3 1 2 1 1 3 11 3", + "3 1 2 1 1 3 2 3" + ] + }, + { + "SQL": "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) right outer join (t3 natural join t4) using (c);", + "Res": [ + "3 1 2 1 1 3 11 3", + "3 1 2 1 1 3 2 3" + ] + }, + { + "SQL": "select t1.*, t2.*, t3.*, t4.* from (t1 natural join t2) right outer join (t3 natural join t4) using (c,b);", + "Res": [ + "3 1 2 1 1 3 11 3", + "3 1 2 1 1 3 2 3" + ] + }, + { + "SQL": "select * from (t1 natural join t2) natural join (t3 natural join t4);", + "Res": [ + "1 3 2 11", + "1 3 2 2" + ] + }, + { + "SQL": "select * from (t1 natural join t2) join (t3 natural join t4) using (b);", + "Res": [ + "1 10 2 3 11", + "1 10 2 3 2", + "1 3 2 3 11", + "1 3 2 3 2" + ] + }, + { + "SQL": "select * from (t1 natural join t2) left outer join (t3 natural join t4) using (b);", + "Res": [ + "1 10 2 3 11", + "1 10 2 3 2", + "1 3 2 3 11", + "1 3 2 3 2" + ] + }, + { + "SQL": "select * from (t1 natural join t2) right outer join (t3 natural join t4) using (c,b);", + "Res": [ + "3 1 11 2", + "3 1 2 2" + ] + } + ] + }, { "Name": "TestIndexScanWithYearCol", "Cases": [ diff --git a/executor/tiflash_test.go b/executor/tiflash_test.go index 7c2ba3c000c8e..1e618181d40ba 100644 --- a/executor/tiflash_test.go +++ b/executor/tiflash_test.go @@ -143,6 +143,35 @@ func (s *tiflashTestSuite) TestReadUnsigedPK(c *C) { tk.MustQuery("select count(*) from t1 , t where t1.a = t.a and ((t1.a < 9223372036854775800 and t1.a > 2) or (t1.a <= 1 and t1.a > -1))").Check(testkit.Rows("3")) } +// to fix https://github.com/pingcap/tidb/issues/27952 +func (s *tiflashTestSuite) TestJoinRace(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int not null, b int not null)") + tk.MustExec("alter table t set tiflash replica 1") + tb := testGetTableByName(c, tk.Se, "test", "t") + err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) + c.Assert(err, IsNil) + tk.MustExec("insert into t values(1,1)") + tk.MustExec("insert into t values(2,1)") + tk.MustExec("insert into t values(3,1)") + tk.MustExec("insert into t values(1,2)") + tk.MustExec("insert into t values(2,2)") + tk.MustExec("insert into t values(3,2)") + tk.MustExec("insert into t values(1,2)") + tk.MustExec("insert into t values(2,2)") + tk.MustExec("insert into t values(3,2)") + tk.MustExec("insert into t values(1,3)") + tk.MustExec("insert into t values(2,3)") + tk.MustExec("insert into t values(3,4)") + tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") + tk.MustExec("set @@session.tidb_enforce_mpp=ON") + tk.MustExec("set @@tidb_opt_broadcast_cartesian_join=0") + tk.MustQuery("select count(*) from (select count(a) x from t group by b) t1 join (select count(a) x from t group by b) t2 on t1.x > t2.x").Check(testkit.Rows("6")) + +} + func (s *tiflashTestSuite) TestMppExecution(c *C) { if israce.RaceEnabled { c.Skip("skip race test because of long running") @@ -661,6 +690,56 @@ func (s *tiflashTestSuite) TestMppUnionAll(c *C) { } +func (s *tiflashTestSuite) TestAvgOverflow(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + // avg decimal + tk.MustExec("drop table if exists td;") + tk.MustExec("create table td (col_bigint bigint(20), col_smallint smallint(6));") + tk.MustExec("alter table td set tiflash replica 1") + tb := testGetTableByName(c, tk.Se, "test", "td") + err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) + c.Assert(err, IsNil) + tk.MustExec("insert into td values (null, 22876);") + tk.MustExec("insert into td values (9220557287087669248, 32767);") + tk.MustExec("insert into td values (28030, 32767);") + tk.MustExec("insert into td values (-3309864251140603904,32767);") + tk.MustExec("insert into td values (4,0);") + tk.MustExec("insert into td values (null,0);") + tk.MustExec("insert into td values (4,-23828);") + tk.MustExec("insert into td values (54720,32767);") + tk.MustExec("insert into td values (0,29815);") + tk.MustExec("insert into td values (10017,-32661);") + tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") + tk.MustExec("set @@session.tidb_enforce_mpp=ON") + tk.MustQuery(" SELECT AVG( col_bigint / col_smallint) AS field1 FROM td;").Sort().Check(testkit.Rows("25769363061037.62077260")) + tk.MustQuery(" SELECT AVG(col_bigint) OVER (PARTITION BY col_smallint) as field2 FROM td where col_smallint = -23828;").Sort().Check(testkit.Rows("4.0000")) + tk.MustExec("drop table if exists td;") +} + +func (s *tiflashTestSuite) TestUnionWithEmptyDualTable(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t (a int not null, b int, c varchar(20))") + tk.MustExec("create table t1 (a int, b int not null, c double)") + tk.MustExec("alter table t set tiflash replica 1") + tk.MustExec("alter table t1 set tiflash replica 1") + tb := testGetTableByName(c, tk.Se, "test", "t") + err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) + c.Assert(err, IsNil) + tb = testGetTableByName(c, tk.Se, "test", "t1") + err = domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) + c.Assert(err, IsNil) + tk.MustExec("insert into t values(1,2,3)") + tk.MustExec("insert into t1 values(1,2,3)") + tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") + tk.MustExec("set @@session.tidb_enforce_mpp=ON") + tk.MustQuery("select count(*) from (select a , b from t union all select a , c from t1 where false) tt").Check(testkit.Rows("1")) +} + func (s *tiflashTestSuite) TestMppApply(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") diff --git a/executor/update.go b/executor/update.go index cbc61f6121b88..dffc05b8ddc2b 100644 --- a/executor/update.go +++ b/executor/update.go @@ -92,7 +92,7 @@ func (e *UpdateExec) prepare(row []types.Datum) (err error) { break } } - if e.unmatchedOuterRow(content, row) { + if unmatchedOuterRow(content, row) { updatable = false } e.tableUpdatable = append(e.tableUpdatable, updatable) @@ -207,7 +207,7 @@ func (e *UpdateExec) exec(ctx context.Context, schema *expression.Schema, row, n // the inner handle field is filled with a NULL value. // // This fixes: https://github.com/pingcap/tidb/issues/7176. -func (e *UpdateExec) unmatchedOuterRow(tblPos plannercore.TblColPosInfo, waitUpdateRow []types.Datum) bool { +func unmatchedOuterRow(tblPos plannercore.TblColPosInfo, waitUpdateRow []types.Datum) bool { firstHandleIdx := tblPos.HandleCols.GetCol(0) return waitUpdateRow[firstHandleIdx.Index].IsNull() } diff --git a/executor/window_test.go b/executor/window_test.go index 970d7f7354100..364d4d0682b2a 100644 --- a/executor/window_test.go +++ b/executor/window_test.go @@ -470,3 +470,23 @@ func (s *testSuite7) TestIssue24264(c *C) { "8297270320597030697", "")) } + +func (s *testSuite7) TestIssue29947(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec(`drop table if exists t_tir89b, t_vejdy`) + + tk.MustExec("CREATE TABLE `t_tir89b` (`c_3pcik` int(11) DEFAULT NULL,`c_0b6nxb` text DEFAULT NULL,`c_qytrlc` double NOT NULL,`c_sroc_c` int(11) DEFAULT NULL,PRIMARY KEY (`c_qytrlc`) /*T![clustered_index] NONCLUSTERED */ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;") + + tk.MustExec(`INSERT INTO t_tir89b VALUES (66,'cjd1o',87.77,NULL),(134217728,'d_unpd',76.66,NULL),(50,'_13gs',1.46,32),(49,'xclvsc',64.7,48),(7,'1an13',70.86,7),(29,NULL,6.26,6),(8,'hc485b',47.44,2),(84,'d_nlmd',99.3,76),(14,'lbny1c',61.1,47),(45,'9r5bid',25.37,95),(49,'jbz5r',72.99,49),(18,'uode3d',7.21,992),(-8945040,'ftrtib',47.47,20),(29,'algrj',6.28,24),(96,NULL,67.83,24),(5,'s1gfz',89.18,78),(74,'ggqbl',83.89,68),(61,'5n1q7',26.92,6),(10,'4gflb',33.84,28),(48,'xoe0cd',84.71,77),(6,'xkh6i',53.83,19),(5,NULL,89.1,46),(49,'4q6nx',31.5,384),(1,'pgs1',66.8,77),(19,'lltflc',33.49,63),(87,'vd4htc',39.92,-5367008),(47,NULL,28.3,10),(29,'15jqfc',100.11,64),(45,'ii6pm',52.41,61),(0,NULL,85.27,19),(104,'ikpxnb',40.66,955),(40,'gzryzd',36.23,42),(18,'7UPNE',84.27,14),(32,NULL,84.8,53),(51,'2c5lfb',18.98,74),(97,NULL,22.89,6),(70,'guyzyc',96.29,89),(34,'dvdoqb',53.82,1),(94,'6eop6b',81.77,90),(42,'p7vsnd',62.54,NULL);`) + + tk.MustExec("CREATE TABLE `t_vejdy` (`c_iovir` int(11) NOT NULL,`c_r_mw3d` double DEFAULT NULL,`c_uxhghb` int(11) DEFAULT NULL,`c_rb7otb` int(11) NOT NULL,`c_dplyac` int(11) DEFAULT NULL,`c_lmcqed` double DEFAULT NULL,`c_ayaoed` text DEFAULT NULL,`c__zbqr` int(11) DEFAULT NULL,PRIMARY KEY (`c_iovir`,`c_rb7otb`) /*T![clustered_index] NONCLUSTERED */,KEY `t_e1ejcd` (`c_uxhghb`),KEY `t_o6ui_b` (`c_iovir`,`c_r_mw3d`,`c_uxhghb`,`c_rb7otb`,`c_dplyac`,`c_lmcqed`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;") + + tk.MustExec(`INSERT INTO t_vejdy VALUES (49,100.11,68,57,44,17.93,NULL,84),(38,56.91,78,30,0,53.28,'cjd1o',2),(6,NULL,NULL,88,81,93.47,'0jftkb',54),(73,91.51,31,82,3,38.12,'buesob',40),(7,26.73,7,78,9,NULL,'fd5kgd',49),(80,70.57,4,47,43,25.59,'glpoq',44),(79,94.16,15,0,0,79.55,'0ok94d',56),(58,NULL,50,69,2,65.46,'sm6rj',29),(41472,6.51,70,1080,100,43.18,'fofk4c',43),(0,6.2,57,97,2,56.17,'zqpzq',56),(72,76.66,97,88,95,75.47,'hikxqb',34),(27,1.11,134217728,57,25,NULL,'4gflb',0),(64,NULL,47,69,6,72.5,'w7jmhd',45),(-134217679,88.74,33,82,85,59.89,NULL,26),(59,97.98,37,28,33,61.1,'xioxdd',45),(6,47.31,0,0,-19,38.77,'uxmdlc',17),(82,28.62,36,70,39,11.79,'zzi8cc',2),(33,37.3,55,86,69,60.56,'mn_xx',0),(7,NULL,80,0,17,59.79,'5n1q7',97),(88,50.81,15,30,63,25.37,'ordwed',29),(48,4.32,90,48,38,84.62,'lclx',32),(10,NULL,95,75,1,21.64,NULL,85),(62,NULL,0,30,10,NULL,'7bacud',5),(50,38.81,6,0,6,64.28,'gpibn',57),(1,46.8,21,32,46,33.38,NULL,6),(29,NULL,38,7,91,31.5,'pdzdl',24),(54,6.26,1,85,22,75.63,'gl4_7',29),(1,90.37,63,63,6,61.2,'wvw23b',86),(47,NULL,82,73,0,95.79,'uipcf',NULL),(46,48.1,37,6,1,52.33,'gthpic',0),(41,75.1,7,44,5,84.16,'fe_e5',58),(43,87.71,81,32,28,91.98,'9e5nvc',66),(20,58.21,88,75,92,43.64,'kagroc',66),(91,52.75,22,14,80,NULL,'\'_YN6MD\'',6),(72,94.83,0,49,5,57.82,NULL,23),(7,100.11,0,92,13,6.28,NULL,0);`) + + tk.MustExec("begin") + tk.MustExec("delete from t_tir89b where t_tir89b.c_3pcik >= t_tir89b.c_sroc_c;") + result := tk.MustQuery("select * from (select count(*) over (partition by ref_0.c_0b6nxb order by ref_0.c_3pcik) as c0 from t_tir89b as ref_0) as subq_0 where subq_0.c0 <> 1;") + result.Check(testkit.Rows("2", "3")) + tk.MustExec("commit") +} diff --git a/expression/aggregation/descriptor.go b/expression/aggregation/descriptor.go index 4415b0688ce09..ba2c24a98a67f 100644 --- a/expression/aggregation/descriptor.go +++ b/expression/aggregation/descriptor.go @@ -41,6 +41,7 @@ type AggFuncDesc struct { } // NewAggFuncDesc creates an aggregation function signature descriptor. +// this func cannot be called twice as the TypeInfer has changed the type of args in the first time. func NewAggFuncDesc(ctx sessionctx.Context, name string, args []expression.Expression, hasDistinct bool) (*AggFuncDesc, error) { b, err := newBaseFuncDesc(ctx, name, args) if err != nil { @@ -49,6 +50,14 @@ func NewAggFuncDesc(ctx sessionctx.Context, name string, args []expression.Expre return &AggFuncDesc{baseFuncDesc: b, HasDistinct: hasDistinct}, nil } +// NewAggFuncDescForWindowFunc creates an aggregation function from window functions, where baseFuncDesc may be ready. +func NewAggFuncDescForWindowFunc(ctx sessionctx.Context, Desc *WindowFuncDesc, hasDistinct bool) (*AggFuncDesc, error) { + if Desc.RetTp == nil { // safety check + return NewAggFuncDesc(ctx, Desc.Name, Desc.Args, hasDistinct) + } + return &AggFuncDesc{baseFuncDesc: baseFuncDesc{Desc.Name, Desc.Args, Desc.RetTp}, HasDistinct: hasDistinct}, nil +} + // String implements the fmt.Stringer interface. func (a *AggFuncDesc) String() string { buffer := bytes.NewBufferString(a.Name) diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go index 7daddd0dbe689..01134e39760c7 100644 --- a/expression/builtin_cast.go +++ b/expression/builtin_cast.go @@ -1902,17 +1902,17 @@ func WrapWithCastAsString(ctx sessionctx.Context, expr Expression) Expression { return expr } argLen := exprTp.Flen - // If expr is decimal, we should take the decimal point and negative sign - // into consideration, so we set `expr.GetType().Flen + 2` as the `argLen`. + // If expr is decimal, we should take the decimal point ,negative sign and the leading zero(0.xxx) + // into consideration, so we set `expr.GetType().Flen + 3` as the `argLen`. // Since the length of float and double is not accurate, we do not handle // them. if exprTp.Tp == mysql.TypeNewDecimal && argLen != int(types.UnspecifiedFsp) { - argLen += 2 + argLen += 3 } if exprTp.EvalType() == types.ETInt { argLen = mysql.MaxIntWidth } - // because we can't control the length of cast(float as char) for now, we can't determine the argLen + // Because we can't control the length of cast(float as char) for now, we can't determine the argLen. if exprTp.Tp == mysql.TypeFloat || exprTp.Tp == mysql.TypeDouble { argLen = -1 } diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index 5c9367004ec89..51e131aa5c8dd 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -634,11 +634,7 @@ func (b *builtinGreatestTimeSig) Clone() builtinFunc { // evalString evals a builtinGreatestTimeSig. // See http://dev.mysql.com/doc/refman/5.7/en/comparison-operators.html#function_greatest -func (b *builtinGreatestTimeSig) evalString(row chunk.Row) (res string, isNull bool, err error) { - var ( - strRes string - timeRes types.Time - ) +func (b *builtinGreatestTimeSig) evalString(row chunk.Row) (strRes string, isNull bool, err error) { sc := b.ctx.GetSessionVars().StmtCtx for i := 0; i < len(b.args); i++ { v, isNull, err := b.args[i].EvalString(b.ctx, row) @@ -657,16 +653,8 @@ func (b *builtinGreatestTimeSig) evalString(row chunk.Row) (res string, isNull b if i == 0 || strings.Compare(v, strRes) > 0 { strRes = v } - if i == 0 || t.Compare(timeRes) > 0 { - timeRes = t - } - } - if timeRes.IsZero() { - res = strRes - } else { - res = timeRes.String() } - return res, false, nil + return strRes, false, nil } type leastFunctionClass struct { @@ -852,12 +840,7 @@ func (b *builtinLeastTimeSig) Clone() builtinFunc { // evalString evals a builtinLeastTimeSig. // See http://dev.mysql.com/doc/refman/5.7/en/comparison-operators.html#functionleast -func (b *builtinLeastTimeSig) evalString(row chunk.Row) (res string, isNull bool, err error) { - var ( - // timeRes will be converted to a strRes only when the arguments is a valid datetime value. - strRes string // Record the strRes of each arguments. - timeRes types.Time // Record the time representation of a valid arguments. - ) +func (b *builtinLeastTimeSig) evalString(row chunk.Row) (strRes string, isNull bool, err error) { sc := b.ctx.GetSessionVars().StmtCtx for i := 0; i < len(b.args); i++ { v, isNull, err := b.args[i].EvalString(b.ctx, row) @@ -875,17 +858,9 @@ func (b *builtinLeastTimeSig) evalString(row chunk.Row) (res string, isNull bool if i == 0 || strings.Compare(v, strRes) < 0 { strRes = v } - if i == 0 || t.Compare(timeRes) < 0 { - timeRes = t - } } - if timeRes.IsZero() { - res = strRes - } else { - res = timeRes.String() - } - return res, false, nil + return strRes, false, nil } type intervalFunctionClass struct { diff --git a/expression/builtin_compare_test.go b/expression/builtin_compare_test.go index 9d11d8b5ad18a..b8d3db9ae9937 100644 --- a/expression/builtin_compare_test.go +++ b/expression/builtin_compare_test.go @@ -296,11 +296,11 @@ func (s *testEvaluatorSuite) TestGreatestLeastFunc(c *C) { }, { []interface{}{tm, "invalid_time_1", "invalid_time_2", tmWithFsp}, - curTimeWithFspString, curTimeString, false, false, + "invalid_time_2", curTimeString, false, false, }, { []interface{}{tm, "invalid_time_2", "invalid_time_1", tmWithFsp}, - curTimeWithFspString, curTimeString, false, false, + "invalid_time_2", curTimeString, false, false, }, { []interface{}{tm, "invalid_time", nil, tmWithFsp}, diff --git a/expression/builtin_control.go b/expression/builtin_control.go index e40311eb614a5..e8092583b021f 100644 --- a/expression/builtin_control.go +++ b/expression/builtin_control.go @@ -227,6 +227,14 @@ func (c *caseWhenFunctionClass) getFunction(ctx sessionctx.Context, args []Expre return nil, err } bf.tp = fieldTp + if fieldTp.Tp == mysql.TypeEnum || fieldTp.Tp == mysql.TypeSet { + switch tp { + case types.ETInt: + fieldTp.Tp = mysql.TypeLonglong + case types.ETString: + fieldTp.Tp = mysql.TypeVarchar + } + } switch tp { case types.ETInt: diff --git a/expression/builtin_time_test.go b/expression/builtin_time_test.go index 4021f1517d93e..c9b03ba0618e6 100644 --- a/expression/builtin_time_test.go +++ b/expression/builtin_time_test.go @@ -1413,6 +1413,7 @@ func (s *testEvaluatorSuite) TestStrToDate(c *C) { {"15-01-2001 1:59:58.", "%d-%m-%Y %H:%i:%s.%f", true, types.KindMysqlTime, time.Date(2001, 1, 15, 1, 59, 58, 000000000, time.Local)}, {"15-01-2001 1:9:8.999", "%d-%m-%Y %H:%i:%s.%f", true, types.KindMysqlTime, time.Date(2001, 1, 15, 1, 9, 8, 999000000, time.Local)}, {"15-01-2001 1:9:8.999", "%d-%m-%Y %H:%i:%S.%f", true, types.KindMysqlTime, time.Date(2001, 1, 15, 1, 9, 8, 999000000, time.Local)}, + {"2003-01-02 10:11:12.0012", "%Y-%m-%d %H:%i:%S.%f", true, types.KindMysqlTime, time.Date(2003, 1, 2, 10, 11, 12, 1200000, time.Local)}, {"2003-01-02 10:11:12 PM", "%Y-%m-%d %H:%i:%S %p", false, types.KindMysqlTime, time.Time{}}, {"10:20:10AM", "%H:%i:%S%p", false, types.KindMysqlTime, time.Time{}}, // test %@(skip alpha), %#(skip number), %.(skip punct) diff --git a/expression/builtin_time_vec.go b/expression/builtin_time_vec.go index 6bbad73063468..a596c85e06965 100644 --- a/expression/builtin_time_vec.go +++ b/expression/builtin_time_vec.go @@ -964,12 +964,11 @@ func (b *builtinMicroSecondSig) vecEvalInt(input *chunk.Chunk, result *chunk.Col result.ResizeInt64(n, false) result.MergeNulls(buf) i64s := result.Int64s() - ds := buf.GoDurations() for i := 0; i < n; i++ { if result.IsNull(i) { continue } - i64s[i] = int64((ds[i] % time.Second) / time.Microsecond) + i64s[i] = int64(buf.GetDuration(i, int(types.UnspecifiedFsp)).MicroSecond()) } return nil } @@ -1942,12 +1941,11 @@ func (b *builtinHourSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) er result.ResizeInt64(n, false) result.MergeNulls(buf) i64s := result.Int64s() - ds := buf.GoDurations() for i := 0; i < n; i++ { if result.IsNull(i) { continue } - i64s[i] = int64(ds[i].Hours()) + i64s[i] = int64(buf.GetDuration(i, int(types.UnspecifiedFsp)).Hour()) } return nil } diff --git a/expression/constant.go b/expression/constant.go index f7fd64eebf3e9..1c07e75a12b9b 100644 --- a/expression/constant.go +++ b/expression/constant.go @@ -363,6 +363,18 @@ func (c *Constant) HashCode(sc *stmtctx.StatementContext) []byte { if len(c.hashcode) > 0 { return c.hashcode } + + if c.DeferredExpr != nil { + c.hashcode = c.DeferredExpr.HashCode(sc) + return c.hashcode + } + + if c.ParamMarker != nil { + c.hashcode = append(c.hashcode, parameterFlag) + c.hashcode = codec.EncodeInt(c.hashcode, int64(c.ParamMarker.order)) + return c.hashcode + } + _, err := c.Eval(chunk.Row{}) if err != nil { terror.Log(err) diff --git a/expression/constant_fold.go b/expression/constant_fold.go index c698d4a0a9536..04c6a6ff4764c 100644 --- a/expression/constant_fold.go +++ b/expression/constant_fold.go @@ -140,7 +140,7 @@ func caseWhenHandler(expr *ScalarFunction) (Expression, bool) { foldedExpr.GetType().Decimal = expr.GetType().Decimal return foldedExpr, isDeferredConst } - return BuildCastFunction(expr.GetCtx(), foldedExpr, foldedExpr.GetType()), isDeferredConst + return foldedExpr, isDeferredConst } return expr, isDeferredConst } diff --git a/expression/expression.go b/expression/expression.go index 795661b98df62..941c53f17722e 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -45,6 +45,7 @@ const ( constantFlag byte = 0 columnFlag byte = 1 scalarFunctionFlag byte = 3 + parameterFlag byte = 4 ) // EvalAstExpr evaluates ast expression directly. diff --git a/expression/integration_test.go b/expression/integration_test.go index 85f6283a4618e..7b966b35dcba0 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -3906,8 +3906,7 @@ func (s *testIntegrationSuite) TestCompareBuiltin(c *C) { result.Check(testkit.Rows("3 c 1.3 2")) tk.MustQuery("show warnings").Check(testkit.Rows()) result = tk.MustQuery(`select greatest(cast("2017-01-01" as datetime), "123", "234", cast("2018-01-01" as date)), greatest(cast("2017-01-01" as date), "123", null)`) - // todo: MySQL returns "2018-01-01 " - result.Check(testkit.Rows("2018-01-01 00:00:00 ")) + result.Check(testkit.Rows("234 ")) tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1292|Incorrect time value: '123'", "Warning|1292|Incorrect time value: '234'", "Warning|1292|Incorrect time value: '123'")) // for least result = tk.MustQuery(`select least(1, 2, 3), least("a", "b", "c"), least(1.1, 1.2, 1.3), least("123a", 1, 2)`) @@ -9653,6 +9652,35 @@ func (s *testIntegrationSuite) TestGlobalCacheCorrectness(c *C) { tk.MustExec("SET GLOBAL max_connections=151") } +func (s *testIntegrationSuite) TestRedundantColumnResolve(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("create table t1(a int not null)") + tk.MustExec("create table t2(a int not null)") + tk.MustExec("insert into t1 values(1)") + tk.MustExec("insert into t2 values(1)") + tk.MustQuery("select a, count(*) from t1 join t2 using (a) group by a").Check(testkit.Rows("1 1")) + tk.MustQuery("select a, count(*) from t1 natural join t2 group by a").Check(testkit.Rows("1 1")) + err := tk.ExecToErr("select a, count(*) from t1 join t2 on t1.a=t2.a group by a") + c.Assert(err.Error(), Equals, "[planner:1052]Column 'a' in field list is ambiguous") + tk.MustQuery("select t1.a, t2.a from t1 join t2 using (a) group by t1.a").Check(testkit.Rows("1 1")) + err = tk.ExecToErr("select t1.a, t2.a from t1 join t2 using(a) group by a") + c.Assert(err.Error(), Equals, "[planner:1052]Column 'a' in group statement is ambiguous") + tk.MustQuery("select t2.a from t1 join t2 using (a) group by t1.a").Check(testkit.Rows("1")) + tk.MustQuery("select t1.a from t1 join t2 using (a) group by t1.a").Check(testkit.Rows("1")) + tk.MustQuery("select t2.a from t1 join t2 using (a) group by t2.a").Check(testkit.Rows("1")) + // The test below cannot pass now since we do not infer functional dependencies from filters as MySQL, hence would fail in only_full_group_by check. + // tk.MustQuery("select t1.a from t1 join t2 using (a) group by t2.a").Check(testkit.Rows("1")) + tk.MustQuery("select count(*) from t1 join t2 using (a) group by t2.a").Check(testkit.Rows("1")) + tk.MustQuery("select t2.a from t1 join t2 using (a) group by a").Check(testkit.Rows("1")) + tk.MustQuery("select t1.a from t1 join t2 using (a) group by a").Check(testkit.Rows("1")) + tk.MustQuery("select * from t1 join t2 using(a)").Check(testkit.Rows("1")) + tk.MustQuery("select t1.a, t2.a from t1 join t2 using(a)").Check(testkit.Rows("1 1")) + tk.MustQuery("select * from t1 natural join t2").Check(testkit.Rows("1")) + tk.MustQuery("select t1.a, t2.a from t1 natural join t2").Check(testkit.Rows("1 1")) +} + func (s *testIntegrationSuite) TestControlFunctionWithEnumOrSet(c *C) { defer s.cleanEnv(c) @@ -9737,6 +9765,19 @@ func (s *testIntegrationSuite) TestControlFunctionWithEnumOrSet(c *C) { tk.MustExec("insert into t values(1,1,1),(2,1,1),(1,1,1),(2,1,1);") tk.MustQuery("select if(A, null,b)=1 from t;").Check(testkit.Rows("", "", "", "")) tk.MustQuery("select if(A, null,b)='a' from t;").Check(testkit.Rows("", "", "", "")) + + // issue 29357 + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(`a` enum('y','b','Abc','null','1','2','0')) CHARSET=binary;") + tk.MustExec("insert into t values(\"1\");") + tk.MustQuery("SELECT count(*) from t where (null like 'a') = (case when cast('2015' as real) <=> round(\"1200\",\"1\") then a end);\n").Check(testkit.Rows("0")) + tk.MustQuery("SELECT (null like 'a') = (case when cast('2015' as real) <=> round(\"1200\",\"1\") then a end) from t;\n").Check(testkit.Rows("")) + tk.MustQuery("SELECT 5 = (case when 0 <=> 0 then a end) from t;").Check(testkit.Rows("1")) + tk.MustQuery("SELECT '1' = (case when 0 <=> 0 then a end) from t;").Check(testkit.Rows("1")) + tk.MustQuery("SELECT 5 = (case when 0 <=> 1 then a end) from t;").Check(testkit.Rows("")) + tk.MustQuery("SELECT '1' = (case when 0 <=> 1 then a end) from t;").Check(testkit.Rows("")) + tk.MustQuery("SELECT 5 = (case when 0 <=> 1 then a else a end) from t;").Check(testkit.Rows("1")) + tk.MustQuery("SELECT '1' = (case when 0 <=> 1 then a else a end) from t;").Check(testkit.Rows("1")) } func (s *testIntegrationSuite) TestComplexShowVariables(c *C) { @@ -9861,6 +9902,33 @@ func (s *testIntegrationSerialSuite) TestIssue26662(c *C) { Check(testkit.Rows()) } +func (s *testIntegrationSuite) TestIssue29434(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 datetime);") + tk.MustExec("insert into t1 values('2021-12-12 10:10:10.000');") + tk.MustExec("set tidb_enable_vectorized_expression = on;") + tk.MustQuery("select greatest(c1, '99999999999999') from t1;").Check(testkit.Rows("99999999999999")) + tk.MustExec("set tidb_enable_vectorized_expression = off;") + tk.MustQuery("select greatest(c1, '99999999999999') from t1;").Check(testkit.Rows("99999999999999")) + + tk.MustExec("set tidb_enable_vectorized_expression = on;") + tk.MustQuery("select least(c1, '99999999999999') from t1;").Check(testkit.Rows("2021-12-12 10:10:10")) + tk.MustExec("set tidb_enable_vectorized_expression = off;") + tk.MustQuery("select least(c1, '99999999999999') from t1;").Check(testkit.Rows("2021-12-12 10:10:10")) +} + +func (s *testIntegrationSuite) TestIssue29417(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1 (f1 decimal(5,5));") + tk.MustExec("insert into t1 values (-0.12345);") + tk.MustQuery("select concat(f1) from t1;").Check(testkit.Rows("-0.12345")) +} + func (s *testIntegrationSuite) TestConstPropNullFunctions(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -9879,6 +9947,19 @@ func (s *testIntegrationSuite) TestConstPropNullFunctions(c *C) { tk.MustQuery("select * from t2 where t2.i2=((select count(1) from t1 where t1.i1=t2.i2))").Check(testkit.Rows("1 0.1")) } +func (s *testIntegrationSuite) TestIssue28643(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a time(4));") + tk.MustExec("insert into t values(\"-838:59:59.000000\");") + tk.MustExec("insert into t values(\"838:59:59.000000\");") + tk.MustExec("set tidb_enable_vectorized_expression = on;") + tk.MustQuery("select hour(a) from t;").Check(testkit.Rows("838", "838")) + tk.MustExec("set tidb_enable_vectorized_expression = off;") + tk.MustQuery("select hour(a) from t;").Check(testkit.Rows("838", "838")) +} + func (s *testIntegrationSuite) TestIssue29513(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -9891,3 +9972,30 @@ func (s *testIntegrationSuite) TestIssue29513(c *C) { tk.MustQuery("select '123' union select cast(a as char) from t;").Sort().Check(testkit.Rows("123", "45678")) tk.MustQuery("select '123' union select cast(a as char(2)) from t;").Sort().Check(testkit.Rows("123", "45")) } + +func (s *testIntegrationSuite) TestIssue27831(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a enum(\"a\", \"b\"), b enum(\"a\", \"b\"), c bool)") + tk.MustExec("insert into t values(\"a\", \"a\", 1);") + tk.MustQuery("select * from t t1 right join t t2 on t1.a=t2.b and t1.a= t2.c;").Check(testkit.Rows("a a 1 a a 1")) + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a enum(\"a\", \"b\"), b enum(\"a\", \"b\"), c bool, d int, index idx(d))") + tk.MustExec("insert into t values(\"a\", \"a\", 1, 1);") + tk.MustQuery("select /*+ inl_hash_join(t1) */ * from t t1 right join t t2 on t1.a=t2.b and t1.a= t2.c and t1.d=t2.d;").Check(testkit.Rows("a a 1 1 a a 1 1")) +} + +func (s *testIntegrationSuite) TestIssue29244(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a time(4));") + tk.MustExec("insert into t values(\"-700:10:10.123456111\");") + tk.MustExec("insert into t values(\"700:10:10.123456111\");") + tk.MustExec("set tidb_enable_vectorized_expression = on;") + tk.MustQuery("select microsecond(a) from t;").Check(testkit.Rows("123500", "123500")) + tk.MustExec("set tidb_enable_vectorized_expression = off;") + tk.MustQuery("select microsecond(a) from t;").Check(testkit.Rows("123500", "123500")) +} diff --git a/expression/simple_rewriter.go b/expression/simple_rewriter.go index e0e9f41fd1bbf..72a2e0818373e 100644 --- a/expression/simple_rewriter.go +++ b/expression/simple_rewriter.go @@ -128,6 +128,12 @@ func FindFieldName(names types.NameSlice, astCol *ast.ColumnName) (int, error) { if idx == -1 { idx = i } else { + if names[idx].Redundant || name.Redundant { + if !name.Redundant { + idx = i + } + continue + } return -1, errNonUniq.GenWithStackByArgs(astCol.String(), "field list") } } diff --git a/expression/typeinfer_test.go b/expression/typeinfer_test.go index 3d9d06946f0e3..ceed7211ac7cc 100644 --- a/expression/typeinfer_test.go +++ b/expression/typeinfer_test.go @@ -407,7 +407,7 @@ func (s *testInferTypeSuite) createTestCase4StrFuncs() []typeInferTestCase { {"reverse(c_bigint_d )", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength}, {"reverse(c_float_d )", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, types.UnspecifiedLength, types.UnspecifiedLength}, {"reverse(c_double_d )", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, types.UnspecifiedLength, types.UnspecifiedLength}, - {"reverse(c_decimal )", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 8, types.UnspecifiedLength}, + {"reverse(c_decimal )", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 9, types.UnspecifiedLength}, {"reverse(c_char )", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength}, {"reverse(c_varchar )", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 20, types.UnspecifiedLength}, {"reverse(c_text_d )", mysql.TypeVarString, charset.CharsetUTF8MB4, 0, 65535, types.UnspecifiedLength}, diff --git a/go.mod b/go.mod index 3bf700425a526..797fa36d69993 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/pingcap/parser v0.0.0-20210618053735-57843e8185c4 github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3 github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible - github.com/pingcap/tipb v0.0.0-20210628060001-1793e022b962 + github.com/pingcap/tipb v0.0.0-20211025074540-e1c7362eeeb4 github.com/prometheus/client_golang v1.5.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.9.1 @@ -68,6 +68,7 @@ require ( go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b go.uber.org/atomic v1.8.0 go.uber.org/automaxprocs v1.2.0 + go.uber.org/goleak v0.10.0 go.uber.org/zap v1.17.0 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 diff --git a/go.sum b/go.sum index 256602760ce37..7ff55ac2bdbdc 100644 --- a/go.sum +++ b/go.sum @@ -451,8 +451,8 @@ github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3/go.mod h1:tckvA041 github.com/pingcap/tidb-dashboard v0.0.0-20210312062513-eef5d6404638/go.mod h1:OzFN8H0EDMMqeulPhPMw2i2JaiZWOKFQ7zdRPhENNgo= github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible h1:ceznmu/lLseGHP/jKyOa/3u/5H3wtLLLqkH2V3ssSjg= github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20210628060001-1793e022b962 h1:9Y9Eci9LwAEhyXAlAU0bSix7Nemm3G267oyN3GVK+j0= -github.com/pingcap/tipb v0.0.0-20210628060001-1793e022b962/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pingcap/tipb v0.0.0-20211025074540-e1c7362eeeb4 h1:9Ef4j3DLmUidURfob0tf94v+sqvozqdCTr7e5hi19qU= +github.com/pingcap/tipb v0.0.0-20211025074540-e1c7362eeeb4/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 1f017ea3654d9..7493d29f64921 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -1494,6 +1494,9 @@ func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAn if len(oneColumnRan) == 0 { return nil, true, nil } + if sc.MemTracker != nil { + sc.MemTracker.Consume(2 * types.EstimatedMemUsage(oneColumnRan[0].LowVal, len(oneColumnRan))) + } for _, ran := range ranges { ran.LowVal[i] = oneColumnRan[0].LowVal[0] ran.HighVal[i] = oneColumnRan[0].HighVal[0] @@ -1507,6 +1510,9 @@ func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAn newRange.HighVal[i] = oneColumnRan[ranIdx].HighVal[0] newRanges = append(newRanges, newRange) } + if sc.MemTracker != nil && len(newRanges) != 0 { + sc.MemTracker.Consume(2 * types.EstimatedMemUsage(newRanges[0].LowVal, len(newRanges))) + } ranges = append(ranges, newRanges...) } j++ diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index e9f233507998c..52db4f2ccd1ee 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -1821,13 +1821,13 @@ func findFieldNameFromNaturalUsingJoin(p LogicalPlan, v *ast.ColumnName) (col *e case *LogicalLimit, *LogicalSelection, *LogicalTopN, *LogicalSort, *LogicalMaxOneRow: return findFieldNameFromNaturalUsingJoin(p.Children()[0], v) case *LogicalJoin: - if x.redundantSchema != nil { - idx, err := expression.FindFieldName(x.redundantNames, v) + if x.fullSchema != nil { + idx, err := expression.FindFieldName(x.fullNames, v) if err != nil { return nil, nil, err } if idx >= 0 { - return x.redundantSchema.Columns[idx], x.redundantNames[idx], nil + return x.fullSchema.Columns[idx], x.fullNames[idx], nil } } } diff --git a/planner/core/fragment.go b/planner/core/fragment.go index 8adcbf127ff26..da81c591aca9d 100644 --- a/planner/core/fragment.go +++ b/planner/core/fragment.go @@ -246,6 +246,11 @@ func (e *mppTaskGenerator) generateMPPTasksForFragment(f *Fragment) (tasks []*kv } if f.TableScan != nil { tasks, err = e.constructMPPTasksImpl(context.Background(), f.TableScan) + if err == nil && len(tasks) == 0 { + err = errors.New( + "In mpp mode, the number of tasks for table scan should not be zero. " + + "Please set tidb_allow_mpp = 0, and then rerun sql.") + } } else { childrenTasks := make([]*kv.MPPTask, 0) for _, r := range f.ExchangeReceivers { diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index a8db9802903cd..624ff5ea84906 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -3993,6 +3993,12 @@ func (s *testIntegrationSuite) TestIssue26559(c *C) { func (s *testIntegrationSuite) TestIssue27797(c *C) { tk := testkit.NewTestKit(c, s.store) + origin := tk.MustQuery("SELECT @@session.tidb_partition_prune_mode") + originStr := origin.Rows()[0][0].(string) + defer func() { + tk.MustExec("set @@session.tidb_partition_prune_mode = '" + originStr + "'") + }() + tk.MustExec("set @@session.tidb_partition_prune_mode = 'static'") tk.MustExec("use test") tk.MustExec("drop table if exists t27797") tk.MustExec("create table t27797(a int, b int, c int, d int) " + @@ -4087,3 +4093,55 @@ func (s *testIntegrationSuite) TestIssues27130(c *C) { " └─IndexRangeScan 10.00 cop[tikv] table:t3, index:a(a, b, c) range:[1,1], keep order:false, stats:pseudo", )) } + +func (s *testIntegrationSuite) TestIssue29705(c *C) { + tk := testkit.NewTestKit(c, s.store) + origin := tk.MustQuery("SELECT @@session.tidb_partition_prune_mode") + originStr := origin.Rows()[0][0].(string) + defer func() { + tk.MustExec("set @@session.tidb_partition_prune_mode = '" + originStr + "'") + }() + tk.MustExec("set @@session.tidb_partition_prune_mode = 'static'") + tk.MustExec("use test") + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(id int) partition by hash(id) partitions 4;") + tk.MustExec("insert into t values(1);") + result := tk.MustQuery("SELECT COUNT(1) FROM ( SELECT COUNT(1) FROM t b GROUP BY id) a;") + result.Check(testkit.Rows("1")) +} + +func (s *testIntegrationSuite) TestIssues29711(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + tk.MustExec("drop table if exists tbl_29711") + tk.MustExec("CREATE TABLE `tbl_29711` (" + + "`col_250` text COLLATE utf8_unicode_ci NOT NULL," + + "`col_251` enum('Alice','Bob','Charlie','David') COLLATE utf8_unicode_ci NOT NULL DEFAULT 'Charlie'," + + "PRIMARY KEY (`col_251`,`col_250`(1)) NONCLUSTERED);") + tk.MustQuery("explain " + + "select col_250,col_251 from tbl_29711 where col_251 between 'Bob' and 'David' order by col_250,col_251 limit 6;"). + Check(testkit.Rows( + "TopN_9 6.00 root test.tbl_29711.col_250, test.tbl_29711.col_251, offset:0, count:6", + "└─IndexLookUp_22 6.00 root ", + " ├─IndexRangeScan_19(Build) 30.00 cop[tikv] table:tbl_29711, index:PRIMARY(col_251, col_250) range:[\"Bob\",\"Bob\"], [\"Charlie\",\"Charlie\"], [\"David\",\"David\"], keep order:false, stats:pseudo", + " └─TopN_21(Probe) 6.00 cop[tikv] test.tbl_29711.col_250, test.tbl_29711.col_251, offset:0, count:6", + " └─TableRowIDScan_20 30.00 cop[tikv] table:tbl_29711 keep order:false, stats:pseudo", + )) + + tk.MustExec("drop table if exists t29711") + tk.MustExec("CREATE TABLE `t29711` (" + + "`a` varchar(10) DEFAULT NULL," + + "`b` int(11) DEFAULT NULL," + + "`c` int(11) DEFAULT NULL," + + "KEY `ia` (`a`(2)))") + tk.MustQuery("explain select * from t29711 use index (ia) order by a limit 10;"). + Check(testkit.Rows( + "TopN_8 10.00 root test.t29711.a, offset:0, count:10", + "└─IndexLookUp_17 10.00 root ", + " ├─IndexFullScan_14(Build) 10000.00 cop[tikv] table:t29711, index:ia(a) keep order:false, stats:pseudo", + " └─TopN_16(Probe) 10.00 cop[tikv] test.t29711.a, offset:0, count:10", + " └─TableRowIDScan_15 10000.00 cop[tikv] table:t29711 keep order:false, stats:pseudo", + )) + +} diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 03caaaa14756d..f360722e4dd9b 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -271,8 +271,8 @@ func (b *PlanBuilder) buildAggregation(ctx context.Context, p LogicalPlan, aggFu schema4Agg.Append(newCol) names = append(names, p.OutputNames()[i]) } - if join, isJoin := p.(*LogicalJoin); isJoin && join.redundantSchema != nil { - for i, col := range join.redundantSchema.Columns { + if join, isJoin := p.(*LogicalJoin); isJoin && join.fullSchema != nil { + for i, col := range join.fullSchema.Columns { if p.Schema().Contains(col) { continue } @@ -284,7 +284,7 @@ func (b *PlanBuilder) buildAggregation(ctx context.Context, p LogicalPlan, aggFu newCol, _ := col.Clone().(*expression.Column) newCol.RetType = newFunc.RetTp schema4Agg.Append(newCol) - names = append(names, join.redundantNames[i]) + names = append(names, join.fullNames[i]) } } hasGroupBy := len(gbyItems) > 0 @@ -715,25 +715,50 @@ func (b *PlanBuilder) buildJoin(ctx context.Context, joinNode *ast.Join) (Logica joinPlan.JoinType = InnerJoin } - // Merge sub join's redundantSchema into this join plan. When handle query like - // select t2.a from (t1 join t2 using (a)) join t3 using (a); - // we can simply search in the top level join plan to find redundant column. + // Merge sub-plan's fullSchema into this join plan. + // Please read the comment of LogicalJoin.fullSchema for the details. var ( - lRedundantSchema, rRedundantSchema *expression.Schema - lRedundantNames, rRedundantNames types.NameSlice + lFullSchema, rFullSchema *expression.Schema + lFullNames, rFullNames types.NameSlice ) - if left, ok := leftPlan.(*LogicalJoin); ok && left.redundantSchema != nil { - lRedundantSchema = left.redundantSchema - lRedundantNames = left.redundantNames + if left, ok := leftPlan.(*LogicalJoin); ok && left.fullSchema != nil { + lFullSchema = left.fullSchema + lFullNames = left.fullNames + } else { + lFullSchema = leftPlan.Schema() + lFullNames = leftPlan.OutputNames() + } + if right, ok := rightPlan.(*LogicalJoin); ok && right.fullSchema != nil { + rFullSchema = right.fullSchema + rFullNames = right.fullNames + } else { + rFullSchema = rightPlan.Schema() + rFullNames = rightPlan.OutputNames() + } + if joinNode.Tp == ast.RightJoin { + // Make sure lFullSchema means outer full schema and rFullSchema means inner full schema. + lFullSchema, rFullSchema = rFullSchema, lFullSchema + lFullNames, rFullNames = rFullNames, lFullNames + } + joinPlan.fullSchema = expression.MergeSchema(lFullSchema, rFullSchema) + + // Clear NotNull flag for the inner side schema if it's an outer join. + if joinNode.Tp == ast.LeftJoin || joinNode.Tp == ast.RightJoin { + resetNotNullFlag(joinPlan.fullSchema, lFullSchema.Len(), joinPlan.fullSchema.Len()) + } + + // Merge sub-plan's fullNames into this join plan, similar to the fullSchema logic above. + joinPlan.fullNames = make([]*types.FieldName, 0, len(lFullNames)+len(rFullNames)) + for _, lName := range lFullNames { + name := *lName + name.Redundant = true + joinPlan.fullNames = append(joinPlan.fullNames, &name) } - if right, ok := rightPlan.(*LogicalJoin); ok && right.redundantSchema != nil { - rRedundantSchema = right.redundantSchema - rRedundantNames = right.redundantNames + for _, rName := range rFullNames { + name := *rName + name.Redundant = true + joinPlan.fullNames = append(joinPlan.fullNames, &name) } - joinPlan.redundantSchema = expression.MergeSchema(lRedundantSchema, rRedundantSchema) - joinPlan.redundantNames = make([]*types.FieldName, len(lRedundantNames)+len(rRedundantNames)) - copy(joinPlan.redundantNames, lRedundantNames) - copy(joinPlan.redundantNames[len(lRedundantNames):], rRedundantNames) // Set preferred join algorithm if some join hints is specified by user. joinPlan.setPreferredJoinType(b.TableHints()) @@ -838,6 +863,7 @@ func (b *PlanBuilder) coalesceCommonColumns(p *LogicalJoin, leftPlan, rightPlan lColumns, rColumns := lsc.Columns, rsc.Columns lNames, rNames := leftPlan.OutputNames().Shallow(), rightPlan.OutputNames().Shallow() if joinTp == ast.RightJoin { + leftPlan, rightPlan = rightPlan, leftPlan lNames, rNames = rNames, lNames lColumns, rColumns = rsc.Columns, lsc.Columns } @@ -935,19 +961,7 @@ func (b *PlanBuilder) coalesceCommonColumns(p *LogicalJoin, leftPlan, rightPlan p.SetSchema(expression.NewSchema(schemaCols...)) p.names = names - if joinTp == ast.RightJoin { - leftPlan, rightPlan = rightPlan, leftPlan - } - // We record the full `rightPlan.Schema` as `redundantSchema` in order to - // record the redundant column in `rightPlan` and the output columns order - // of the `rightPlan`. - // For SQL like `select t1.*, t2.* from t1 left join t2 using(a)`, we can - // retrieve the column order of `t2.*` from the `redundantSchema`. - p.redundantSchema = expression.MergeSchema(p.redundantSchema, expression.NewSchema(rightPlan.Schema().Clone().Columns...)) - p.redundantNames = append(p.redundantNames.Shallow(), rightPlan.OutputNames().Shallow()...) - if joinTp == ast.RightJoin || joinTp == ast.LeftJoin { - resetNotNullFlag(p.redundantSchema, 0, p.redundantSchema.Len()) - } + p.OtherConditions = append(conds, p.OtherConditions...) return nil @@ -1194,9 +1208,9 @@ func findColFromNaturalUsingJoin(p LogicalPlan, col *expression.Column) (name *t case *LogicalLimit, *LogicalSelection, *LogicalTopN, *LogicalSort, *LogicalMaxOneRow: return findColFromNaturalUsingJoin(p.Children()[0], col) case *LogicalJoin: - if x.redundantSchema != nil { - idx := x.redundantSchema.ColumnIndex(col) - return x.redundantNames[idx] + if x.fullSchema != nil { + idx := x.fullSchema.ColumnIndex(col) + return x.fullNames[idx] } } return nil @@ -1969,9 +1983,9 @@ func (a *havingWindowAndOrderbyExprResolver) resolveFromPlan(v *ast.ColumnNameEx case *LogicalLimit, *LogicalSelection, *LogicalTopN, *LogicalSort, *LogicalMaxOneRow: return a.resolveFromPlan(v, p.Children()[0]) case *LogicalJoin: - if len(x.redundantNames) != 0 { - idx, err = expression.FindFieldName(x.redundantNames, v.Name) - schemaCols, outputNames = x.redundantSchema.Columns, x.redundantNames + if len(x.fullNames) != 0 { + idx, err = expression.FindFieldName(x.fullNames, v.Name) + schemaCols, outputNames = x.fullSchema.Columns, x.fullNames } } if err != nil || idx < 0 { @@ -2530,6 +2544,7 @@ func (g *gbyResolver) Leave(inNode ast.Node) (ast.Node, bool) { var index int index, g.err = resolveFromSelectFields(v, g.fields, false) if g.err != nil { + g.err = ErrAmbiguous.GenWithStackByArgs(v.Name.Name.L, clauseMsg[groupByClause]) return inNode, false } if idx >= 0 { @@ -3096,14 +3111,11 @@ func (b *PlanBuilder) unfoldWildStar(p LogicalPlan, selectFields []*ast.SelectFi return nil, ErrInvalidWildCard } list := unfoldWildStar(field, p.OutputNames(), p.Schema().Columns) - // For sql like `select t1.*, t2.* from t1 join t2 using(a)`, we should - // not coalesce the `t2.a` in the output result. Thus we need to unfold - // the wildstar from the underlying join.redundantSchema. - if isJoin && join.redundantSchema != nil && field.WildCard.Table.L != "" { - redundantList := unfoldWildStar(field, join.redundantNames, join.redundantSchema.Columns) - if len(redundantList) > len(list) { - list = redundantList - } + // For sql like `select t1.*, t2.* from t1 join t2 using(a)` or `select t1.*, t2.* from t1 natual join t2`, + // the schema of the Join doesn't contain enough columns because the join keys are coalesced in this schema. + // We should collect the columns from the fullSchema. + if isJoin && join.fullSchema != nil && field.WildCard.Table.L != "" { + list = unfoldWildStar(field, join.fullNames, join.fullSchema.Columns) } if len(list) == 0 { return nil, ErrBadTable.GenWithStackByArgs(field.WildCard.Table) diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 9d3df6ab911a4..ab3e26ffc08af 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -604,6 +604,9 @@ func (s *testPlanSuite) TestProjectionEliminator(c *C) { { sql: "select 1+num from (select 1+a as num from t) t1;", best: "DataScan(t)->Projection", + }, { + sql: "select count(*) from t where a in (select b from t2 where a is null);", + best: "Join{DataScan(t)->Dual->Aggr(firstrow(test.t2.b))}(test.t.a,test.t2.b)->Aggr(count(1))->Projection", }, } @@ -621,6 +624,29 @@ func (s *testPlanSuite) TestProjectionEliminator(c *C) { } } +func (s *testPlanSuite) TestCS3389(c *C) { + defer testleak.AfterTest(c)() + + ctx := context.Background() + stmt, err := s.ParseOneStmt("select count(*) from t where a in (select b from t2 where a is null);", "", "") + c.Assert(err, IsNil) + p, _, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) + c.Assert(err, IsNil) + p, err = logicalOptimize(context.TODO(), flagBuildKeyInfo|flagPrunColumns|flagPrunColumnsAgain|flagEliminateProjection|flagJoinReOrder, p.(LogicalPlan)) + c.Assert(err, IsNil) + + // Assert that all Projection is not empty and there is no Projection between Aggregation and Join. + proj, isProj := p.(*LogicalProjection) + c.Assert(isProj, IsTrue) + c.Assert(len(proj.Exprs) > 0, IsTrue) + child := proj.Children()[0] + agg, isAgg := child.(*LogicalAggregation) + c.Assert(isAgg, IsTrue) + child = agg.Children()[0] + _, isJoin := child.(*LogicalJoin) + c.Assert(isJoin, IsTrue) +} + func (s *testPlanSuite) TestAllocID(c *C) { ctx := MockContext() pA := DataSource{}.Init(ctx, 0) @@ -1456,7 +1482,7 @@ func (s *testPlanSuite) TestNameResolver(c *C) { {"select * from t", ""}, {"select t.* from t", ""}, {"select t2.* from t", "[planner:1051]Unknown table 't2'"}, - {"select b as a, c as a from t group by a", "[planner:1052]Column 'c' in field list is ambiguous"}, + {"select b as a, c as a from t group by a", "[planner:1052]Column 'a' in group statement is ambiguous"}, {"select 1 as a, b as a, c as a from t group by a", ""}, {"select a, b as a from t group by a+1", ""}, {"select c, a as c from t order by c+1", ""}, diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index e7e4d49324fb2..f9d19daf4edd5 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -144,10 +144,22 @@ type LogicalJoin struct { // Currently, only `aggregation push down` phase will set this. DefaultValues []types.Datum - // redundantSchema contains columns which are eliminated in join. - // For select * from a join b using (c); a.c will in output schema, and b.c will only in redundantSchema. - redundantSchema *expression.Schema - redundantNames types.NameSlice + // fullSchema contains all the columns that the Join can output. It's ordered as [outer schema..., inner schema...]. + // This is useful for natural joins and "using" joins. In these cases, the join key columns from the + // inner side (or the right side when it's an inner join) will not be in the schema of Join. + // But upper operators should be able to find those "redundant" columns, and the user also can specifically select + // those columns, so we put the "redundant" columns here to make them be able to be found. + // + // For example: + // create table t1(a int, b int); create table t2(a int, b int); + // select * from t1 join t2 using (b); + // schema of the Join will be [t1.b, t1.a, t2.a]; fullSchema will be [t1.a, t1.b, t2.a, t2.b]. + // + // We record all columns and keep them ordered is for correctly handling SQLs like + // select t1.*, t2.* from t1 join t2 using (b); + // (*PlanBuilder).unfoldWildStar() handles the schema for such case. + fullSchema *expression.Schema + fullNames types.NameSlice // equalCondOutCnt indicates the estimated count of joined rows after evaluating `EqualConditions`. equalCondOutCnt float64 diff --git a/planner/core/partition_pruner_test.go b/planner/core/partition_pruner_test.go index 73755e71c27cf..07dee6f73741e 100644 --- a/planner/core/partition_pruner_test.go +++ b/planner/core/partition_pruner_test.go @@ -85,6 +85,83 @@ func (s *testPartitionPruneSuit) TestHashPartitionPruner(c *C) { } } +func (s *testPartitionPruneSuit) TestRangeColumnPartitionPruningForIn(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("drop database if exists test_range_col_in") + tk.MustExec("create database test_range_col_in") + tk.MustExec("use test_range_col_in") + tk.MustExec(`set @@session.tidb_enable_list_partition = 1`) + tk.MustExec("set @@session.tidb_partition_prune_mode='static'") + + // case in issue-26739 + tk.MustExec(`CREATE TABLE t1 ( + id bigint(20) NOT NULL AUTO_INCREMENT, + dt date, + PRIMARY KEY (id,dt)) + PARTITION BY RANGE COLUMNS(dt) ( + PARTITION p20201125 VALUES LESS THAN ("20201126"), + PARTITION p20201126 VALUES LESS THAN ("20201127"), + PARTITION p20201127 VALUES LESS THAN ("20201128"), + PARTITION p20201128 VALUES LESS THAN ("20201129"), + PARTITION p20201129 VALUES LESS THAN ("20201130"))`) + tk.MustQuery(`explain format='brief' select /*+ HASH_AGG() */ count(1) from t1 where dt in ('2020-11-27','2020-11-28')`).Check( + testkit.Rows("HashAgg 1.00 root funcs:count(Column#5)->Column#4", + "└─PartitionUnion 2.00 root ", + " ├─HashAgg 1.00 root funcs:count(Column#7)->Column#5", + " │ └─IndexReader 1.00 root index:HashAgg", + " │ └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#7", + " │ └─Selection 20.00 cop[tikv] in(test_range_col_in.t1.dt, 2020-11-27 00:00:00.000000, 2020-11-28 00:00:00.000000)", + " │ └─IndexFullScan 10000.00 cop[tikv] table:t1, partition:p20201127, index:PRIMARY(id, dt) keep order:false, stats:pseudo", + " └─HashAgg 1.00 root funcs:count(Column#10)->Column#5", + " └─IndexReader 1.00 root index:HashAgg", + " └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#10", + " └─Selection 20.00 cop[tikv] in(test_range_col_in.t1.dt, 2020-11-27 00:00:00.000000, 2020-11-28 00:00:00.000000)", + " └─IndexFullScan 10000.00 cop[tikv] table:t1, partition:p20201128, index:PRIMARY(id, dt) keep order:false, stats:pseudo")) + + tk.MustExec(`insert into t1 values (1, "2020-11-25")`) + tk.MustExec(`insert into t1 values (2, "2020-11-26")`) + tk.MustExec(`insert into t1 values (3, "2020-11-27")`) + tk.MustExec(`insert into t1 values (4, "2020-11-28")`) + tk.MustQuery(`select id from t1 where dt in ('2020-11-27','2020-11-28') order by id`).Check(testkit.Rows("3", "4")) + tk.MustQuery(`select id from t1 where dt in (20201127,'2020-11-28') order by id`).Check(testkit.Rows("3", "4")) + tk.MustQuery(`select id from t1 where dt in (20201127,20201128) order by id`).Check(testkit.Rows("3", "4")) + tk.MustQuery(`select id from t1 where dt in (20201127,20201128,null) order by id`).Check(testkit.Rows("3", "4")) + tk.MustQuery(`select id from t1 where dt in ('2020-11-26','2020-11-25','2020-11-28') order by id`).Check(testkit.Rows("1", "2", "4")) + tk.MustQuery(`select id from t1 where dt in ('2020-11-26','wrong','2020-11-28') order by id`).Check(testkit.Rows("2", "4")) + + // int + tk.MustExec(`create table t2 (a int) partition by range columns(a) ( + partition p0 values less than (0), + partition p1 values less than (10), + partition p2 values less than (20))`) + tk.MustExec(`insert into t2 values (-1), (1), (11), (null)`) + tk.MustQuery(`select a from t2 where a in (-1, 1) order by a`).Check(testkit.Rows("-1", "1")) + tk.MustQuery(`select a from t2 where a in (1, 11, null) order by a`).Check(testkit.Rows("1", "11")) + tk.MustQuery(`explain format='brief' select a from t2 where a in (-1, 1)`).Check(testkit.Rows("PartitionUnion 40.00 root ", + "├─TableReader 20.00 root data:Selection", + "│ └─Selection 20.00 cop[tikv] in(test_range_col_in.t2.a, -1, 1)", + "│ └─TableFullScan 10000.00 cop[tikv] table:t2, partition:p0 keep order:false, stats:pseudo", + "└─TableReader 20.00 root data:Selection", + " └─Selection 20.00 cop[tikv] in(test_range_col_in.t2.a, -1, 1)", + " └─TableFullScan 10000.00 cop[tikv] table:t2, partition:p1 keep order:false, stats:pseudo")) + + // for other types, the in-pruning shouldn't be working for safety + tk.MustExec(`create table t3 (a varchar(10)) partition by range columns(a) ( + partition p0 values less than ("aaa"), + partition p1 values less than ("bbb"), + partition p2 values less than ("ccc"))`) + tk.MustQuery(`explain format='brief' select a from t3 where a in ('aaa', 'aab')`).Check(testkit.Rows("PartitionUnion 60.00 root ", + "├─TableReader 20.00 root data:Selection", + "│ └─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, \"aaa\", \"aab\")", + "│ └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p0 keep order:false, stats:pseudo", + "├─TableReader 20.00 root data:Selection", + "│ └─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, \"aaa\", \"aab\")", + "│ └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p1 keep order:false, stats:pseudo", + "└─TableReader 20.00 root data:Selection", + " └─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, \"aaa\", \"aab\")", + " └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p2 keep order:false, stats:pseudo")) +} + func (s *testPartitionPruneSuit) TestListPartitionPruner(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("drop database if exists test_partition;") @@ -488,13 +565,21 @@ func (s *testPartitionPruneSuit) TestRangePartitionPredicatePruner(c *C) { } } -func (s *testPartitionPruneSuit) TestIssue26551(c *C) { +func (s *testPartitionPruneSuit) TestHashPartitionPruning(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("set @@tidb_partition_prune_mode='static'") tk.MustExec("USE test;") tk.MustExec("DROP TABLE IF EXISTS t;") tk.MustExec("CREATE TABLE t (`COL1` int, `COL3` bigint) PARTITION BY HASH ((`COL1` * `COL3`))PARTITIONS 13;") - tk.MustQuery("explain format = 'brief' SELECT * FROM t WHERE col3 =2659937067964964513 and col1 = 783367513002;").Check(testkit.Rows( - "TableDual 0.00 root rows:0")) - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1690 BIGINT value is out of range in '(test.t.col1 * test.t.col3)'")) + tk.MustQuery("SELECT * FROM t WHERE col3 =2659937067964964513 and col1 = 783367513002;").Check(testkit.Rows()) + tk.MustExec("drop table if exists t;") + tk.MustExec("CREATE TABLE `t` (" + + "`COL1` int NOT NULL DEFAULT '25' COMMENT 'NUMERIC PK'," + + "`COL3` bigint NOT NULL," + + "PRIMARY KEY (`COL1`,`COL3`)" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin " + + "PARTITION BY HASH ((`COL1` * `COL3`))" + + "PARTITIONS 13;") + tk.MustExec("insert into t(col1, col3) values(0, 3522101843073676459);") + tk.MustQuery("SELECT col1, COL3 FROM t WHERE COL1 IN (0,14158354938390,0) AND COL3 IN (3522101843073676459,-2846203247576845955,838395691793635638);").Check(testkit.Rows("0 3522101843073676459")) } diff --git a/planner/core/plan_to_pb.go b/planner/core/plan_to_pb.go index 985fb0a72878d..7e4f477f72e97 100644 --- a/planner/core/plan_to_pb.go +++ b/planner/core/plan_to_pb.go @@ -242,8 +242,16 @@ func (e *PhysicalExchangeSender) ToPB(ctx sessionctx.Context, storeType kv.Store } hashCols := make([]expression.Expression, 0, len(e.HashCols)) + hashColTypes := make([]*tipb.FieldType, 0, len(e.HashCols)) for _, col := range e.HashCols { hashCols = append(hashCols, col) + tp := expression.ToPBFieldType(col.RetType) + hashColTypes = append(hashColTypes, tp) + } + allFieldTypes := make([]*tipb.FieldType, 0, len(e.Schema().Columns)) + for _, column := range e.Schema().Columns { + pbType := expression.ToPBFieldType(column.RetType) + allFieldTypes = append(allFieldTypes, pbType) } hashColPb, err := expression.ExpressionsToPBList(ctx.GetSessionVars().StmtCtx, hashCols, ctx.GetClient()) if err != nil { @@ -254,6 +262,8 @@ func (e *PhysicalExchangeSender) ToPB(ctx sessionctx.Context, storeType kv.Store EncodedTaskMeta: encodedTask, PartitionKeys: hashColPb, Child: child, + Types: hashColTypes, + AllFieldTypes: allFieldTypes, } executorID := e.ExplainID().String() return &tipb.Executor{ diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index 9ff7d1d3689aa..02f986da4da38 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -147,7 +147,21 @@ func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column) la.GroupByItems = []expression.Expression{expression.NewOne()} } } - return child.PruneColumns(selfUsedCols) + err := child.PruneColumns(selfUsedCols) + if err != nil { + return err + } + // Do an extra Projection Elimination here. This is specially for empty Projection below Aggregation. + // This kind of Projection would cause some bugs for MPP plan and is safe to be removed. + // This kind of Projection should be removed in Projection Elimination, but currently PrunColumnsAgain is + // the last rule. So we specially handle this case here. + if childProjection, isProjection := child.(*LogicalProjection); isProjection { + if len(childProjection.Exprs) == 0 && childProjection.Schema().Len() == 0 { + childOfChild := childProjection.children[0] + la.SetChildren(childOfChild) + } + } + return nil } func pruneByItems(old []*util.ByItems) (new []*util.ByItems, parentUsedCols []*expression.Column) { @@ -225,6 +239,23 @@ func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column) erro p.schema.Columns = append(p.schema.Columns[:i], p.schema.Columns[i+1:]...) } } + // It's possible that the child operator adds extra columns to the schema. + // Currently, (*LogicalAggregation).PruneColumns() might do this. + // But we don't need such columns, so we add an extra Projection to prune this column when this happened. + for i, child := range p.Children() { + if p.schema.Len() < child.Schema().Len() { + schema := p.schema.Clone() + exprs := make([]expression.Expression, len(p.schema.Columns)) + for j, col := range schema.Columns { + exprs[j] = col + } + proj := LogicalProjection{Exprs: exprs, AvoidColumnEvaluator: true}.Init(p.ctx, p.blockOffset) + proj.SetSchema(schema) + + proj.SetChildren(child) + p.children[i] = proj + } + } } return nil } diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index 041420218de5b..8e42e4cadf916 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -148,7 +148,8 @@ func (s *partitionProcessor) findUsedPartitions(ctx sessionctx.Context, tbl tabl highLowVals = append(highLowVals, r.LowVal...) pos, isNull, err := pe.EvalInt(ctx, chunk.MutRowFromDatums(highLowVals).ToRow()) if err != nil { - return nil, nil, err + // If we failed to get the point position, we can just skip and ignore it. + continue } if isNull { pos = 0 @@ -327,8 +328,7 @@ func (s *partitionProcessor) processHashPartition(ds *DataSource, pi *model.Part } used, err := s.pruneHashPartition(ds.SCtx(), ds.table, ds.partitionNames, ds.allConds, ds.TblCols, names) if err != nil { - // Just report warning and generate the tableDual - ds.SCtx().GetSessionVars().StmtCtx.AppendWarning(err) + return nil, err } if used != nil { return s.makeUnionAllChildren(ds, pi, convertToRangeOr(used, pi)) @@ -954,6 +954,9 @@ func partitionRangeForExpr(sctx sessionctx.Context, expr expression.Expression, if p, ok := pruner.(*rangePruner); ok { newRange := partitionRangeForInExpr(sctx, op.GetArgs(), p) return result.intersection(newRange) + } else if p, ok := pruner.(*rangeColumnsPruner); ok { + newRange := partitionRangeColumnForInExpr(sctx, op.GetArgs(), p) + return result.intersection(newRange) } return result } @@ -1014,6 +1017,43 @@ func partitionRangeForOrExpr(sctx sessionctx.Context, expr1, expr2 expression.Ex return tmp1.union(tmp2) } +func partitionRangeColumnForInExpr(sctx sessionctx.Context, args []expression.Expression, + pruner *rangeColumnsPruner) partitionRangeOR { + col, ok := args[0].(*expression.Column) + if !ok || col.ID != pruner.partCol.ID { + return pruner.fullRange() + } + + var result partitionRangeOR + for i := 1; i < len(args); i++ { + constExpr, ok := args[i].(*expression.Constant) + if !ok { + return pruner.fullRange() + } + switch constExpr.Value.Kind() { + case types.KindInt64, types.KindUint64, types.KindMysqlTime: // for safety, only support int and datetime now + case types.KindNull: + result = append(result, partitionRange{0, 1}) + continue + default: + return pruner.fullRange() + } + + // convert all elements to EQ-exprs and prune them one by one + sf, err := expression.NewFunction(sctx, ast.EQ, types.NewFieldType(types.KindInt64), []expression.Expression{col, args[i]}...) + if err != nil { + return pruner.fullRange() + } + start, end, ok := pruner.partitionRangeForExpr(sctx, sf) + if !ok { + return pruner.fullRange() + } + result = append(result, partitionRange{start, end}) + } + + return result.simplify() +} + func partitionRangeForInExpr(sctx sessionctx.Context, args []expression.Expression, pruner *rangePruner) partitionRangeOR { col, ok := args[0].(*expression.Column) diff --git a/planner/core/rule_predicate_push_down.go b/planner/core/rule_predicate_push_down.go index c4a350420b3ee..c49c7a21e73a3 100644 --- a/planner/core/rule_predicate_push_down.go +++ b/planner/core/rule_predicate_push_down.go @@ -286,10 +286,14 @@ func (p *LogicalProjection) appendExpr(expr expression.Expression) *expression.C col := &expression.Column{ UniqueID: p.ctx.GetSessionVars().AllocPlanColumnID(), - RetType: expr.GetType(), + RetType: expr.GetType().Clone(), } col.SetCoercibility(expr.Coercibility()) p.schema.Append(col) + // reset ParseToJSONFlag in order to keep the flag away from json column + if col.GetType().Tp == mysql.TypeJSON { + col.GetType().Flag &= ^mysql.ParseToJSONFlag + } return col } diff --git a/planner/core/stringer.go b/planner/core/stringer.go index 346b5b50e5742..5d11bf109892a 100644 --- a/planner/core/stringer.go +++ b/planner/core/stringer.go @@ -27,10 +27,25 @@ func ToString(p Plan) string { return strings.Join(strs, "->") } +func needIncludeChildrenString(plan Plan) bool { + switch x := plan.(type) { + case *LogicalUnionAll, *PhysicalUnionAll, *LogicalPartitionUnionAll: + // after https://github.com/pingcap/tidb/pull/25218, the union may contain less than 2 children, + // but we still wants to include its child plan's information when calling `toString` on union. + return true + case LogicalPlan: + return len(x.Children()) > 1 + case PhysicalPlan: + return len(x.Children()) > 1 + default: + return false + } +} + func toString(in Plan, strs []string, idxs []int) ([]string, []int) { switch x := in.(type) { case LogicalPlan: - if len(x.Children()) > 1 { + if needIncludeChildrenString(in) { idxs = append(idxs, len(strs)) } @@ -39,7 +54,7 @@ func toString(in Plan, strs []string, idxs []int) ([]string, []int) { } case *PhysicalExchangeReceiver: // do nothing case PhysicalPlan: - if len(x.Children()) > 1 { + if needIncludeChildrenString(in) { idxs = append(idxs, len(strs)) } diff --git a/planner/core/task.go b/planner/core/task.go index cc3486c160c77..882c0a588b276 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -1215,14 +1215,6 @@ func (p *PhysicalTopN) canPushDown(storeTp kv.StoreType) bool { return expression.CanExprsPushDown(p.ctx.GetSessionVars().StmtCtx, exprs, p.ctx.GetClient(), storeTp) } -func (p *PhysicalTopN) allColsFromSchema(schema *expression.Schema) bool { - cols := make([]*expression.Column, 0, len(p.ByItems)) - for _, item := range p.ByItems { - cols = append(cols, expression.ExtractColumns(item.Expr)...) - } - return len(schema.ColumnsIndices(cols)) > 0 -} - // GetCost computes the cost of in memory sort. func (p *PhysicalSort) GetCost(count float64, schema *expression.Schema) float64 { if count < 2.0 { @@ -1280,14 +1272,36 @@ func (p *PhysicalTopN) getPushedDownTopN(childPlan PhysicalPlan) *PhysicalTopN { return topN } +// canPushToIndexPlan checks if this TopN can be pushed to the index side of copTask. +// It can be pushed to the index side when all columns used by ByItems are available from the index side and +// there's no prefix index column. +func (p *PhysicalTopN) canPushToIndexPlan(indexPlan PhysicalPlan, byItemCols []*expression.Column) bool { + schema := indexPlan.Schema() + for _, col := range byItemCols { + pos := schema.ColumnIndex(col) + if pos == -1 { + return false + } + if schema.Columns[pos].IsPrefix { + return false + } + } + return true +} + func (p *PhysicalTopN) attach2Task(tasks ...task) task { t := tasks[0].copy() inputCount := t.count() - if copTask, ok := t.(*copTask); ok && p.canPushDown(copTask.getStoreType()) && len(copTask.rootTaskConds) == 0 { + cols := make([]*expression.Column, 0, len(p.ByItems)) + for _, item := range p.ByItems { + cols = append(cols, expression.ExtractColumns(item.Expr)...) + } + needPushDown := len(cols) > 0 + if copTask, ok := t.(*copTask); ok && needPushDown && p.canPushDown(copTask.getStoreType()) && len(copTask.rootTaskConds) == 0 { // If all columns in topN are from index plan, we push it to index plan, otherwise we finish the index plan and // push it to table plan. var pushedDownTopN *PhysicalTopN - if !copTask.indexPlanFinished && p.allColsFromSchema(copTask.indexPlan.Schema()) { + if !copTask.indexPlanFinished && p.canPushToIndexPlan(copTask.indexPlan, cols) { pushedDownTopN = p.getPushedDownTopN(copTask.indexPlan) copTask.indexPlan = pushedDownTopN } else { @@ -1296,7 +1310,7 @@ func (p *PhysicalTopN) attach2Task(tasks ...task) task { copTask.tablePlan = pushedDownTopN } copTask.addCost(pushedDownTopN.GetCost(inputCount, false)) - } else if mppTask, ok := t.(*mppTask); ok && p.canPushDown(kv.TiFlash) { + } else if mppTask, ok := t.(*mppTask); ok && needPushDown && p.canPushDown(kv.TiFlash) { pushedDownTopN := p.getPushedDownTopN(mppTask.p) mppTask.p = pushedDownTopN } diff --git a/planner/core/testdata/analyze_suite_out.json b/planner/core/testdata/analyze_suite_out.json index cb0dd2137c515..bdd31cd56f8cd 100644 --- a/planner/core/testdata/analyze_suite_out.json +++ b/planner/core/testdata/analyze_suite_out.json @@ -435,14 +435,14 @@ { "Name": "TestAnalyze", "Cases": [ - "Analyze{Table(_tidb_rowid, a, b, _tidb_rowid)}", + "Analyze{Index(a),Table(a, b)}", "TableReader(Table(t)->Sel([le(test.t.a, 2)]))", "IndexReader(Index(t.b)[[-inf,2)])", "TableReader(Table(t)->Sel([eq(test.t.a, 1) le(test.t.b, 2)]))", "TableReader(Table(t1)->Sel([le(test.t1.a, 2)]))", "IndexLookUp(Index(t1.a)[[1,1]], Table(t1)->Sel([le(test.t1.b, 2)]))", "TableReader(Table(t2)->Sel([le(test.t2.a, 2)]))", - "Analyze{Table(_tidb_rowid, a, b, _tidb_rowid)}", + "Analyze{Index(a),Index(b)}", "PartitionUnionAll{TableReader(Table(t4)->Sel([le(test.t4.a, 2)]))->TableReader(Table(t4)->Sel([le(test.t4.a, 2)]))}", "PartitionUnionAll{IndexReader(Index(t4.b)[[-inf,2)])->IndexReader(Index(t4.b)[[-inf,2)])}", "TableReader(Table(t4)->Sel([eq(test.t4.a, 1) le(test.t4.b, 2)]))" diff --git a/planner/core/testdata/integration_serial_suite_out.json b/planner/core/testdata/integration_serial_suite_out.json index c54a5250592ab..5ff9af9f22f77 100644 --- a/planner/core/testdata/integration_serial_suite_out.json +++ b/planner/core/testdata/integration_serial_suite_out.json @@ -251,59 +251,56 @@ { "SQL": "explain format = 'brief' select count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader 1.00 root data:ExchangeSender", - " └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin 8.00 batchCop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", - " ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ", - " │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast", - " │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false", - " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false" + "StreamAgg 1.00 root funcs:count(1)->Column#11", + "└─TableReader 8.00 root data:ExchangeSender", + " └─ExchangeSender 8.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", + " ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan 2.00 cop[tiflash] table:d1_t keep order:false", + " └─Selection(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain format = 'brief' select count(*) from fact_t, d1_t, d2_t, d3_t where fact_t.d1_k = d1_t.d1_k and fact_t.d2_k = d2_t.d2_k and fact_t.d3_k = d3_t.d3_k", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#18)->Column#17", - "└─TableReader 1.00 root data:ExchangeSender", - " └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#18", - " └─HashJoin 8.00 batchCop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]", - " ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ", - " │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast", - " │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d3_t.d3_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d3_t keep order:false", - " └─HashJoin(Probe) 8.00 batchCop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]", - " ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ", - " │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast", - " │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d2_t.d2_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d2_t keep order:false", - " └─HashJoin(Probe) 8.00 batchCop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", - " ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ", - " │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast", - " │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false", - " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))", - " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false" + "StreamAgg 1.00 root funcs:count(1)->Column#17", + "└─TableReader 8.00 root data:ExchangeSender", + " └─ExchangeSender 8.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]", + " ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 2.00 cop[tiflash] not(isnull(test.d3_t.d3_k))", + " │ └─TableFullScan 2.00 cop[tiflash] table:d3_t keep order:false", + " └─HashJoin(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]", + " ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 2.00 cop[tiflash] not(isnull(test.d2_t.d2_k))", + " │ └─TableFullScan 2.00 cop[tiflash] table:d2_t keep order:false", + " └─HashJoin(Probe) 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", + " ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan 2.00 cop[tiflash] table:d1_t keep order:false", + " └─Selection(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))", + " └─TableFullScan 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { "SQL": "explain format = 'brief' select count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader 1.00 root data:ExchangeSender", - " └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin 8.00 batchCop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", - " ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ", - " │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast", - " │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false", - " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false" + "StreamAgg 1.00 root funcs:count(1)->Column#11", + "└─TableReader 8.00 root data:ExchangeSender", + " └─ExchangeSender 8.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", + " ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan 2.00 cop[tiflash] table:d1_t keep order:false", + " └─Selection(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { @@ -337,17 +334,16 @@ { "SQL": "explain format = 'brief' select count(*) from fact_t join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > d1_t.value", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader 1.00 root data:ExchangeSender", - " └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin 8.00 batchCop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)], other cond:gt(test.fact_t.col1, test.d1_t.value)", - " ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ", - " │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast", - " │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false", - " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", - " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false" + "StreamAgg 1.00 root funcs:count(1)->Column#11", + "└─TableReader 8.00 root data:ExchangeSender", + " └─ExchangeSender 8.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin 8.00 cop[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)], other cond:gt(test.fact_t.col1, test.d1_t.value)", + " ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 2.00 cop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", + " │ └─TableFullScan 2.00 cop[tiflash] table:d1_t keep order:false", + " └─Selection(Probe) 8.00 cop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", + " └─TableFullScan 8.00 cop[tiflash] table:fact_t keep order:false" ] }, { diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 7aea53ef666b7..1ad39968c87b3 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -984,8 +984,8 @@ { "SQL": "select * from t1 where t1.a = 1 and t1.b < \"333\"", "Plan": [ - "TableReader 0.82 root data:TableRangeScan", - "└─TableRangeScan 0.82 cop[tikv] table:t1 range:[1 -inf,1 \"333\"), keep order:false" + "TableReader 0.67 root data:TableRangeScan", + "└─TableRangeScan 0.67 cop[tikv] table:t1 range:[1 -inf,1 \"333\"), keep order:false" ], "Res": [ "1 111 1.1000000000 11" diff --git a/session/session.go b/session/session.go index c1d4c17a1a297..a9c2544b1ced4 100644 --- a/session/session.go +++ b/session/session.go @@ -69,6 +69,7 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/statistics/handle" + "github.com/pingcap/tidb/store/driver/txn" "github.com/pingcap/tidb/store/tikv" tikvstore "github.com/pingcap/tidb/store/tikv/kv" tikvutil "github.com/pingcap/tidb/store/tikv/util" @@ -544,14 +545,15 @@ func (s *session) doCommit(ctx context.Context) error { type temporaryTableKVFilter map[int64]tableutil.TempTable -func (m temporaryTableKVFilter) IsUnnecessaryKeyValue(key, value []byte, flags tikvstore.KeyFlags) bool { +func (m temporaryTableKVFilter) IsUnnecessaryKeyValue(key, value []byte, flags tikvstore.KeyFlags) (bool, error) { tid := tablecodec.DecodeTableID(key) if _, ok := m[tid]; ok { - return true + return true, nil } // This is the default filter for all tables. - return tablecodec.IsUntouchedIndexKValue(key, value) + defaultFilter := txn.TiDBKVFilter{} + return defaultFilter.IsUnnecessaryKeyValue(key, value, flags) } // errIsNoisy is used to filter DUPLCATE KEY errors. diff --git a/session/session_test.go b/session/session_test.go index b6169318d4251..df568c8969737 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -4824,3 +4824,27 @@ func (s *testStatisticsSuite) TestNewCollationStatsWithPrefixIndex(c *C) { "1 3 15 0 2 0", )) } + +func (s *testSessionSuite) TestFixSetTiDBSnapshotTS(c *C) { + tk := testkit.NewTestKit(c, s.store) + safePointName := "tikv_gc_safe_point" + safePointValue := "20160102-15:04:05 -0700" + safePointComment := "All versions after safe point can be accessed. (DO NOT EDIT)" + updateSafePoint := fmt.Sprintf(`INSERT INTO mysql.tidb VALUES ('%[1]s', '%[2]s', '%[3]s') + ON DUPLICATE KEY + UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) + tk.MustExec(updateSafePoint) + tk.MustExec("create database t123") + time.Sleep(time.Second) + ts := time.Now().Format("2006-1-2 15:04:05") + time.Sleep(time.Second) + tk.MustExec("drop database t123") + err := tk.ExecToErr("use t123") + c.Assert(err, NotNil) + c.Assert(err.Error(), Matches, ".*Unknown database.*") + tk.MustExec(fmt.Sprintf("set @@tidb_snapshot='%s'", ts)) + tk.MustExec("use t123") + // update any session variable and assert whether infoschema is changed + tk.MustExec("SET SESSION sql_mode = 'STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER';") + tk.MustExec("use t123") +} diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 2325f50fdb46e..6331994022304 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -416,6 +416,7 @@ type SessionVars struct { // mppTaskIDAllocator is used to allocate mpp task id for a session. mppTaskIDAllocator struct { + mu sync.Mutex lastTS uint64 taskID int64 } @@ -866,6 +867,8 @@ type SessionVars struct { // AllocMPPTaskID allocates task id for mpp tasks. It will reset the task id if the query's // startTs is different. func (s *SessionVars) AllocMPPTaskID(startTS uint64) int64 { + s.mppTaskIDAllocator.mu.Lock() + defer s.mppTaskIDAllocator.mu.Unlock() if s.mppTaskIDAllocator.lastTS == startTS { s.mppTaskIDAllocator.taskID++ return s.mppTaskIDAllocator.taskID diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 8c739f443a727..bbaa0a36fdf07 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1252,7 +1252,7 @@ var defaultSysVars = []*SysVar{ s.EnableFastAnalyze = TiDBOptOn(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBSkipIsolationLevelCheck, skipInit: true, Value: BoolToOnOff(DefTiDBSkipIsolationLevelCheck), Type: TypeBool}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBSkipIsolationLevelCheck, Value: BoolToOnOff(DefTiDBSkipIsolationLevelCheck), Type: TypeBool}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableRateLimitAction, Value: BoolToOnOff(DefTiDBEnableRateLimitAction), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { s.EnabledRateLimitAction = TiDBOptOn(val) return nil diff --git a/sessionctx/variable/sysvar_test.go b/sessionctx/variable/sysvar_test.go index 58f6ac803c563..0687c9a959bd6 100644 --- a/sessionctx/variable/sysvar_test.go +++ b/sessionctx/variable/sysvar_test.go @@ -394,8 +394,21 @@ func (*testSysVarSuite) TestTxnIsolation(c *C) { _, err = sv.Validate(vars, "read-uncommitted", ScopeSession) c.Assert(err.Error(), Equals, "[variable:8048]The isolation level 'READ-UNCOMMITTED' is not supported. Set tidb_skip_isolation_level_check=1 to skip this error") - vars.systems[TiDBSkipIsolationLevelCheck] = "ON" + // Enable global skip isolation check doesn't affect current session + c.Assert(GetSysVar(TiDBSkipIsolationLevelCheck).SetGlobalFromHook(vars, "ON", true), IsNil) + _, err = sv.Validate(vars, "Serializable", ScopeSession) + c.Assert(err.Error(), Equals, "[variable:8048]The isolation level 'SERIALIZABLE' is not supported. Set tidb_skip_isolation_level_check=1 to skip this error") + + // Enable session skip isolation check + c.Assert(GetSysVar(TiDBSkipIsolationLevelCheck).SetSessionFromHook(vars, "ON"), IsNil) + + val, err = sv.Validate(vars, "Serializable", ScopeSession) + c.Assert(err, IsNil) + c.Assert(val, Equals, "SERIALIZABLE") + // Init TiDBSkipIsolationLevelCheck like what loadCommonGlobalVariables does + vars = NewSessionVars() + c.Assert(vars.SetSystemVarWithRelaxedValidation(TiDBSkipIsolationLevelCheck, "1"), IsNil) val, err = sv.Validate(vars, "Serializable", ScopeSession) c.Assert(err, IsNil) c.Assert(val, Equals, "SERIALIZABLE") diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 65f6f01751745..e8b5990e471c7 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -715,7 +715,7 @@ const ( DefTiDBEnableAsyncCommit = false DefTiDBEnable1PC = false DefTiDBGuaranteeLinearizability = true - DefTiDBAnalyzeVersion = 2 + DefTiDBAnalyzeVersion = 1 DefTiDBEnableIndexMergeJoin = false DefTiDBTrackAggregateMemoryUsage = true DefTiDBEnableExchangePartition = false diff --git a/statistics/handle/ddl_test.go b/statistics/handle/ddl_test.go index 6bdd897270b8d..c62e80d372766 100644 --- a/statistics/handle/ddl_test.go +++ b/statistics/handle/ddl_test.go @@ -179,8 +179,6 @@ func (s *testStatsSuite) TestDDLHistogram(c *C) { rs := testKit.MustQuery("select count(*) from mysql.stats_histograms where table_id = ? and hist_id = 1 and is_index =1", tableInfo.ID) rs.Check(testkit.Rows("1")) rs = testKit.MustQuery("select count(*) from mysql.stats_buckets where table_id = ? and hist_id = 1 and is_index = 1", tableInfo.ID) - rs.Check(testkit.Rows("0")) - rs = testKit.MustQuery("select count(*) from mysql.stats_top_n where table_id = ? and hist_id = 1 and is_index = 1", tableInfo.ID) rs.Check(testkit.Rows("2")) } diff --git a/statistics/handle/update_test.go b/statistics/handle/update_test.go index 644da6dca1321..2d6ecc4a1a67a 100644 --- a/statistics/handle/update_test.go +++ b/statistics/handle/update_test.go @@ -504,8 +504,7 @@ func (s *testStatsSuite) TestAutoUpdate(c *C) { hg, ok := stats.Indices[tableInfo.Indices[0].ID] c.Assert(ok, IsTrue) c.Assert(hg.NDV, Equals, int64(3)) - c.Assert(hg.Len(), Equals, 0) - c.Assert(hg.TopN.Num(), Equals, 3) + c.Assert(hg.Len(), Equals, 3) }) } diff --git a/statistics/testdata/stats_suite_out.json b/statistics/testdata/stats_suite_out.json index c70e967949881..3717321931d2b 100644 --- a/statistics/testdata/stats_suite_out.json +++ b/statistics/testdata/stats_suite_out.json @@ -607,7 +607,7 @@ }, { "SQL": "select * from t where a < 8 and (b > 10 or c < 3 or b > 4) and a > 2", - "Selectivity": 0 + "Selectivity": 0.3125 } ] } diff --git a/store/copr/batch_coprocessor.go b/store/copr/batch_coprocessor.go index 5db4ae5cc64e9..e796b00da5da6 100644 --- a/store/copr/batch_coprocessor.go +++ b/store/copr/batch_coprocessor.go @@ -104,10 +104,11 @@ func (rs *batchCopResponse) RespTime() time.Duration { // if there is only 1 available store, then put the region to the related store // otherwise, use a greedy algorithm to put it into the store with highest weight func balanceBatchCopTask(ctx context.Context, kvStore *kvStore, originalTasks []*batchCopTask, mppStoreLastFailTime map[string]time.Time, ttl time.Duration) []*batchCopTask { - if len(originalTasks) <= 1 { + isMPP := mppStoreLastFailTime != nil + // for mpp, we still need to detect the store availability + if len(originalTasks) <= 1 && !isMPP { return originalTasks } - isMPP := mppStoreLastFailTime != nil cache := kvStore.GetRegionCache() storeTaskMap := make(map[uint64]*batchCopTask) // storeCandidateRegionMap stores all the possible store->region map. Its content is @@ -231,16 +232,28 @@ func balanceBatchCopTask(ctx context.Context, kvStore *kvStore, originalTasks [] } } } - if totalRemainingRegionNum == 0 { - return originalTasks - } - avgStorePerRegion := float64(totalRegionCandidateNum) / float64(totalRemainingRegionNum) - findNextStore := func(candidateStores []uint64) uint64 { - store := uint64(math.MaxUint64) - weightedRegionNum := math.MaxFloat64 - if candidateStores != nil { - for _, storeID := range candidateStores { + if totalRemainingRegionNum > 0 { + avgStorePerRegion := float64(totalRegionCandidateNum) / float64(totalRemainingRegionNum) + findNextStore := func(candidateStores []uint64) uint64 { + store := uint64(math.MaxUint64) + weightedRegionNum := math.MaxFloat64 + if candidateStores != nil { + for _, storeID := range candidateStores { + if _, validStore := storeCandidateRegionMap[storeID]; !validStore { + continue + } + num := float64(len(storeCandidateRegionMap[storeID]))/avgStorePerRegion + float64(len(storeTaskMap[storeID].regionInfos)) + if num < weightedRegionNum { + store = storeID + weightedRegionNum = num + } + } + if store != uint64(math.MaxUint64) { + return store + } + } + for storeID := range storeTaskMap { if _, validStore := storeCandidateRegionMap[storeID]; !validStore { continue } @@ -250,57 +263,44 @@ func balanceBatchCopTask(ctx context.Context, kvStore *kvStore, originalTasks [] weightedRegionNum = num } } - if store != uint64(math.MaxUint64) { - return store - } + return store } - for storeID := range storeTaskMap { - if _, validStore := storeCandidateRegionMap[storeID]; !validStore { - continue + + store := findNextStore(nil) + for totalRemainingRegionNum > 0 { + if store == uint64(math.MaxUint64) { + break } - num := float64(len(storeCandidateRegionMap[storeID]))/avgStorePerRegion + float64(len(storeTaskMap[storeID].regionInfos)) - if num < weightedRegionNum { - store = storeID - weightedRegionNum = num + var key string + var ri RegionInfo + for key, ri = range storeCandidateRegionMap[store] { + // get the first region + break } - } - return store - } - - store := findNextStore(nil) - for totalRemainingRegionNum > 0 { - if store == uint64(math.MaxUint64) { - break - } - var key string - var ri RegionInfo - for key, ri = range storeCandidateRegionMap[store] { - // get the first region - break - } - storeTaskMap[store].regionInfos = append(storeTaskMap[store].regionInfos, ri) - totalRemainingRegionNum-- - for _, id := range ri.AllStores { - if _, ok := storeCandidateRegionMap[id]; ok { - delete(storeCandidateRegionMap[id], key) - totalRegionCandidateNum-- - if len(storeCandidateRegionMap[id]) == 0 { - delete(storeCandidateRegionMap, id) + storeTaskMap[store].regionInfos = append(storeTaskMap[store].regionInfos, ri) + totalRemainingRegionNum-- + for _, id := range ri.AllStores { + if _, ok := storeCandidateRegionMap[id]; ok { + delete(storeCandidateRegionMap[id], key) + totalRegionCandidateNum-- + if len(storeCandidateRegionMap[id]) == 0 { + delete(storeCandidateRegionMap, id) + } } } + if totalRemainingRegionNum > 0 { + avgStorePerRegion = float64(totalRegionCandidateNum) / float64(totalRemainingRegionNum) + // it is not optimal because we only check the stores that affected by this region, in fact in order + // to find out the store with the lowest weightedRegionNum, all stores should be checked, but I think + // check only the affected stores is more simple and will get a good enough result + store = findNextStore(ri.AllStores) + } } if totalRemainingRegionNum > 0 { - avgStorePerRegion = float64(totalRegionCandidateNum) / float64(totalRemainingRegionNum) - // it is not optimal because we only check the stores that affected by this region, in fact in order - // to find out the store with the lowest weightedRegionNum, all stores should be checked, but I think - // check only the affected stores is more simple and will get a good enough result - store = findNextStore(ri.AllStores) + logutil.BgLogger().Warn("Some regions are not used when trying to balance batch cop task, give up balancing") + return originalTasks } } - if totalRemainingRegionNum > 0 { - logutil.BgLogger().Warn("Some regions are not used when trying to balance batch cop task, give up balancing") - return originalTasks - } var ret []*batchCopTask for _, task := range storeTaskMap { diff --git a/store/driver/txn/txn_driver.go b/store/driver/txn/txn_driver.go index 93a4f4a9508df..b7f011d6c1836 100644 --- a/store/driver/txn/txn_driver.go +++ b/store/driver/txn/txn_driver.go @@ -29,6 +29,8 @@ import ( tikverr "github.com/pingcap/tidb/store/tikv/error" tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/util/logutil" + "go.uber.org/zap" ) type tikvTxn struct { @@ -235,6 +237,15 @@ func (txn *tikvTxn) extractKeyExistsErr(key kv.Key) error { type TiDBKVFilter struct{} // IsUnnecessaryKeyValue defines which kinds of KV pairs from TiDB needn't be committed. -func (f TiDBKVFilter) IsUnnecessaryKeyValue(key, value []byte, flags tikvstore.KeyFlags) bool { - return tablecodec.IsUntouchedIndexKValue(key, value) +func (f TiDBKVFilter) IsUnnecessaryKeyValue(key, value []byte, flags tikvstore.KeyFlags) (bool, error) { + isUntouchedValue := tablecodec.IsUntouchedIndexKValue(key, value) + if isUntouchedValue && flags.HasPresumeKeyNotExists() { + logutil.BgLogger().Error("unexpected path the untouched key value with PresumeKeyNotExists flag", + zap.Stringer("key", kv.Key(key)), zap.Stringer("value", kv.Key(value)), + zap.Uint16("flags", uint16(flags)), zap.Stack("stack")) + return false, errors.Errorf( + "unexpected path the untouched key=%s value=%s contains PresumeKeyNotExists flag keyFlags=%v", + kv.Key(key).String(), kv.Key(value).String(), flags) + } + return isUntouchedValue, nil } diff --git a/store/tikv/2pc.go b/store/tikv/2pc.go index f8be95e741a99..37e784b46de4a 100644 --- a/store/tikv/2pc.go +++ b/store/tikv/2pc.go @@ -323,7 +323,7 @@ func (c *twoPhaseCommitter) extractKeyExistsErr(err *tikverr.ErrKeyExist) error // KVFilter is a filter that filters out unnecessary KV pairs. type KVFilter interface { // IsUnnecessaryKeyValue returns whether this KV pair should be committed. - IsUnnecessaryKeyValue(key, value []byte, flags kv.KeyFlags) bool + IsUnnecessaryKeyValue(key, value []byte, flags kv.KeyFlags) (bool, error) } func (c *twoPhaseCommitter) initKeysAndMutations() error { @@ -353,7 +353,13 @@ func (c *twoPhaseCommitter) initKeysAndMutations() error { } else { value = it.Value() if len(value) > 0 { - isUnnecessaryKV := filter != nil && filter.IsUnnecessaryKeyValue(key, value, flags) + var isUnnecessaryKV bool + if filter != nil { + isUnnecessaryKV, err = filter.IsUnnecessaryKeyValue(key, value, flags) + if err != nil { + return err + } + } if isUnnecessaryKV { if !flags.HasLocked() { continue diff --git a/table/tables/index.go b/table/tables/index.go index aef03d0590aaa..e1e44823a911f 100644 --- a/table/tables/index.go +++ b/table/tables/index.go @@ -168,8 +168,22 @@ func (c *index) Create(sctx sessionctx.Context, txn kv.Transaction, indexedValue // should not overwrite the key with un-commit flag. // So if the key exists, just do nothing and return. v, err := txn.GetMemBuffer().Get(ctx, key) - if err == nil && len(v) != 0 { - return nil, nil + if err == nil { + if len(v) != 0 { + return nil, nil + } + // The key is marked as deleted in the memory buffer, as the existence check is done lazily + // for optimistic transactions by default. The "untouched" key could still exist in the store, + // it's needed to commit this key to do the existence check so unset the untouched flag. + if !txn.IsPessimistic() { + keyFlags, err := txn.GetMemBuffer().GetFlags(key) + if err != nil { + return nil, err + } + if keyFlags.HasPresumeKeyNotExists() { + opt.Untouched = false + } + } } } diff --git a/table/tables/tables_test.go b/table/tables/tables_test.go index ec5b82e351c74..5fb6ebf1d1f06 100644 --- a/table/tables/tables_test.go +++ b/table/tables/tables_test.go @@ -746,3 +746,29 @@ func (ts *testSuite) TestViewColumns(c *C) { "Warning|1356|View 'test.va' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them")) } } + +func (ts *testSuite) TestConstraintCheckForOptimisticUntouched(c *C) { + se, err := session.CreateSession4Test(ts.store) + c.Assert(err, IsNil) + c.Assert(se.Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil), IsTrue) + tk := testkit.NewTestKitWithSession(c, ts.store, se) + + tk.MustExec("use test") + tk.MustExec("drop table if exists test_optimistic_untouched_flag;") + tk.MustExec(`create table test_optimistic_untouched_flag(c0 int, c1 varchar(20), c2 varchar(20), unique key uk(c0));`) + tk.MustExec(`insert into test_optimistic_untouched_flag(c0, c1, c2) values (1, null, 'green');`) + + // Insert a row with duplicated entry on the unique key, the commit should fail. + tk.MustExec("begin optimistic;") + tk.MustExec(`insert into test_optimistic_untouched_flag(c0, c1, c2) values (1, 'red', 'white');`) + tk.MustExec(`delete from test_optimistic_untouched_flag where c1 is null;`) + tk.MustExec("update test_optimistic_untouched_flag set c2 = 'green' where c2 between 'purple' and 'white';") + err = tk.ExecToErr("commit") + c.Assert(err, NotNil) + + tk.MustExec("begin optimistic;") + tk.MustExec(`insert into test_optimistic_untouched_flag(c0, c1, c2) values (1, 'red', 'white');`) + tk.MustExec("update test_optimistic_untouched_flag set c2 = 'green' where c2 between 'purple' and 'white';") + err = tk.ExecToErr("commit") + c.Assert(err, NotNil) +} diff --git a/tablecodec/tablecodec_test.go b/tablecodec/tablecodec_test.go index ada284edc0bca..cf461438cf82b 100644 --- a/tablecodec/tablecodec_test.go +++ b/tablecodec/tablecodec_test.go @@ -54,6 +54,20 @@ func (s *testTableCodecSuite) TestTableCodec(c *C) { c.Assert(h.IntValue(), Equals, int64(2)) } +// https://github.com/pingcap/tidb/issues/27687. +func (s *testTableCodecSuite) TestTableCodecInvalid(c *C) { + tableID := int64(100) + buf := make([]byte, 0, 11) + buf = append(buf, 't') + buf = codec.EncodeInt(buf, tableID) + buf = append(buf, '_', 'r') + buf = codec.EncodeInt(buf, -9078412423848787968) + buf = append(buf, '0') + _, err := DecodeRowKey(buf) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "invalid encoded key") +} + // column is a structure used for test type column struct { id int64 diff --git a/types/field_name.go b/types/field_name.go index 330c26041e544..918ca0fa3b87d 100644 --- a/types/field_name.go +++ b/types/field_name.go @@ -34,6 +34,8 @@ type FieldName struct { // update stmt can write `writeable` column implicitly but cannot use non-public columns explicit. // e.g. update t set a = 10 where b = 10; which `b` is in `writeOnly` state NotExplicitUsable bool + + Redundant bool } const emptyName = "EMPTY_NAME" diff --git a/types/time.go b/types/time.go index 994362dea08b4..7dbafacad131f 100644 --- a/types/time.go +++ b/types/time.go @@ -3215,7 +3215,7 @@ func microSeconds(t *CoreTime, input string, ctx map[string]int) (string, bool) if !ok { return input, false } - for v > 0 && v*10 < 1000000 { + for i := length; i < 6; i++ { v *= 10 } t.setMicrosecond(uint32(v)) diff --git a/util/chunk/mutrow.go b/util/chunk/mutrow.go index 46a9a3a1fe850..c6bb9d6614346 100644 --- a/util/chunk/mutrow.go +++ b/util/chunk/mutrow.go @@ -210,12 +210,19 @@ func makeMutRowBytesColumn(bin []byte) *Column { return col } +func cleanColOfMutRow(col *Column) { + for i := range col.offsets { + col.offsets[i] = 0 + } + col.nullBitmap[0] = 0 +} + // SetRow sets the MutRow with Row. func (mr MutRow) SetRow(row Row) { for colIdx, rCol := range row.c.columns { mrCol := mr.c.columns[colIdx] + cleanColOfMutRow(mrCol) if rCol.IsNull(row.idx) { - mrCol.nullBitmap[0] = 0 continue } elemLen := len(rCol.elemBuf) @@ -238,8 +245,8 @@ func (mr MutRow) SetValues(vals ...interface{}) { // SetValue sets the MutRow with colIdx and value. func (mr MutRow) SetValue(colIdx int, val interface{}) { col := mr.c.columns[colIdx] + cleanColOfMutRow(col) if val == nil { - col.nullBitmap[0] = 0 return } switch x := val.(type) { @@ -285,8 +292,8 @@ func (mr MutRow) SetDatums(datums ...types.Datum) { // SetDatum sets the MutRow with colIdx and datum. func (mr MutRow) SetDatum(colIdx int, d types.Datum) { col := mr.c.columns[colIdx] + cleanColOfMutRow(col) if d.IsNull() { - col.nullBitmap[0] = 0 return } switch d.Kind() { diff --git a/util/chunk/mutrow_test.go b/util/chunk/mutrow_test.go index d55603ff848c6..15f8081ddffd1 100644 --- a/util/chunk/mutrow_test.go +++ b/util/chunk/mutrow_test.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/json" + "github.com/pingcap/tidb/util/hack" ) func (s *testChunkSuite) TestMutRow(c *check.C) { @@ -87,6 +88,27 @@ func (s *testChunkSuite) TestMutRow(c *check.C) { c.Assert(chk.columns[0].data, check.BytesEquals, mutRow.c.columns[0].data) } +func (s *testChunkSuite) TestIssue29947(c *check.C) { + mutRow := MutRowFromTypes(allTypes) + nilDatum := types.NewDatum(nil) + + dataBefore := make([][]byte, 0, len(mutRow.c.columns)) + elemBufBefore := make([][]byte, 0, len(mutRow.c.columns)) + for _, col := range mutRow.c.columns { + dataBefore = append(dataBefore, col.data) + elemBufBefore = append(elemBufBefore, col.elemBuf) + } + for i, col := range mutRow.c.columns { + mutRow.SetDatum(i, nilDatum) + c.Assert(col.IsNull(0), check.IsTrue) + for _, off := range col.offsets { + c.Assert(off, check.Equals, int64(0)) + } + c.Assert(hack.String(col.data), check.Equals, hack.String(dataBefore[i])) + c.Assert(hack.String(col.elemBuf), check.Equals, hack.String(elemBufBefore[i])) + } +} + func BenchmarkMutRowSetRow(b *testing.B) { b.ReportAllocs() rowChk := newChunk(8, 0) diff --git a/util/codec/codec.go b/util/codec/codec.go index 2f9ce666ed643..4d16470d56537 100644 --- a/util/codec/codec.go +++ b/util/codec/codec.go @@ -664,8 +664,8 @@ func HashChunkSelected(sc *stmtctx.StatementContext, h []hash.Hash64, chk *chunk // If two rows are logically equal, it will generate the same bytes. func HashChunkRow(sc *stmtctx.StatementContext, w io.Writer, row chunk.Row, allTypes []*types.FieldType, colIdx []int, buf []byte) (err error) { var b []byte - for _, idx := range colIdx { - buf[0], b, err = encodeHashChunkRowIdx(sc, row, allTypes[idx], idx) + for i, idx := range colIdx { + buf[0], b, err = encodeHashChunkRowIdx(sc, row, allTypes[i], idx) if err != nil { return errors.Trace(err) } @@ -687,13 +687,16 @@ func EqualChunkRow(sc *stmtctx.StatementContext, row1 chunk.Row, allTypes1 []*types.FieldType, colIdx1 []int, row2 chunk.Row, allTypes2 []*types.FieldType, colIdx2 []int, ) (bool, error) { + if len(colIdx1) != len(colIdx2) { + return false, errors.Errorf("Internal error: Hash columns count mismatch, col1: %d, col2: %d", len(colIdx1), len(colIdx2)) + } for i := range colIdx1 { idx1, idx2 := colIdx1[i], colIdx2[i] - flag1, b1, err := encodeHashChunkRowIdx(sc, row1, allTypes1[idx1], idx1) + flag1, b1, err := encodeHashChunkRowIdx(sc, row1, allTypes1[i], idx1) if err != nil { return false, errors.Trace(err) } - flag2, b2, err := encodeHashChunkRowIdx(sc, row2, allTypes2[idx2], idx2) + flag2, b2, err := encodeHashChunkRowIdx(sc, row2, allTypes2[i], idx2) if err != nil { return false, errors.Trace(err) } @@ -958,7 +961,9 @@ func peek(b []byte) (length int, err error) { return 0, errors.Trace(err) } length += l - if length > originLength { + if length <= 0 { + return 0, errors.New("invalid encoded key") + } else if length > originLength { return 0, errors.Errorf("invalid encoded key, "+ "expected length: %d, actual length: %d", length, originLength) } diff --git a/util/codec/codec_test.go b/util/codec/codec_test.go index 6863a7a5a0902..78a728928b4c2 100644 --- a/util/codec/codec_test.go +++ b/util/codec/codec_test.go @@ -1247,9 +1247,9 @@ func (s *testCodecSuite) TestHashChunkColumns(c *C) { for i := 0; i < 12; i++ { c.Assert(chk.GetRow(0).IsNull(i), Equals, true) err1 := HashChunkSelected(sc, vecHash, chk, tps[i], i, buf, hasNull, sel, false) - err2 := HashChunkRow(sc, rowHash[0], chk.GetRow(0), tps, colIdx[i:i+1], buf) - err3 := HashChunkRow(sc, rowHash[1], chk.GetRow(1), tps, colIdx[i:i+1], buf) - err4 := HashChunkRow(sc, rowHash[2], chk.GetRow(2), tps, colIdx[i:i+1], buf) + err2 := HashChunkRow(sc, rowHash[0], chk.GetRow(0), tps[i:i+1], colIdx[i:i+1], buf) + err3 := HashChunkRow(sc, rowHash[1], chk.GetRow(1), tps[i:i+1], colIdx[i:i+1], buf) + err4 := HashChunkRow(sc, rowHash[2], chk.GetRow(2), tps[i:i+1], colIdx[i:i+1], buf) c.Assert(err1, IsNil) c.Assert(err2, IsNil) c.Assert(err3, IsNil) @@ -1272,9 +1272,9 @@ func (s *testCodecSuite) TestHashChunkColumns(c *C) { c.Assert(chk.GetRow(0).IsNull(i), Equals, false) err1 := HashChunkSelected(sc, vecHash, chk, tps[i], i, buf, hasNull, sel, false) - err2 := HashChunkRow(sc, rowHash[0], chk.GetRow(0), tps, colIdx[i:i+1], buf) - err3 := HashChunkRow(sc, rowHash[1], chk.GetRow(1), tps, colIdx[i:i+1], buf) - err4 := HashChunkRow(sc, rowHash[2], chk.GetRow(2), tps, colIdx[i:i+1], buf) + err2 := HashChunkRow(sc, rowHash[0], chk.GetRow(0), tps[i:i+1], colIdx[i:i+1], buf) + err3 := HashChunkRow(sc, rowHash[1], chk.GetRow(1), tps[i:i+1], colIdx[i:i+1], buf) + err4 := HashChunkRow(sc, rowHash[2], chk.GetRow(2), tps[i:i+1], colIdx[i:i+1], buf) c.Assert(err1, IsNil) c.Assert(err2, IsNil) c.Assert(err3, IsNil) diff --git a/util/logutil/log_test.go b/util/logutil/log_test.go index d381e436874ea..bcc43ac26efb3 100644 --- a/util/logutil/log_test.go +++ b/util/logutil/log_test.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/log" "github.com/stretchr/testify/require" "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) func TestZapLoggerWithKeys(t *testing.T) { @@ -114,11 +115,14 @@ func TestGrpcLoggerCreation(t *testing.T) { } func TestSlowQueryLoggerCreation(t *testing.T) { - level := "warn" + level := "Error" conf := NewLogConfig(level, DefaultLogFormat, "", EmptyFileLogConfig, false) _, prop, err := newSlowQueryLogger(conf) // assert after init slow query logger, the original conf is not changed require.Equal(t, conf.Level, level) require.Nil(t, err) - require.Equal(t, prop.Level.String(), conf.Level) + // slow query logger doesn't use the level of the global log config, and the + // level should be less than WarnLevel which is used by it to log slow query. + require.NotEqual(t, conf.Level, prop.Level.String()) + require.True(t, prop.Level.Level() <= zapcore.WarnLevel) } diff --git a/util/logutil/slow_query_logger.go b/util/logutil/slow_query_logger.go index 433fd5746eca7..cf03dd34e6b80 100644 --- a/util/logutil/slow_query_logger.go +++ b/util/logutil/slow_query_logger.go @@ -15,14 +15,15 @@ var _pool = buffer.NewPool() func newSlowQueryLogger(cfg *LogConfig) (*zap.Logger, *log.ZapProperties, error) { - // copy global config and override slow query log file - // if slow query log filename is empty, slow query log will behave the same as global log + // copy the global log config to slow log config + // if the filename of slow log config is empty, slow log will behave the same as global log. sqConfig := cfg.Config + // level of the global log config doesn't affect the slow query logger which determines whether to + // log by execution duration. + sqConfig.Level = LogConfig{}.Level if len(cfg.SlowQueryFile) != 0 { - sqConfig.File = log.FileLogConfig{ - MaxSize: cfg.File.MaxSize, - Filename: cfg.SlowQueryFile, - } + sqConfig.File = cfg.File + sqConfig.File.Filename = cfg.SlowQueryFile } // create the slow query logger diff --git a/util/memory/tracker.go b/util/memory/tracker.go index 2525ee76c2e0a..52cd9f08e2de9 100644 --- a/util/memory/tracker.go +++ b/util/memory/tracker.go @@ -493,4 +493,8 @@ const ( LabelForSimpleTask int = -18 // LabelForCTEStorage represents the label of CTE storage LabelForCTEStorage int = -19 + // LabelForIndexJoinInnerWorker represents the label of IndexJoin InnerWorker + LabelForIndexJoinInnerWorker int = -20 + // LabelForIndexJoinOuterWorker represents the label of IndexJoin OuterWorker + LabelForIndexJoinOuterWorker int = -21 ) diff --git a/util/ranger/testdata/ranger_suite_out.json b/util/ranger/testdata/ranger_suite_out.json index fb40fffb6fe48..66b140bab9de7 100644 --- a/util/ranger/testdata/ranger_suite_out.json +++ b/util/ranger/testdata/ranger_suite_out.json @@ -265,8 +265,8 @@ { "SQL": "select * from t where a = 1 and ((b = 1) or (b = 2 and c = 3));", "Plan": [ - "TableReader_6 1.71 root data:TableRangeScan_5", - "└─TableRangeScan_5 1.71 cop[tikv] table:t range:[1 1,1 1], [1 2 3,1 2 3], keep order:false" + "TableReader_6 2.00 root data:TableRangeScan_5", + "└─TableRangeScan_5 2.00 cop[tikv] table:t range:[1 1,1 1], [1 2 3,1 2 3], keep order:false" ], "Result": [ "1 1 1" @@ -293,8 +293,8 @@ { "SQL": "select * from t use index(primary) where ((a = 1) or (a = 2 and b = 2)) and c = 3;", "Plan": [ - "TableReader_7 0.68 root data:Selection_6", - "└─Selection_6 0.68 cop[tikv] eq(test.t.c, 3), or(eq(test.t.a, 1), and(eq(test.t.a, 2), eq(test.t.b, 2)))", + "TableReader_7 0.75 root data:Selection_6", + "└─Selection_6 0.75 cop[tikv] eq(test.t.c, 3), or(eq(test.t.a, 1), and(eq(test.t.a, 2), eq(test.t.b, 2)))", " └─TableRangeScan_5 2.00 cop[tikv] table:t range:[1,1], [2,2], keep order:false" ], "Result": [ @@ -304,7 +304,7 @@ { "SQL": "select * from t where (a,b) in ((1,1),(2,2)) and c > 2 and (a,b,c) in ((1,1,1),(2,2,3));", "Plan": [ - "Selection_6 0.44 root gt(test.t.c, 2), or(and(eq(test.t.a, 1), eq(test.t.b, 1)), and(eq(test.t.a, 2), eq(test.t.b, 2)))", + "Selection_6 0.56 root gt(test.t.c, 2), or(and(eq(test.t.a, 1), eq(test.t.b, 1)), and(eq(test.t.a, 2), eq(test.t.b, 2)))", "└─Batch_Point_Get_5 2.00 root table:t, clustered index:PRIMARY(a, b, c) keep order:false, desc:false" ], "Result": [ @@ -314,8 +314,8 @@ { "SQL": "select * from t where (a,b) in ((1,1),(2,2)) and c > 2;", "Plan": [ - "TableReader_6 1.19 root data:TableRangeScan_5", - "└─TableRangeScan_5 1.19 cop[tikv] table:t range:(1 1 2,1 1 +inf], (2 2 2,2 2 +inf], keep order:false" + "TableReader_6 1.00 root data:TableRangeScan_5", + "└─TableRangeScan_5 1.00 cop[tikv] table:t range:(1 1 2,1 1 +inf], (2 2 2,2 2 +inf], keep order:false" ], "Result": [ "2 2 3" @@ -324,8 +324,8 @@ { "SQL": "select * from t where ((a = 1 and b = 1) or (a = 2 and b = 2)) and c > 2;", "Plan": [ - "TableReader_6 1.19 root data:TableRangeScan_5", - "└─TableRangeScan_5 1.19 cop[tikv] table:t range:(1 1 2,1 1 +inf], (2 2 2,2 2 +inf], keep order:false" + "TableReader_6 1.00 root data:TableRangeScan_5", + "└─TableRangeScan_5 1.00 cop[tikv] table:t range:(1 1 2,1 1 +inf], (2 2 2,2 2 +inf], keep order:false" ], "Result": [ "2 2 3" @@ -364,8 +364,8 @@ { "SQL": "select * from t2 where t='aaaa';", "Plan": [ - "TableReader_7 1.00 root data:Selection_6", - "└─Selection_6 1.00 cop[tikv] eq(test.t2.t, \"aaaa\")", + "TableReader_7 0.00 root data:Selection_6", + "└─Selection_6 0.00 cop[tikv] eq(test.t2.t, \"aaaa\")", " └─TableRangeScan_5 2.00 cop[tikv] table:t2 range:[0,+inf], keep order:false" ], "Result": [ @@ -375,8 +375,8 @@ { "SQL": "select * from t2 where t='aaaa' or t = 'a';", "Plan": [ - "TableReader_7 1.60 root data:Selection_6", - "└─Selection_6 1.60 cop[tikv] or(eq(test.t2.t, \"aaaa\"), eq(test.t2.t, \"a\"))", + "TableReader_7 0.80 root data:Selection_6", + "└─Selection_6 0.80 cop[tikv] or(eq(test.t2.t, \"aaaa\"), eq(test.t2.t, \"a\"))", " └─TableRangeScan_5 2.00 cop[tikv] table:t2 range:[0,+inf], keep order:false" ], "Result": [