Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

expression: open push down switcher for part of CAST functions #13837

Merged
merged 10 commits into from
Jan 2, 2020
Merged

expression: open push down switcher for part of CAST functions #13837

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7c4a876
expression: open CAST function push down switcher
lonng Dec 1, 2019
ed288b9
Merge branch 'master' of https://github.com/pingcap/tidb into push-do…
iosmanthus Dec 31, 2019
027734b
Merge remote-tracking branch 'origin/master' into push-down-cast
lonng Dec 31, 2019
377c449
fix execution plan
lonng Dec 31, 2019
a108d28
fix ci
lonng Dec 31, 2019
ad16433
fix ci
lonng Dec 31, 2019
bbf4e69
Forbid cast string as time
iosmanthus Dec 31, 2019
7ce54ec
Merge branch 'master' into push-down-cast
iosmanthus Dec 31, 2019
721782d
forbid json unquote push down
lonng Dec 31, 2019
b768ab9
Merge branch 'master' into push-down-cast
lonng Jan 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 8 additions & 10 deletions cmd/explaintest/r/explain_easy.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -498,19 +498,17 @@ EXPLAIN SELECT COUNT(1) FROM (SELECT COALESCE(b.region_name, '不详') region_na
id count task operator info
StreamAgg_22 1.00 root funcs:count(1)->Column#22
└─HashAgg_25 1.00 root group by:Column#31, Column#32, Column#33, funcs:firstrow(1)->Column#23
└─Projection_50 0.02 root Column#14, Column#15, coalesce(test.test02.region_name, 不详)->Column#33
└─Projection_48 0.02 root Column#14, Column#15, coalesce(test.test02.region_name, 不详)->Column#33
└─IndexMergeJoin_32 0.02 root left outer join, inner:TableReader_30, outer key:Column#16, inner key:test.test02.id
├─Union_37 0.02 root
│ ├─Projection_38 0.01 root test.test01.stat_date, test.test01.show_date, test.test01.region_id, cast(0)->Column#17
│ │ └─Selection_39 0.01 root gt(cast(0), 0)
│ │ └─TableReader_42 0.01 root data:Selection_41
│ │ └─Selection_41 0.01 cop[tikv] eq(test.test01.period, 1), ge(test.test01.stat_date, 20191202), ge(test.test01.stat_date, 20191202), le(test.test01.stat_date, 20191202), le(test.test01.stat_date, 20191202)
│ │ └─TableScan_40 10000.00 cop[tikv] table:test01, range:[-inf,+inf], keep order:false, stats:pseudo
│ └─Projection_43 0.01 root test.test01.stat_date, test.test01.show_date, test.test01.region_id, cast(test.test01.registration_num)->Column#17
│ └─Selection_44 0.01 root gt(cast(test.test01.registration_num), 0)
│ └─TableReader_47 0.01 root data:Selection_46
│ └─Selection_46 0.01 cop[tikv] eq(test.test01.period, 1), ge(test.test01.stat_date, 20191202), ge(test.test01.stat_date, 20191202), le(test.test01.stat_date, 20191202), le(test.test01.stat_date, 20191202)
│ └─TableScan_45 10000.00 cop[tikv] table:test01, range:[-inf,+inf], keep order:false, stats:pseudo
│ │ └─TableReader_41 0.01 root data:Selection_40
│ │ └─Selection_40 0.01 cop[tikv] eq(test.test01.period, 1), ge(test.test01.stat_date, 20191202), ge(test.test01.stat_date, 20191202), gt(cast(0), 0), le(test.test01.stat_date, 20191202), le(test.test01.stat_date, 20191202)
│ │ └─TableScan_39 10000.00 cop[tikv] table:test01, range:[-inf,+inf], keep order:false, stats:pseudo
│ └─Projection_42 0.01 root test.test01.stat_date, test.test01.show_date, test.test01.region_id, cast(test.test01.registration_num)->Column#17
│ └─TableReader_45 0.01 root data:Selection_44
│ └─Selection_44 0.01 cop[tikv] eq(test.test01.period, 1), ge(test.test01.stat_date, 20191202), ge(test.test01.stat_date, 20191202), gt(cast(test.test01.registration_num), 0), le(test.test01.stat_date, 20191202), le(test.test01.stat_date, 20191202)
│ └─TableScan_43 10000.00 cop[tikv] table:test01, range:[-inf,+inf], keep order:false, stats:pseudo
└─TableReader_30 1.00 root data:TableScan_29
└─TableScan_29 1.00 cop[tikv] table:b, range: decided by [Column#16], keep order:true, stats:pseudo
drop table if exists t;
Expand Down
13 changes: 12 additions & 1 deletion expression/expr_to_pb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,8 @@ func (pc PbConverter) canFuncBePushed(sf *ScalarFunction) bool {
// json functions.
ast.JSONType,
ast.JSONExtract,
ast.JSONUnquote,
// FIXME: JSONUnquote is incompatible with Coprocessor
// ast.JSONUnquote,
ast.JSONObject,
ast.JSONArray,
ast.JSONMerge,
Expand Down Expand Up @@ -464,6 +465,16 @@ func (pc PbConverter) canFuncBePushed(sf *ScalarFunction) bool {
tipb.ScalarFuncSig_RoundDec:
return isPushdownEnabled(sf.FuncName.L)
}
case ast.Cast:
switch sf.Function.PbCode() {
case tipb.ScalarFuncSig_CastStringAsInt,
tipb.ScalarFuncSig_CastStringAsReal,
tipb.ScalarFuncSig_CastStringAsTime,
tipb.ScalarFuncSig_CastTimeAsInt:
return false
default:
return isPushdownEnabled(sf.FuncName.L)
}
}
return false
}
Expand Down
189 changes: 97 additions & 92 deletions planner/cascades/testdata/transformation_rules_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,10 +319,15 @@
" Projection_3 input:[Group#1], test.t.b, Column#13",
"Group#1 Schema:[Column#13,test.t.b]",
" Aggregation_2 input:[Group#2], group by:plus(sin(cast(test.t.b)), sin(cast(test.t.c))), test.t.b, funcs:sum(test.t.a), firstrow(test.t.b)",
" Aggregation_7 input:[Group#3], group by:Column#16, test.t.b, funcs:sum(Column#14), firstrow(test.t.b)",
"Group#2 Schema:[test.t.a,test.t.b,test.t.c], UniqueKey:[test.t.a]",
" TiKVSingleGather_5 input:[Group#3], table:t",
"Group#3 Schema:[test.t.a,test.t.b,test.t.c], UniqueKey:[test.t.a]",
" TableScan_4 table:t, pk col:test.t.a"
" TiKVSingleGather_5 input:[Group#4], table:t",
"Group#4 Schema:[test.t.a,test.t.b,test.t.c], UniqueKey:[test.t.a]",
" TableScan_4 table:t, pk col:test.t.a",
"Group#3 Schema:[Column#14,Column#16,test.t.b]",
" TiKVSingleGather_5 input:[Group#5], table:t",
"Group#5 Schema:[Column#14,Column#16,test.t.b]",
" Aggregation_6 input:[Group#4], group by:plus(sin(cast(test.t.b)), sin(cast(test.t.c))), test.t.b, funcs:sum(test.t.a)"
]
}
]
Expand Down Expand Up @@ -350,13 +355,13 @@
{
"SQL": "select b from t limit 2",
"Result": [
"Group#0 Schema:[test.t.b]",
" Projection_2 input:[Group#1], test.t.b",
"Group#1 Schema:[test.t.b]",
" Limit_3 input:[Group#2], offset:0, count:2",
"Group#2 Schema:[test.t.b]",
" TiKVSingleGather_5 input:[Group#3], table:t",
"Group#3 Schema:[test.t.b]",
"Group#0 Schema:[test.t.b]",
" Projection_2 input:[Group#1], test.t.b",
"Group#1 Schema:[test.t.b]",
" Limit_3 input:[Group#2], offset:0, count:2",
"Group#2 Schema:[test.t.b]",
" TiKVSingleGather_5 input:[Group#3], table:t",
"Group#3 Schema:[test.t.b]",
" TableScan_4 table:t"
]
},
Expand Down Expand Up @@ -579,94 +584,94 @@
{
"SQL": "(select a from t) union all (select b from t) limit 2;",
"Result": [
"Group#0 Schema:[Column#25]",
" Limit_8 input:[Group#1], offset:0, count:2",
"Group#1 Schema:[Column#25]",
" Union_5 input:[Group#2,Group#3]",
"Group#2 Schema:[Column#25]",
" Projection_6 input:[Group#4], test.t.a",
"Group#4 Schema:[test.t.a]",
" Projection_4 input:[Group#5], test.t.a",
"Group#5 Schema:[test.t.a]",
" Limit_25 input:[Group#6], offset:0, count:2",
"Group#6 Schema:[test.t.a]",
" TiKVSingleGather_10 input:[Group#7], table:t",
" TiKVSingleGather_22 input:[Group#8], table:t, index:e_d_c_str_prefix",
" TiKVSingleGather_20 input:[Group#9], table:t, index:c_d_e_str",
" TiKVSingleGather_18 input:[Group#10], table:t, index:f_g",
" TiKVSingleGather_16 input:[Group#11], table:t, index:g",
" TiKVSingleGather_14 input:[Group#12], table:t, index:f",
" TiKVSingleGather_12 input:[Group#13], table:t, index:c_d_e",
"Group#7 Schema:[test.t.a]",
" TableScan_9 table:t, pk col:test.t.a",
"Group#8 Schema:[test.t.a]",
" IndexScan_21 table:t, index:e_str, d_str, c_str",
"Group#9 Schema:[test.t.a]",
" IndexScan_19 table:t, index:c_str, d_str, e_str",
"Group#10 Schema:[test.t.a]",
" IndexScan_17 table:t, index:f, g",
"Group#11 Schema:[test.t.a]",
" IndexScan_15 table:t, index:g",
"Group#12 Schema:[test.t.a]",
" IndexScan_13 table:t, index:f",
"Group#13 Schema:[test.t.a]",
" IndexScan_11 table:t, index:c, d, e",
"Group#3 Schema:[Column#25]",
" Projection_7 input:[Group#14], test.t.b",
"Group#14 Schema:[test.t.b]",
" Projection_2 input:[Group#15], test.t.b",
"Group#15 Schema:[test.t.b]",
" Limit_25 input:[Group#16], offset:0, count:2",
"Group#16 Schema:[test.t.b]",
" TiKVSingleGather_24 input:[Group#17], table:t",
"Group#17 Schema:[test.t.b]",
"Group#0 Schema:[Column#25]",
" Limit_8 input:[Group#1], offset:0, count:2",
"Group#1 Schema:[Column#25]",
" Union_5 input:[Group#2,Group#3]",
"Group#2 Schema:[Column#25]",
" Projection_6 input:[Group#4], test.t.a",
"Group#4 Schema:[test.t.a]",
" Projection_4 input:[Group#5], test.t.a",
"Group#5 Schema:[test.t.a]",
" Limit_25 input:[Group#6], offset:0, count:2",
"Group#6 Schema:[test.t.a]",
" TiKVSingleGather_10 input:[Group#7], table:t",
" TiKVSingleGather_22 input:[Group#8], table:t, index:e_d_c_str_prefix",
" TiKVSingleGather_20 input:[Group#9], table:t, index:c_d_e_str",
" TiKVSingleGather_18 input:[Group#10], table:t, index:f_g",
" TiKVSingleGather_16 input:[Group#11], table:t, index:g",
" TiKVSingleGather_14 input:[Group#12], table:t, index:f",
" TiKVSingleGather_12 input:[Group#13], table:t, index:c_d_e",
"Group#7 Schema:[test.t.a]",
" TableScan_9 table:t, pk col:test.t.a",
"Group#8 Schema:[test.t.a]",
" IndexScan_21 table:t, index:e_str, d_str, c_str",
"Group#9 Schema:[test.t.a]",
" IndexScan_19 table:t, index:c_str, d_str, e_str",
"Group#10 Schema:[test.t.a]",
" IndexScan_17 table:t, index:f, g",
"Group#11 Schema:[test.t.a]",
" IndexScan_15 table:t, index:g",
"Group#12 Schema:[test.t.a]",
" IndexScan_13 table:t, index:f",
"Group#13 Schema:[test.t.a]",
" IndexScan_11 table:t, index:c, d, e",
"Group#3 Schema:[Column#25]",
" Projection_7 input:[Group#14], test.t.b",
"Group#14 Schema:[test.t.b]",
" Projection_2 input:[Group#15], test.t.b",
"Group#15 Schema:[test.t.b]",
" Limit_25 input:[Group#16], offset:0, count:2",
"Group#16 Schema:[test.t.b]",
" TiKVSingleGather_24 input:[Group#17], table:t",
"Group#17 Schema:[test.t.b]",
" TableScan_23 table:t"
]
},
{
"SQL": "(select a from t) union all (select b from t) limit 2 offset 5;",
"Result": [
"Group#0 Schema:[Column#25]",
" Limit_8 input:[Group#1], offset:5, count:2",
"Group#1 Schema:[Column#25]",
" Union_5 input:[Group#2,Group#3]",
"Group#2 Schema:[Column#25]",
" Projection_6 input:[Group#4], test.t.a",
"Group#4 Schema:[test.t.a]",
" Projection_4 input:[Group#5], test.t.a",
"Group#5 Schema:[test.t.a]",
" Limit_25 input:[Group#6], offset:0, count:7",
"Group#6 Schema:[test.t.a]",
" TiKVSingleGather_10 input:[Group#7], table:t",
" TiKVSingleGather_22 input:[Group#8], table:t, index:e_d_c_str_prefix",
" TiKVSingleGather_20 input:[Group#9], table:t, index:c_d_e_str",
" TiKVSingleGather_18 input:[Group#10], table:t, index:f_g",
" TiKVSingleGather_16 input:[Group#11], table:t, index:g",
" TiKVSingleGather_14 input:[Group#12], table:t, index:f",
" TiKVSingleGather_12 input:[Group#13], table:t, index:c_d_e",
"Group#7 Schema:[test.t.a]",
" TableScan_9 table:t, pk col:test.t.a",
"Group#8 Schema:[test.t.a]",
" IndexScan_21 table:t, index:e_str, d_str, c_str",
"Group#9 Schema:[test.t.a]",
" IndexScan_19 table:t, index:c_str, d_str, e_str",
"Group#10 Schema:[test.t.a]",
" IndexScan_17 table:t, index:f, g",
"Group#11 Schema:[test.t.a]",
" IndexScan_15 table:t, index:g",
"Group#12 Schema:[test.t.a]",
" IndexScan_13 table:t, index:f",
"Group#13 Schema:[test.t.a]",
" IndexScan_11 table:t, index:c, d, e",
"Group#3 Schema:[Column#25]",
" Projection_7 input:[Group#14], test.t.b",
"Group#14 Schema:[test.t.b]",
" Projection_2 input:[Group#15], test.t.b",
"Group#15 Schema:[test.t.b]",
" Limit_25 input:[Group#16], offset:0, count:7",
"Group#16 Schema:[test.t.b]",
" TiKVSingleGather_24 input:[Group#17], table:t",
"Group#17 Schema:[test.t.b]",
"Group#0 Schema:[Column#25]",
" Limit_8 input:[Group#1], offset:5, count:2",
"Group#1 Schema:[Column#25]",
" Union_5 input:[Group#2,Group#3]",
"Group#2 Schema:[Column#25]",
" Projection_6 input:[Group#4], test.t.a",
"Group#4 Schema:[test.t.a]",
" Projection_4 input:[Group#5], test.t.a",
"Group#5 Schema:[test.t.a]",
" Limit_25 input:[Group#6], offset:0, count:7",
"Group#6 Schema:[test.t.a]",
" TiKVSingleGather_10 input:[Group#7], table:t",
" TiKVSingleGather_22 input:[Group#8], table:t, index:e_d_c_str_prefix",
" TiKVSingleGather_20 input:[Group#9], table:t, index:c_d_e_str",
" TiKVSingleGather_18 input:[Group#10], table:t, index:f_g",
" TiKVSingleGather_16 input:[Group#11], table:t, index:g",
" TiKVSingleGather_14 input:[Group#12], table:t, index:f",
" TiKVSingleGather_12 input:[Group#13], table:t, index:c_d_e",
"Group#7 Schema:[test.t.a]",
" TableScan_9 table:t, pk col:test.t.a",
"Group#8 Schema:[test.t.a]",
" IndexScan_21 table:t, index:e_str, d_str, c_str",
"Group#9 Schema:[test.t.a]",
" IndexScan_19 table:t, index:c_str, d_str, e_str",
"Group#10 Schema:[test.t.a]",
" IndexScan_17 table:t, index:f, g",
"Group#11 Schema:[test.t.a]",
" IndexScan_15 table:t, index:g",
"Group#12 Schema:[test.t.a]",
" IndexScan_13 table:t, index:f",
"Group#13 Schema:[test.t.a]",
" IndexScan_11 table:t, index:c, d, e",
"Group#3 Schema:[Column#25]",
" Projection_7 input:[Group#14], test.t.b",
"Group#14 Schema:[test.t.b]",
" Projection_2 input:[Group#15], test.t.b",
"Group#15 Schema:[test.t.b]",
" Limit_25 input:[Group#16], offset:0, count:7",
"Group#16 Schema:[test.t.b]",
" TiKVSingleGather_24 input:[Group#17], table:t",
"Group#17 Schema:[test.t.b]",
" TableScan_23 table:t"
]
},
Expand Down
12 changes: 6 additions & 6 deletions planner/core/testdata/integration_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,12 @@
{
"SQL": "explain select * from t t1 left join t t2 on t1.a = t2.a where cast(t1.b as date) >= '2019-01-01'",
"Plan": [
"HashLeftJoin_9 10000.00 root left outer join, inner:Selection_10 (REVERSED), equal:[eq(test.t.a, test.t.a)]",
"├─Selection_10 8000.00 root ge(cast(test.t.b), 2019-01-01 00:00:00.000000)",
"│ └─TableReader_12 10000.00 root data:TableScan_11",
"│ └─TableScan_11 10000.00 cop[tikv] table:t1, range:[-inf,+inf], keep order:false, stats:pseudo",
"└─TableReader_14 10000.00 root data:TableScan_13",
" └─TableScan_13 10000.00 cop[tikv] table:t2, range:[-inf,+inf], keep order:false, stats:pseudo"
"HashLeftJoin_8 10000.00 root left outer join, inner:TableReader_11 (REVERSED), equal:[eq(test.t.a, test.t.a)]",
"├─TableReader_11 8000.00 root data:Selection_10",
"│ └─Selection_10 8000.00 cop[tikv] ge(cast(test.t.b), 2019-01-01 00:00:00.000000)",
"│ └─TableScan_9 10000.00 cop[tikv] table:t1, range:[-inf,+inf], keep order:false, stats:pseudo",
"└─TableReader_13 10000.00 root data:TableScan_12",
" └─TableScan_12 10000.00 cop[tikv] table:t2, range:[-inf,+inf], keep order:false, stats:pseudo"
]
}
]
Expand Down
2 changes: 1 addition & 1 deletion planner/core/testdata/plan_suite_unexported_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
"DataScan(t)->Projection->Projection",
"DataScan(t)->Projection->Projection",
"Join{DataScan(ta)->DataScan(tb)}(test.t.d,test.t.b)(test.t.a,test.t.c)->Projection",
"Join{DataScan(t1)->Sel([like(cast(test.t.d), abc, 92)])->DataScan(t2)->Sel([like(cast(test.t.d), abc, 92)])}(test.t.a,test.t.b)(test.t.d,test.t.d)->Projection",
"Join{DataScan(t1)->DataScan(t2)}(test.t.a,test.t.b)(test.t.d,test.t.d)->Projection",
"Join{DataScan(ta)->DataScan(tb)}(test.t.d,test.t.d)->Projection",
"Join{DataScan(ta)->DataScan(tb)}(test.t.d,test.t.d)->Projection",
"Join{DataScan(ta)->DataScan(tb)}(test.t.d,test.t.d)->Projection",
Expand Down
11 changes: 10 additions & 1 deletion util/codec/codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -984,10 +984,19 @@ func (decoder *Decoder) DecodeOne(b []byte, colIdx int, ft *types.FieldType) (re
chk.AppendBytes(colIdx, v)
case decimalFlag:
var dec *types.MyDecimal
b, dec, _, _, err = DecodeDecimal(b)
var frac int
b, dec, _, frac, err = DecodeDecimal(b)
if err != nil {
return nil, errors.Trace(err)
}
if ft.Decimal != types.UnspecifiedLength && frac > ft.Decimal {
qw4990 marked this conversation as resolved.
Show resolved Hide resolved
to := new(types.MyDecimal)
err := dec.Round(to, ft.Decimal, types.ModeHalfEven)
if err != nil {
return nil, errors.Trace(err)
}
dec = to
}
chk.AppendMyDecimal(colIdx, dec)
case durationFlag:
var r int64
Expand Down