From 133530186441e04a2718e72f0b08d2e7d803d4cf Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Tue, 21 Dec 2021 14:33:53 +0800 Subject: [PATCH 1/2] expression, cmd: let crc32() support gbk --- .../r/new_character_set_builtin.result | 11 +++++ .../t/new_character_set_builtin.test | 9 ++++ expression/builtin_convert_charset.go | 2 +- expression/builtin_math_test.go | 43 +++++++++++-------- 4 files changed, 47 insertions(+), 18 deletions(-) diff --git a/cmd/explaintest/r/new_character_set_builtin.result b/cmd/explaintest/r/new_character_set_builtin.result index f587a5ac1370e..74673a02b0832 100644 --- a/cmd/explaintest/r/new_character_set_builtin.result +++ b/cmd/explaintest/r/new_character_set_builtin.result @@ -523,3 +523,14 @@ select hex(aes_encrypt(a, '123')), hex(aes_encrypt(b, '123')), hex(aes_encrypt(c hex(aes_encrypt(a, '123')) hex(aes_encrypt(b, '123')) hex(aes_encrypt(c, '123')) C54279F381B0710E145E94106F03C94C 7A747EC6F1906276D036B1F3CE27BAAB A0E5E01289017B8A3691CCFBDE81A59ED4A9D5BF50A298D41287E395CDDCAD56 set @@tidb_enable_vectorized_expression = false; +drop table if exists t; +create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20)); +insert into t values ('一二三', '一二三', '一二三'); +select crc32(a), crc32(b), crc32(c) from t; +crc32(a) crc32(b) crc32(c) +1785250883 3461331449 4092198678 +set @@tidb_enable_vectorized_expression = true; +select crc32(a), crc32(b), crc32(c) from t; +crc32(a) crc32(b) crc32(c) +1785250883 3461331449 4092198678 +set @@tidb_enable_vectorized_expression = false; diff --git a/cmd/explaintest/t/new_character_set_builtin.test b/cmd/explaintest/t/new_character_set_builtin.test index bb0a6321e8a53..ae9ab76f9093e 100644 --- a/cmd/explaintest/t/new_character_set_builtin.test +++ b/cmd/explaintest/t/new_character_set_builtin.test @@ -239,3 +239,12 @@ select hex(aes_encrypt(a, '123', '1234567890123456')), hex(aes_encrypt(b, '123', set @@block_encryption_mode='aes-128-ecb'; select hex(aes_encrypt(a, '123')), hex(aes_encrypt(b, '123')), hex(aes_encrypt(c, '123')) from t; set @@tidb_enable_vectorized_expression = false; + +-- test for builtin crc32() +drop table if exists t; +create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20)); +insert into t values ('一二三', '一二三', '一二三'); +select crc32(a), crc32(b), crc32(c) from t; +set @@tidb_enable_vectorized_expression = true; +select crc32(a), crc32(b), crc32(c) from t; +set @@tidb_enable_vectorized_expression = false; diff --git a/expression/builtin_convert_charset.go b/expression/builtin_convert_charset.go index 5c0ea98f752a5..b7b6cfffa60e3 100644 --- a/expression/builtin_convert_charset.go +++ b/expression/builtin_convert_charset.go @@ -279,7 +279,7 @@ var convertActionMap = map[funcProp][]string{ funcPropAuto: { /* string functions */ ast.Concat, ast.ConcatWS, ast.ExportSet, ast.Field, ast.FindInSet, ast.InsertFunc, ast.Instr, ast.Lpad, ast.Locate, ast.Lpad, ast.MakeSet, ast.Position, - ast.Replace, ast.Rpad, ast.SubstringIndex, ast.Trim, + ast.Replace, ast.Rpad, ast.SubstringIndex, ast.Trim, ast.CRC32, /* operators */ ast.GE, ast.LE, ast.GT, ast.LT, ast.EQ, ast.NE, ast.NullEQ, ast.If, ast.Ifnull, ast.In, ast.Case, ast.Cast, diff --git a/expression/builtin_math_test.go b/expression/builtin_math_test.go index 898693d14b394..d1ae3ca6ed30c 100644 --- a/expression/builtin_math_test.go +++ b/expression/builtin_math_test.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/charset" "github.com/pingcap/tidb/parser/mysql" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/testkit/trequire" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" @@ -531,27 +532,35 @@ func TestTruncate(t *testing.T) { func TestCRC32(t *testing.T) { ctx := createContext(t) tbl := []struct { - Arg []interface{} - Ret interface{} + input []interface{} + chs string + result int64 + isNull bool }{ - {[]interface{}{nil}, nil}, - {[]interface{}{""}, 0}, - {[]interface{}{-1}, 808273962}, - {[]interface{}{"-1"}, 808273962}, - {[]interface{}{"mysql"}, 2501908538}, - {[]interface{}{"MySQL"}, 3259397556}, - {[]interface{}{"hello"}, 907060870}, + {[]interface{}{nil}, "utf8", 0, true}, + {[]interface{}{""}, "utf8", 0, false}, + {[]interface{}{-1}, "utf8", 808273962, false}, + {[]interface{}{"-1"}, "utf8", 808273962, false}, + {[]interface{}{"mysql"}, "utf8", 2501908538, false}, + {[]interface{}{"MySQL"}, "utf8", 3259397556, false}, + {[]interface{}{"hello"}, "utf8", 907060870, false}, + {[]interface{}{"一二三"}, "utf8", 1785250883, false}, + {[]interface{}{"一"}, "utf8", 2416838398, false}, + {[]interface{}{"一二三"}, "gbk", 3461331449, false}, + {[]interface{}{"一"}, "gbk", 2925846374, false}, } - - Dtbl := tblToDtbl(tbl) - - for _, tt := range Dtbl { - fc := funcs[ast.CRC32] - f, err := fc.getFunction(ctx, datumsToConstants(tt["Arg"])) + for _, c := range tbl { + err := ctx.GetSessionVars().SetSystemVar(variable.CharacterSetConnection, c.chs) require.NoError(t, err) - v, err := evalBuiltinFunc(f, chunk.Row{}) + f, err := newFunctionForTest(ctx, ast.CRC32, primitiveValsToConstants(ctx, c.input)...) require.NoError(t, err) - trequire.DatumEqual(t, tt["Ret"][0], v) + d, err := f.Eval(chunk.Row{}) + require.NoError(t, err) + if c.isNull { + require.True(t, d.IsNull()) + } else { + require.Equal(t, c.result, d.GetInt64()) + } } } From f36956be5701cb13a9875347fa38062ee008ace7 Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Tue, 21 Dec 2021 14:42:08 +0800 Subject: [PATCH 2/2] move ast.CRC32 to math --- expression/builtin_convert_charset.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/expression/builtin_convert_charset.go b/expression/builtin_convert_charset.go index b7b6cfffa60e3..bc7ad4dfafbbe 100644 --- a/expression/builtin_convert_charset.go +++ b/expression/builtin_convert_charset.go @@ -279,7 +279,7 @@ var convertActionMap = map[funcProp][]string{ funcPropAuto: { /* string functions */ ast.Concat, ast.ConcatWS, ast.ExportSet, ast.Field, ast.FindInSet, ast.InsertFunc, ast.Instr, ast.Lpad, ast.Locate, ast.Lpad, ast.MakeSet, ast.Position, - ast.Replace, ast.Rpad, ast.SubstringIndex, ast.Trim, ast.CRC32, + ast.Replace, ast.Rpad, ast.SubstringIndex, ast.Trim, /* operators */ ast.GE, ast.LE, ast.GT, ast.LT, ast.EQ, ast.NE, ast.NullEQ, ast.If, ast.Ifnull, ast.In, ast.Case, ast.Cast, @@ -287,6 +287,8 @@ var convertActionMap = map[funcProp][]string{ ast.Like, ast.Strcmp, /* regex */ ast.Regexp, + /* math */ + ast.CRC32, }, }