From d9c751368f2facf975b544af8e0faa8e18397803 Mon Sep 17 00:00:00 2001 From: yuanwhy Date: Sat, 29 Jul 2017 17:12:50 +0800 Subject: [PATCH 01/11] support '\N' for null in select sql --- parser/lexer.go | 12 ++++++++++++ server/driver_tidb.go | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/parser/lexer.go b/parser/lexer.go index aeb317aaffa99..7113de482d003 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -204,6 +204,14 @@ func (s *Scanner) scan() (tok int, pos Pos, lit string) { ch0 = s.skipWhitespace() } pos = s.r.pos() + + if ch0 == '\\' { + if s.r.dataByIndex(pos.Offset+1) == 'N' { + s.r.inc() + return null, pos, "\\N" + } + } + if s.r.eof() { // when scanner meets EOF, the returned token should be 0, // because 0 is a special token id to remind the parser that stream is end. @@ -740,6 +748,10 @@ func (r *reader) data(from *Pos) string { return r.s[from.Offset:r.p.Offset] } +func (r *reader) dataByIndex(index int) rune { + return rune(r.s[index]) +} + func (r *reader) incAsLongAs(fn func(rune) bool) rune { for { ch := r.peek() diff --git a/server/driver_tidb.go b/server/driver_tidb.go index d7f851639f8b0..7dd9a73255693 100644 --- a/server/driver_tidb.go +++ b/server/driver_tidb.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/tidb" "github.com/pingcap/tidb/ast" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/model" "github.com/pingcap/tidb/mysql" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/types" @@ -306,6 +307,10 @@ func (trs *tidbResultSet) Columns() ([]*ColumnInfo, error) { } var columns []*ColumnInfo for _, v := range fields { + if v.ColumnAsName.O == "\\N" { + v.ColumnAsName = model.NewCIStr("NULL") + } + columns = append(columns, convertColumnInfo(v)) } return columns, nil From ee9a3442287ed024bed74b6a3a700bcea0f5f2f1 Mon Sep 17 00:00:00 2001 From: yuanwhy Date: Sat, 29 Jul 2017 17:16:36 +0800 Subject: [PATCH 02/11] skip N when scanning --- parser/lexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/lexer.go b/parser/lexer.go index 7113de482d003..8599a31692e7e 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -207,7 +207,7 @@ func (s *Scanner) scan() (tok int, pos Pos, lit string) { if ch0 == '\\' { if s.r.dataByIndex(pos.Offset+1) == 'N' { - s.r.inc() + s.r.incN(2) return null, pos, "\\N" } } From 25bb705c586e0f61e30cf3ad79f4cc6da0f86c33 Mon Sep 17 00:00:00 2001 From: yuanwhy Date: Sat, 29 Jul 2017 17:41:00 +0800 Subject: [PATCH 03/11] rename to charByIndex --- parser/lexer.go | 4 ++-- parser/lexer_test.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/parser/lexer.go b/parser/lexer.go index 8599a31692e7e..c197c20d6f58c 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -206,7 +206,7 @@ func (s *Scanner) scan() (tok int, pos Pos, lit string) { pos = s.r.pos() if ch0 == '\\' { - if s.r.dataByIndex(pos.Offset+1) == 'N' { + if s.r.charByIndex(pos.Offset+1) == 'N' { s.r.incN(2) return null, pos, "\\N" } @@ -748,7 +748,7 @@ func (r *reader) data(from *Pos) string { return r.s[from.Offset:r.p.Offset] } -func (r *reader) dataByIndex(index int) rune { +func (r *reader) charByIndex(index int) rune { return rune(r.s[index]) } diff --git a/parser/lexer_test.go b/parser/lexer_test.go index 72abf4560b3c2..dfa69ac2bae38 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -121,6 +121,7 @@ func (s *testLexerSuite) TestLiteral(c *C) { {".1_t_1_x", int('.')}, {"N'some text'", underscoreCS}, {"n'some text'", underscoreCS}, + {"\\N", null}, } runTest(c, table) } From f5f5158c085709f7e2c9c52dee7d3e038182214b Mon Sep 17 00:00:00 2001 From: yuanwhy Date: Sat, 29 Jul 2017 17:42:48 +0800 Subject: [PATCH 04/11] add comment --- parser/lexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/parser/lexer.go b/parser/lexer.go index c197c20d6f58c..f6a56864f7b1d 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -206,6 +206,7 @@ func (s *Scanner) scan() (tok int, pos Pos, lit string) { pos = s.r.pos() if ch0 == '\\' { + // support \N as shortcut for NULL if s.r.charByIndex(pos.Offset+1) == 'N' { s.r.incN(2) return null, pos, "\\N" From 23bf1d63aacb8f21a6c7ee6c3c444e050f4e1f04 Mon Sep 17 00:00:00 2001 From: yuanwhy Date: Sun, 30 Jul 2017 17:50:29 +0800 Subject: [PATCH 05/11] use initTokenString directly --- parser/lexer.go | 12 ------------ parser/misc.go | 1 + 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/parser/lexer.go b/parser/lexer.go index f6a56864f7b1d..94b5da28e40d2 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -205,14 +205,6 @@ func (s *Scanner) scan() (tok int, pos Pos, lit string) { } pos = s.r.pos() - if ch0 == '\\' { - // support \N as shortcut for NULL - if s.r.charByIndex(pos.Offset+1) == 'N' { - s.r.incN(2) - return null, pos, "\\N" - } - } - if s.r.eof() { // when scanner meets EOF, the returned token should be 0, // because 0 is a special token id to remind the parser that stream is end. @@ -749,10 +741,6 @@ func (r *reader) data(from *Pos) string { return r.s[from.Offset:r.p.Offset] } -func (r *reader) charByIndex(index int) rune { - return rune(r.s[index]) -} - func (r *reader) incAsLongAs(fn func(rune) bool) rune { for { ch := r.peek() diff --git a/parser/misc.go b/parser/misc.go index e341f9eb08aec..3502624b82d9b 100644 --- a/parser/misc.go +++ b/parser/misc.go @@ -119,6 +119,7 @@ func init() { initTokenString("<>", neqSynonym) initTokenString("<<", lsh) initTokenString(">>", rsh) + initTokenString("\\N", null) initTokenFunc("@", startWithAt) initTokenFunc("/", startWithSlash) From 1ad89737b258e28b3ae042198a9508f61c4b165d Mon Sep 17 00:00:00 2001 From: yuanwhy Date: Sun, 30 Jul 2017 17:53:25 +0800 Subject: [PATCH 06/11] delete blank line --- parser/lexer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/parser/lexer.go b/parser/lexer.go index 94b5da28e40d2..aeb317aaffa99 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -204,7 +204,6 @@ func (s *Scanner) scan() (tok int, pos Pos, lit string) { ch0 = s.skipWhitespace() } pos = s.r.pos() - if s.r.eof() { // when scanner meets EOF, the returned token should be 0, // because 0 is a special token id to remind the parser that stream is end. From da665a765fedd53fb4eb4430e41ccc54ba6f7381 Mon Sep 17 00:00:00 2001 From: yuanwhy Date: Sun, 30 Jul 2017 19:29:23 +0800 Subject: [PATCH 07/11] rename cloumn \N to NULL in buildProjection func --- plan/logical_plan_builder.go | 5 +++++ server/driver_tidb.go | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plan/logical_plan_builder.go b/plan/logical_plan_builder.go index 7ea3528e2b2d7..c6f9252502b96 100644 --- a/plan/logical_plan_builder.go +++ b/plan/logical_plan_builder.go @@ -431,6 +431,11 @@ func (b *planBuilder) buildProjection(p LogicalPlan, fields []*ast.SelectField, } } } + + if colName.O == "\\N" { + colName = model.NewCIStr("NULL") + } + col := &expression.Column{ FromID: proj.id, TblName: tblName, diff --git a/server/driver_tidb.go b/server/driver_tidb.go index 7dd9a73255693..d7f851639f8b0 100644 --- a/server/driver_tidb.go +++ b/server/driver_tidb.go @@ -20,7 +20,6 @@ import ( "github.com/pingcap/tidb" "github.com/pingcap/tidb/ast" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/model" "github.com/pingcap/tidb/mysql" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/types" @@ -307,10 +306,6 @@ func (trs *tidbResultSet) Columns() ([]*ColumnInfo, error) { } var columns []*ColumnInfo for _, v := range fields { - if v.ColumnAsName.O == "\\N" { - v.ColumnAsName = model.NewCIStr("NULL") - } - columns = append(columns, convertColumnInfo(v)) } return columns, nil From f3c0500ad5c8408828787b6aee3d15ba303fa067 Mon Sep 17 00:00:00 2001 From: yuanwhy Date: Mon, 31 Jul 2017 02:14:05 +0800 Subject: [PATCH 08/11] add select \N test for value and column name --- executor/executor_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/executor/executor_test.go b/executor/executor_test.go index 9484d3d83b629..d8516a3da6675 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -217,6 +217,22 @@ func (s *testSuite) TestSelectWithoutFrom(c *C) { r.Check(testkit.Rows("string")) } +func (s *testSuite) TestSelectBackslashN(c *C) { + defer testleak.AfterTest(c)() + tk := testkit.NewTestKit(c, s.store) + sql := `select \N;` + + r := tk.MustQuery(sql) + r.Check(testkit.Rows("")) + + rs, err := tk.Exec(sql) + c.Check(err, IsNil) + fields, err := rs.Fields() + c.Check(err, IsNil) + c.Check(len(fields), Equals, 1) + c.Check(fields[0].Column.Name.O, Equals, "NULL") +} + func (s *testSuite) TestSelectLimit(c *C) { tk := testkit.NewTestKit(c, s.store) defer func() { From 7f8cf9a4fade6af27b669b302111ece6a063cd26 Mon Sep 17 00:00:00 2001 From: yuanwhy Date: Mon, 31 Jul 2017 12:33:42 +0800 Subject: [PATCH 09/11] mark NULL column name in Lex --- parser/lexer.go | 1 + plan/logical_plan_builder.go | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/parser/lexer.go b/parser/lexer.go index a9eb9aec49638..909b7afa7d682 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -162,6 +162,7 @@ func (s *Scanner) Lex(v *yySymType) int { return tok case null: v.item = nil + v.ident = "NULL" case quotedIdentifier: tok = identifier } diff --git a/plan/logical_plan_builder.go b/plan/logical_plan_builder.go index c6f9252502b96..7ea3528e2b2d7 100644 --- a/plan/logical_plan_builder.go +++ b/plan/logical_plan_builder.go @@ -431,11 +431,6 @@ func (b *planBuilder) buildProjection(p LogicalPlan, fields []*ast.SelectField, } } } - - if colName.O == "\\N" { - colName = model.NewCIStr("NULL") - } - col := &expression.Column{ FromID: proj.id, TblName: tblName, From 600fa9c666184992e7b9d38c91237b6fccd76261 Mon Sep 17 00:00:00 2001 From: yuanwhy Date: Mon, 31 Jul 2017 23:59:17 +0800 Subject: [PATCH 10/11] add some test cases and fix select * --- executor/executor_test.go | 42 +++++++++++++++++++++++++++++++++--- parser/lexer.go | 1 - plan/logical_plan_builder.go | 8 ++++++- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index d53abf3b749e5..366d0bea603ae 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -218,19 +218,55 @@ func (s *testSuite) TestSelectWithoutFrom(c *C) { } func (s *testSuite) TestSelectBackslashN(c *C) { - defer testleak.AfterTest(c)() + defer func() { + s.cleanEnv(c) + testleak.AfterTest(c)() + }() tk := testkit.NewTestKit(c, s.store) - sql := `select \N;` + sql := `select \N;` r := tk.MustQuery(sql) r.Check(testkit.Rows("")) - rs, err := tk.Exec(sql) c.Check(err, IsNil) fields, err := rs.Fields() c.Check(err, IsNil) c.Check(len(fields), Equals, 1) c.Check(fields[0].Column.Name.O, Equals, "NULL") + + sql = `select "\N";` + r = tk.MustQuery(sql) + r.Check(testkit.Rows("N")) + rs, err = tk.Exec(sql) + c.Check(err, IsNil) + fields, err = rs.Fields() + c.Check(err, IsNil) + c.Check(len(fields), Equals, 1) + c.Check(fields[0].Column.Name.O, Equals, `"\N"`) + + tk.MustExec("use test;") + tk.MustExec("create table test (`\\N` int);") + tk.MustExec("insert into test values (1);") + tk.CheckExecResult(1, 0) + sql = "select * from test;" + r = tk.MustQuery(sql) + r.Check(testkit.Rows("1")) + rs, err = tk.Exec(sql) + c.Check(err, IsNil) + fields, err = rs.Fields() + c.Check(err, IsNil) + c.Check(len(fields), Equals, 1) + c.Check(fields[0].Column.Name.O, Equals, `\N`) + + sql = "select \\N from test;" + r = tk.MustQuery(sql) + r.Check(testkit.Rows("")) + rs, err = tk.Exec(sql) + c.Check(err, IsNil) + fields, err = rs.Fields() + c.Check(err, IsNil) + c.Check(len(fields), Equals, 1) + c.Check(fields[0].Column.Name.O, Equals, `NULL`) } func (s *testSuite) TestSelectLimit(c *C) { diff --git a/parser/lexer.go b/parser/lexer.go index 909b7afa7d682..a9eb9aec49638 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -162,7 +162,6 @@ func (s *Scanner) Lex(v *yySymType) int { return tok case null: v.item = nil - v.ident = "NULL" case quotedIdentifier: tok = identifier } diff --git a/plan/logical_plan_builder.go b/plan/logical_plan_builder.go index 7ea3528e2b2d7..4aa68f5562019 100644 --- a/plan/logical_plan_builder.go +++ b/plan/logical_plan_builder.go @@ -426,8 +426,14 @@ func (b *planBuilder) buildProjection(p LogicalPlan, fields []*ast.SelectField, if _, ok := innerExpr.(*ast.ValueExpr); ok && innerExpr.Text() != "" { colName = model.NewCIStr(innerExpr.Text()) } else { + //Change column name \N to NUll, just when original sql contains \N column + fieldText := field.Text() + if fieldText == "\\N" { + fieldText = "NULL" + } + // Remove special comment code for field part, see issue #3739 for detail. - colName = model.NewCIStr(parser.SpecFieldPattern.ReplaceAllStringFunc(field.Text(), parser.TrimComment)) + colName = model.NewCIStr(parser.SpecFieldPattern.ReplaceAllStringFunc(fieldText, parser.TrimComment)) } } } From 85fd9a5f4a726230a0902efaebd79a9ae2b4025e Mon Sep 17 00:00:00 2001 From: yuanwhy Date: Tue, 1 Aug 2017 12:36:27 +0800 Subject: [PATCH 11/11] add more test case --- executor/executor_test.go | 10 ++++++++++ plan/logical_plan_builder.go | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 366d0bea603ae..4856cb8aa5cc7 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -267,6 +267,16 @@ func (s *testSuite) TestSelectBackslashN(c *C) { c.Check(err, IsNil) c.Check(len(fields), Equals, 1) c.Check(fields[0].Column.Name.O, Equals, `NULL`) + + sql = "select `\\N` from test;" + r = tk.MustQuery(sql) + r.Check(testkit.Rows("1")) + rs, err = tk.Exec(sql) + c.Check(err, IsNil) + fields, err = rs.Fields() + c.Check(err, IsNil) + c.Check(len(fields), Equals, 1) + c.Check(fields[0].Column.Name.O, Equals, `\N`) } func (s *testSuite) TestSelectLimit(c *C) { diff --git a/plan/logical_plan_builder.go b/plan/logical_plan_builder.go index 4aa68f5562019..a9f01aec239c3 100644 --- a/plan/logical_plan_builder.go +++ b/plan/logical_plan_builder.go @@ -426,7 +426,7 @@ func (b *planBuilder) buildProjection(p LogicalPlan, fields []*ast.SelectField, if _, ok := innerExpr.(*ast.ValueExpr); ok && innerExpr.Text() != "" { colName = model.NewCIStr(innerExpr.Text()) } else { - //Change column name \N to NUll, just when original sql contains \N column + //Change column name \N to NULL, just when original sql contains \N column fieldText := field.Text() if fieldText == "\\N" { fieldText = "NULL"