support dumping binary data in hexadecimal representation (pingcap#19)

* bugfix: make RowIter.HasNext idemponent

* add a test for rowIter

* support dumping binary data in hexadecimal representation

* address comments

* add sql_type.go

* fix bugs and typo

v4/export/ir.go

@@ -19,6 +19,15 @@ type SQLRowIter interface {
 	HasNext() bool
 }
 
+type RowReceiverStringer interface {
+	RowReceiver
+	Stringer
+}
+
+type Stringer interface {
+	ToString() string
+}
+
 type RowReceiver interface {
 	BindAddress([]interface{})
 	ReportSize() uint64

v4/export/ir_impl_test.go

@@ -9,6 +9,18 @@ var _ = Suite(&testIRImplSuite{})
 
 type testIRImplSuite struct{}
 
+type simpleRowReceiver struct {
+	data string
+}
+
+func (s *simpleRowReceiver) BindAddress(arg []interface{}) {
+	arg[0] = &s.data
+}
+
+func (s *simpleRowReceiver) ReportSize() uint64 {
+	panic("not implement")
+}
+
 func (s *testIRImplSuite) TestRowIter(c *C) {
 	db, mock, err := sqlmock.New()
 	c.Assert(err, IsNil)
@@ -26,15 +38,15 @@ func (s *testIRImplSuite) TestRowIter(c *C) {
 	for i := 0; i < 100; i += 1 {
 		c.Assert(iter.HasNext(), IsTrue)
 	}
-	res := make(dumplingRow, 1)
+	res := &simpleRowReceiver{}
 	c.Assert(iter.Next(res), IsNil)
-	c.Assert(res[0].String, Equals, "1")
+	c.Assert(res.data, Equals, "1")
 	c.Assert(iter.HasNext(), IsTrue)
 	c.Assert(iter.HasNext(), IsTrue)
 	c.Assert(iter.Next(res), IsNil)
-	c.Assert(res[0].String, Equals, "2")
+	c.Assert(res.data, Equals, "2")
 	c.Assert(iter.HasNext(), IsTrue)
 	c.Assert(iter.Next(res), IsNil)
-	c.Assert(res[0].String, Equals, "3")
+	c.Assert(res.data, Equals, "3")
 	c.Assert(iter.HasNext(), IsFalse)
 }

v4/export/sql_test.go

@@ -57,7 +57,7 @@ func (s *testDumpSuite) TestBuildSelectAllQuery(c *C) {
 	mockConf := DefaultConfig()
 	mockConf.SortByPk = true
 
-	// Test when the server is TiDB.
+	// Test TiDB server.
 	mockConf.ServerInfo.ServerType = ServerTypeTiDB
 
 	// _tidb_rowid is available.
@@ -73,7 +73,7 @@ func (s *testDumpSuite) TestBuildSelectAllQuery(c *C) {
 	c.Assert(q, Equals, "SELECT * FROM test.t")
 	c.Assert(mock.ExpectationsWereMet(), IsNil)
 
-	// Test other server.
+	// Test other servers.
 	otherServers := []ServerType{ServerTypeUnknown, ServerTypeMySQL, ServerTypeMariaDB}
 
 	// Test table with primary key.

v4/export/sql_type.go

@@ -0,0 +1,147 @@
+package export
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+)
+
+var colTypeRowReceiverMap = map[string]func() RowReceiverStringer{}
+
+func init() {
+	for _, s := range dataTypeString {
+		colTypeRowReceiverMap[s] = SQLTypeStringMaker
+	}
+	for _, s := range dataTypeNum {
+		colTypeRowReceiverMap[s] = SQLTypeNumberMaker
+	}
+	for _, s := range dataTypeBin {
+		colTypeRowReceiverMap[s] = SQLTypeBytesMaker
+	}
+}
+
+var dataTypeString = []string{
+	"CHAR", "NCHAR", "VARCHAR", "NVARCHAR", "CHARACTER", "VARCHARACTER",
+	"TIMESTAMP", "DATETIME", "DATE", "TIME", "YEAR", "SQL_TSI_YEAR",
+	"TEXT", "TINYTEXT", "MEDIUMTEXT", "LONGTEXT",
+	"ENUM", "SET", "JSON",
+}
+
+var dataTypeNum = []string{
+	"INTEGER", "BIGINT", "TINYINT", "SMALLINT", "MEDIUMINT",
+	"INT", "INT1", "INT2", "INT3", "INT8",
+	"FLOAT", "REAL", "DOUBLE", "DOUBLE PRECISION",
+	"DECIMAL", "NUMERIC", "FIXED",
+	"BOOL", "BOOLEAN",
+}
+
+var dataTypeBin = []string{
+	"BLOB", "TINYBLOB", "MEDIUMBLOB", "LONGBLOB", "LONG",
+	"BINARY", "VARBINARY",
+	"BIT",
+}
+
+func SQLTypeStringMaker() RowReceiverStringer {
+	return &SQLTypeString{}
+}
+
+func SQLTypeBytesMaker() RowReceiverStringer {
+	return &SQLTypeBytes{}
+}
+
+func SQLTypeNumberMaker() RowReceiverStringer {
+	return &SQLTypeNumber{}
+}
+
+func MakeRowReceiver(colTypes []string) RowReceiverStringer {
+	rowReceiverArr := make(RowReceiverArr, len(colTypes))
+	for i, colTp := range colTypes {
+		recMaker, ok := colTypeRowReceiverMap[colTp]
+		if !ok {
+			recMaker = SQLTypeStringMaker
+		}
+		rowReceiverArr[i] = recMaker()
+	}
+	return rowReceiverArr
+}
+
+type RowReceiverArr []RowReceiverStringer
+
+func (r RowReceiverArr) BindAddress(args []interface{}) {
+	for i := range args {
+		var singleAddr [1]interface{}
+		r[i].BindAddress(singleAddr[:])
+		args[i] = singleAddr[0]
+	}
+}
+func (r RowReceiverArr) ReportSize() uint64 {
+	var sum uint64
+	for _, receiver := range r {
+		sum += receiver.ReportSize()
+	}
+	return sum
+}
+func (r RowReceiverArr) ToString() string {
+	var sb strings.Builder
+	sb.WriteString("(")
+	for i, receiver := range r {
+		sb.WriteString(receiver.ToString())
+		if i != len(r)-1 {
+			sb.WriteString(", ")
+		}
+	}
+	sb.WriteString(")")
+	return sb.String()
+}
+
+type SQLTypeNumber struct {
+	SQLTypeString
+}
+
+func (s SQLTypeNumber) ToString() string {
+	if s.Valid {
+		return s.String
+	} else {
+		return "NULL"
+	}
+}
+
+type SQLTypeString struct {
+	sql.NullString
+}
+
+func (s *SQLTypeString) BindAddress(arg []interface{}) {
+	arg[0] = s
+}
+func (s *SQLTypeString) ReportSize() uint64 {
+	if s.Valid {
+		return uint64(len(s.String))
+	}
+	return uint64(len("NULL"))
+}
+func (s *SQLTypeString) ToString() string {
+	if s.Valid {
+		return fmt.Sprintf(`'%s'`, escape(s.String))
+	} else {
+		return "NULL"
+	}
+}
+
+func escape(src string) string {
+	src = strings.ReplaceAll(src, "'", "''")
+	return strings.ReplaceAll(src, `\`, `\\`)
+}
+
+type SQLTypeBytes struct {
+	bytes []byte
+}
+
+func (s *SQLTypeBytes) BindAddress(arg []interface{}) {
+	arg[0] = &s.bytes
+}
+func (s *SQLTypeBytes) ReportSize() uint64 {
+	return uint64(len(s.bytes))
+}
+func (s *SQLTypeBytes) ToString() string {
+	return fmt.Sprintf("x'%x'", s.bytes)
+}

v4/export/test_util.go

@@ -2,7 +2,10 @@ package export
 
 import (
 	"database/sql"
+	"database/sql/driver"
 	"fmt"
+
+	"github.com/DATA-DOG/go-sqlmock"
 )
 
 type mockStringWriter struct {
@@ -71,65 +74,60 @@ func newMockMetaIR(targetName string, meta string, specialComments []string) Met
 	}
 }
 
-func makeNullString(ss []string) []sql.NullString {
-	var ns []sql.NullString
-	for _, s := range ss {
-		if len(s) != 0 {
-			ns = append(ns, sql.NullString{String: s, Valid: true})
-		} else {
-			ns = append(ns, sql.NullString{Valid: false})
-		}
-	}
-	return ns
-}
-
-type mockTableDataIR struct {
+type mockTableIR struct {
 	dbName   string
 	tblName  string
-	data     [][]sql.NullString
+	data     [][]driver.Value
 	specCmt  []string
 	colTypes []string
 }
 
-func (m *mockTableDataIR) ColumnTypes() []string {
-	return m.colTypes
+func (m *mockTableIR) DatabaseName() string {
+	return m.dbName
 }
 
-func newMockTableDataIR(databaseName, tableName string, data [][]string, specialComments []string, colTypes []string) TableDataIR {
-	var nData [][]sql.NullString
-	for _, ss := range data {
-		nData = append(nData, makeNullString(ss))
-	}
-
-	return &mockTableDataIR{
-		dbName:   databaseName,
-		tblName:  tableName,
-		data:     nData,
-		specCmt:  specialComments,
-		colTypes: colTypes,
-	}
+func (m *mockTableIR) TableName() string {
+	return m.tblName
 }
 
-func (m *mockTableDataIR) DatabaseName() string {
-	return m.dbName
+func (m *mockTableIR) ColumnCount() uint {
+	return uint(len(m.colTypes))
 }
 
-func (m *mockTableDataIR) TableName() string {
-	return "employee"
+func (m *mockTableIR) ColumnTypes() []string {
+	return m.colTypes
 }
 
-func (m *mockTableDataIR) ColumnCount() uint {
-	return 5
+func (m *mockTableIR) SpecialComments() StringIter {
+	return newStringIter(m.specCmt...)
 }
 
-func (m *mockTableDataIR) SpecialComments() StringIter {
-	return newStringIter(m.specCmt...)
+func (m *mockTableIR) Rows() SQLRowIter {
+	mockRows := sqlmock.NewRows(m.colTypes)
+	for _, datum := range m.data {
+		mockRows.AddRow(datum...)
+	}
+	db, mock, err := sqlmock.New()
+	if err != nil {
+		panic(fmt.Sprintf("sqlmock.New return error: %v", err))
+	}
+	defer db.Close()
+	mock.ExpectQuery("select 1").WillReturnRows(mockRows)
+	rows, err := db.Query("select 1")
+	if err != nil {
+		panic(fmt.Sprintf("sqlmock.New return error: %v", err))
+	}
+
+	return newRowIter(rows, len(m.colTypes))
 }
 
-func (m *mockTableDataIR) Rows() SQLRowIter {
-	return &mockSQLRowIterator{
-		idx:  0,
-		data: m.data,
+func newMockTableIR(databaseName, tableName string, data [][]driver.Value, specialComments, colTypes []string) TableDataIR {
+	return &mockTableIR{
+		dbName:   databaseName,
+		tblName:  tableName,
+		data:     data,
+		specCmt:  specialComments,
+		colTypes: colTypes,
 	}
 }