From b364b89b48b54fe8f8d92d912888024bf198434a Mon Sep 17 00:00:00 2001 From: lysu Date: Mon, 9 Dec 2019 14:49:08 +0800 Subject: [PATCH] rowcodec: make rowcodec can be used for mocktikv & unistore (#13774) --- util/rowcodec/bench_test.go | 94 ++++ util/rowcodec/common.go | 156 ++++--- util/rowcodec/decoder.go | 443 +++++++++++++----- util/rowcodec/encoder.go | 302 +++++-------- util/rowcodec/export_test.go | 49 ++ util/rowcodec/row.go | 192 ++++++++ util/rowcodec/rowcodec_test.go | 804 +++++++++++++++++++++++++++------ 7 files changed, 1543 insertions(+), 497 deletions(-) create mode 100644 util/rowcodec/bench_test.go create mode 100644 util/rowcodec/export_test.go create mode 100644 util/rowcodec/row.go diff --git a/util/rowcodec/bench_test.go b/util/rowcodec/bench_test.go new file mode 100644 index 0000000000000..39ba1ddc09c04 --- /dev/null +++ b/util/rowcodec/bench_test.go @@ -0,0 +1,94 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package rowcodec_test + +import ( + "testing" + "time" + + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/sessionctx/stmtctx" + "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/rowcodec" +) + +func BenchmarkEncode(b *testing.B) { + b.ReportAllocs() + oldRow := types.MakeDatums(1, "abc", 1.1) + var xb rowcodec.Encoder + var buf []byte + colIDs := []int64{1, 2, 3} + var err error + for i := 0; i < b.N; i++ { + buf, err = xb.Encode(nil, colIDs, oldRow, buf) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeFromOldRow(b *testing.B) { + b.ReportAllocs() + oldRow := types.MakeDatums(1, "abc", 1.1) + oldRowData, err := tablecodec.EncodeRow(new(stmtctx.StatementContext), oldRow, []int64{1, 2, 3}, nil, nil) + if err != nil { + b.Fatal(err) + } + var xb rowcodec.Encoder + var buf []byte + for i := 0; i < b.N; i++ { + buf, err = rowcodec.EncodeFromOldRow(&xb, nil, oldRowData, buf) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDecode(b *testing.B) { + b.ReportAllocs() + oldRow := types.MakeDatums(1, "abc", 1.1) + colIDs := []int64{-1, 2, 3} + tps := []*types.FieldType{ + types.NewFieldType(mysql.TypeLonglong), + types.NewFieldType(mysql.TypeString), + types.NewFieldType(mysql.TypeDouble), + } + var xb rowcodec.Encoder + xRowData, err := xb.Encode(nil, colIDs, oldRow, nil) + if err != nil { + b.Fatal(err) + } + cols := make([]rowcodec.ColInfo, len(tps)) + for i, tp := range tps { + cols[i] = rowcodec.ColInfo{ + ID: colIDs[i], + Tp: int32(tp.Tp), + Flag: int32(tp.Flag), + Flen: tp.Flen, + Decimal: tp.Decimal, + Elems: tp.Elems, + } + } + decoder := rowcodec.NewChunkDecoder(cols, -1, nil, time.Local) + chk := chunk.NewChunkWithCapacity(tps, 1) + for i := 0; i < b.N; i++ { + chk.Reset() + err = decoder.DecodeToChunk(xRowData, 1, chk) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/util/rowcodec/common.go b/util/rowcodec/common.go index 3ce6f9621755b..ef1a20b035e42 100644 --- a/util/rowcodec/common.go +++ b/util/rowcodec/common.go @@ -26,68 +26,19 @@ const CodecVer = 128 var errInvalidCodecVer = errors.New("invalid codec version") -// row is the struct type used to access a row. -// There are two types of row, small and large. -// A small row takes one byte colID and two bytes offset, optimized for most cases. -// If the max colID is larger than 255 or total value size is larger than 65535, the row type would be large. -// A large row takes four bytes colID and four bytes offset. -type row struct { - isLarge bool - numNotNullCols uint16 - numNullCols uint16 - data []byte - - // for small rows - colIDs []byte - offsets []uint16 - - // for large row - colIDs32 []uint32 - offsets32 []uint32 -} - -func (r *row) getData(i int) []byte { - var start, end uint32 - if r.isLarge { - if i > 0 { - start = r.offsets32[i-1] - } - end = r.offsets32[i] - } else { - if i > 0 { - start = uint32(r.offsets[i-1]) - } - end = uint32(r.offsets[i]) - } - return r.data[start:end] -} - -func (r *row) setRowData(rowData []byte) error { - if rowData[0] != CodecVer { - return errInvalidCodecVer - } - r.isLarge = rowData[1]&1 > 0 - r.numNotNullCols = binary.LittleEndian.Uint16(rowData[2:]) - r.numNullCols = binary.LittleEndian.Uint16(rowData[4:]) - cursor := 6 - if r.isLarge { - colIDsLen := int(r.numNotNullCols+r.numNullCols) * 4 - r.colIDs32 = bytesToU32Slice(rowData[cursor : cursor+colIDsLen]) - cursor += colIDsLen - offsetsLen := int(r.numNotNullCols) * 4 - r.offsets32 = bytesToU32Slice(rowData[cursor : cursor+offsetsLen]) - cursor += offsetsLen - } else { - colIDsLen := int(r.numNotNullCols + r.numNullCols) - r.colIDs = rowData[cursor : cursor+colIDsLen] - cursor += colIDsLen - offsetsLen := int(r.numNotNullCols) * 2 - r.offsets = bytes2U16Slice(rowData[cursor : cursor+offsetsLen]) - cursor += offsetsLen - } - r.data = rowData[cursor:] - return nil -} +// First byte in the encoded value which specifies the encoding type. +const ( + NilFlag byte = 0 + BytesFlag byte = 1 + CompactBytesFlag byte = 2 + IntFlag byte = 3 + UintFlag byte = 4 + FloatFlag byte = 5 + DecimalFlag byte = 6 + VarintFlag byte = 8 + VaruintFlag byte = 9 + JSONFlag byte = 10 +) func bytesToU32Slice(b []byte) []uint32 { if len(b) == 0 { @@ -196,3 +147,84 @@ func decodeUint(val []byte) uint64 { return binary.LittleEndian.Uint64(val) } } + +type largeNotNullSorter Encoder + +func (s *largeNotNullSorter) Less(i, j int) bool { + return s.colIDs32[i] < s.colIDs32[j] +} + +func (s *largeNotNullSorter) Len() int { + return int(s.numNotNullCols) +} + +func (s *largeNotNullSorter) Swap(i, j int) { + s.colIDs32[i], s.colIDs32[j] = s.colIDs32[j], s.colIDs32[i] + s.values[i], s.values[j] = s.values[j], s.values[i] +} + +type smallNotNullSorter Encoder + +func (s *smallNotNullSorter) Less(i, j int) bool { + return s.colIDs[i] < s.colIDs[j] +} + +func (s *smallNotNullSorter) Len() int { + return int(s.numNotNullCols) +} + +func (s *smallNotNullSorter) Swap(i, j int) { + s.colIDs[i], s.colIDs[j] = s.colIDs[j], s.colIDs[i] + s.values[i], s.values[j] = s.values[j], s.values[i] +} + +type smallNullSorter Encoder + +func (s *smallNullSorter) Less(i, j int) bool { + nullCols := s.colIDs[s.numNotNullCols:] + return nullCols[i] < nullCols[j] +} + +func (s *smallNullSorter) Len() int { + return int(s.numNullCols) +} + +func (s *smallNullSorter) Swap(i, j int) { + nullCols := s.colIDs[s.numNotNullCols:] + nullCols[i], nullCols[j] = nullCols[j], nullCols[i] +} + +type largeNullSorter Encoder + +func (s *largeNullSorter) Less(i, j int) bool { + nullCols := s.colIDs32[s.numNotNullCols:] + return nullCols[i] < nullCols[j] +} + +func (s *largeNullSorter) Len() int { + return int(s.numNullCols) +} + +func (s *largeNullSorter) Swap(i, j int) { + nullCols := s.colIDs32[s.numNotNullCols:] + nullCols[i], nullCols[j] = nullCols[j], nullCols[i] +} + +const ( + // Length of rowkey. + rowKeyLen = 19 + // Index of record flag 'r' in rowkey used by master tidb-server. + // The rowkey format is t{8 bytes id}_r{8 bytes handle} + recordPrefixIdx = 10 +) + +// IsRowKey determine whether key is row key. +// this method will be used in unistore. +func IsRowKey(key []byte) bool { + return len(key) == rowKeyLen && key[0] == 't' && key[recordPrefixIdx] == 'r' +} + +// IsNewFormat checks whether row data is in new-format. +func IsNewFormat(rowData []byte) bool { + return rowData[0] == CodecVer +} diff --git a/util/rowcodec/decoder.go b/util/rowcodec/decoder.go index 46b091d0f49bb..347542e0c8ae3 100644 --- a/util/rowcodec/decoder.go +++ b/util/rowcodec/decoder.go @@ -14,148 +14,254 @@ package rowcodec import ( - "math" + "fmt" "time" "github.com/pingcap/errors" + "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" - "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/json" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/codec" ) -// Decoder decodes the row to chunk.Chunk. -type Decoder struct { +// decoder contains base util for decode row. +type decoder struct { row - requestColIDs []int64 - handleColID int64 - requestTypes []*types.FieldType - origDefaults [][]byte - loc *time.Location -} - -// NewDecoder creates a NewDecoder. -// requestColIDs is the columnIDs to decode. tps is the field types for request columns. -// origDefault is the original default value in old format, if the column ID is not found in the row, -// the origDefault will be used. -func NewDecoder(requestColIDs []int64, handleColID int64, tps []*types.FieldType, origDefaults [][]byte, - sc *stmtctx.StatementContext) (*Decoder, error) { - xOrigDefaultVals := make([][]byte, len(origDefaults)) - for i := 0; i < len(origDefaults); i++ { - if len(origDefaults[i]) == 0 { + columns []ColInfo + handleColID int64 + loc *time.Location +} + +// NewDecoder creates a decoder. +func NewDecoder(columns []ColInfo, handleColID int64, loc *time.Location) *decoder { + return &decoder{ + columns: columns, + handleColID: handleColID, + loc: loc, + } +} + +// ColInfo is used as column meta info for row decoder. +type ColInfo struct { + ID int64 + Tp int32 + Flag int32 + IsPKHandle bool + + Flen int + Decimal int + Elems []string +} + +// DatumMapDecoder decodes the row to datum map. +type DatumMapDecoder struct { + decoder +} + +// NewDatumMapDecoder creates a DatumMapDecoder. +func NewDatumMapDecoder(columns []ColInfo, handleColID int64, loc *time.Location) *DatumMapDecoder { + return &DatumMapDecoder{decoder{ + columns: columns, + handleColID: handleColID, + loc: loc, + }} +} + +// DecodeToDatumMap decodes byte slices to datum map. +func (decoder *DatumMapDecoder) DecodeToDatumMap(rowData []byte, handle int64, row map[int64]types.Datum) (map[int64]types.Datum, error) { + if row == nil { + row = make(map[int64]types.Datum, len(decoder.columns)) + } + err := decoder.fromBytes(rowData) + if err != nil { + return nil, err + } + for _, col := range decoder.columns { + if col.ID == decoder.handleColID { + row[col.ID] = types.NewIntDatum(handle) continue } - xDefaultVal, err := convertDefaultValue(origDefaults[i], sc) - if err != nil { - return nil, err + idx, isNil, notFound := decoder.row.findColID(col.ID) + if !notFound && !isNil { + colData := decoder.getData(idx) + d, err := decoder.decodeColDatum(&col, colData) + if err != nil { + return nil, err + } + row[col.ID] = d + continue + } + + if isNil { + var d types.Datum + d.SetNull() + row[col.ID] = d + continue } - xOrigDefaultVals[i] = xDefaultVal } - return &Decoder{ - requestColIDs: requestColIDs, - handleColID: handleColID, - requestTypes: tps, - origDefaults: xOrigDefaultVals, - loc: sc.TimeZone, - }, nil + return row, nil } -func convertDefaultValue(defaultVal []byte, sc *stmtctx.StatementContext) (colVal []byte, err error) { +func (decoder *DatumMapDecoder) decodeColDatum(col *ColInfo, colData []byte) (types.Datum, error) { var d types.Datum - _, d, err = codec.DecodeOne(defaultVal) - if err != nil { - return + switch byte(col.Tp) { + case mysql.TypeLonglong, mysql.TypeLong, mysql.TypeInt24, mysql.TypeShort, mysql.TypeTiny, mysql.TypeYear: + if mysql.HasUnsignedFlag(uint(col.Flag)) { + d.SetUint64(decodeUint(colData)) + } else { + d.SetInt64(decodeInt(colData)) + } + case mysql.TypeFloat: + _, fVal, err := codec.DecodeFloat(colData) + if err != nil { + return d, err + } + d.SetFloat32(float32(fVal)) + case mysql.TypeDouble: + _, fVal, err := codec.DecodeFloat(colData) + if err != nil { + return d, err + } + d.SetFloat64(fVal) + case mysql.TypeVarString, mysql.TypeVarchar, mysql.TypeString, + mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: + d.SetBytes(colData) + case mysql.TypeNewDecimal: + _, dec, _, _, err := codec.DecodeDecimal(colData) + if err != nil { + return d, err + } + d.SetMysqlDecimal(dec) + case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp: + var t types.Time + t.Type = uint8(col.Tp) + t.Fsp = int8(col.Decimal) + err := t.FromPackedUint(decodeUint(colData)) + if err != nil { + return d, err + } + if byte(col.Tp) == mysql.TypeTimestamp && !t.IsZero() { + err = t.ConvertTimeZone(time.UTC, decoder.loc) + if err != nil { + return d, err + } + } + d.SetMysqlTime(t) + case mysql.TypeDuration: + var dur types.Duration + dur.Duration = time.Duration(decodeInt(colData)) + dur.Fsp = int8(col.Decimal) + d.SetMysqlDuration(dur) + case mysql.TypeEnum: + // ignore error deliberately, to read empty enum value. + enum, err := types.ParseEnumValue(col.Elems, decodeUint(colData)) + if err != nil { + enum = types.Enum{} + } + d.SetMysqlEnum(enum) + case mysql.TypeSet: + set, err := types.ParseSetValue(col.Elems, decodeUint(colData)) + if err != nil { + return d, err + } + d.SetMysqlSet(set) + case mysql.TypeBit: + byteSize := (col.Flen + 7) >> 3 + d.SetMysqlBit(types.NewBinaryLiteralFromUint(decodeUint(colData), byteSize)) + case mysql.TypeJSON: + var j json.BinaryJSON + j.TypeCode = colData[0] + j.Value = colData[1:] + d.SetMysqlJSON(j) + default: + return d, errors.Errorf("unknown type %d", col.Tp) + } + return d, nil +} + +// ChunkDecoder decodes the row to chunk.Chunk. +type ChunkDecoder struct { + decoder + defDatum func(i int) (types.Datum, error) +} + +// NewChunkDecoder creates a NewChunkDecoder. +func NewChunkDecoder(columns []ColInfo, handleColID int64, defDatum func(i int) (types.Datum, error), loc *time.Location) *ChunkDecoder { + return &ChunkDecoder{ + decoder: decoder{ + columns: columns, + handleColID: handleColID, + loc: loc, + }, + defDatum: defDatum, } - return encodeDatum(nil, d, sc) } -// Decode decodes a row to chunk. -func (decoder *Decoder) Decode(rowData []byte, handle int64, chk *chunk.Chunk) error { - err := decoder.setRowData(rowData) +// DecodeToChunk decodes a row to chunk. +func (decoder *ChunkDecoder) DecodeToChunk(rowData []byte, handle int64, chk *chunk.Chunk) error { + err := decoder.fromBytes(rowData) if err != nil { return err } - for colIdx, colID := range decoder.requestColIDs { - if colID == decoder.handleColID { + + for colIdx, col := range decoder.columns { + if col.ID == decoder.handleColID { chk.AppendInt64(colIdx, handle) continue } - // Search the column in not-null columns array. - i, j := 0, int(decoder.numNotNullCols) - var found bool - for i < j { - h := int(uint(i+j) >> 1) // avoid overflow when computing h - // i ≤ h < j - var v int64 - if decoder.isLarge { - v = int64(decoder.colIDs32[h]) - } else { - v = int64(decoder.colIDs[h]) - } - if v < colID { - i = h + 1 - } else if v > colID { - j = h - } else { - found = true - colData := decoder.getData(h) - err := decoder.decodeColData(colIdx, colData, chk) - if err != nil { - return err - } - break + + idx, isNil, notFound := decoder.row.findColID(col.ID) + if !notFound && !isNil { + colData := decoder.getData(idx) + err := decoder.decodeColToChunk(colIdx, &col, colData, chk) + if err != nil { + return err } - } - if found { continue } - // Search the column in null columns array. - i, j = int(decoder.numNotNullCols), int(decoder.numNotNullCols+decoder.numNullCols) - for i < j { - h := int(uint(i+j) >> 1) // avoid overflow when computing h - // i ≤ h < j - var v int64 - if decoder.isLarge { - v = int64(decoder.colIDs32[h]) - } else { - v = int64(decoder.colIDs[h]) - } - if v < colID { - i = h + 1 - } else if v > colID { - j = h - } else { - found = true - break - } + + if isNil { + chk.AppendNull(colIdx) + continue } - if found || decoder.origDefaults[colIdx] == nil { + + if decoder.defDatum == nil { chk.AppendNull(colIdx) - } else { - err := decoder.decodeColData(colIdx, decoder.origDefaults[colIdx], chk) - if err != nil { - return err - } + continue } + + defDatum, err := decoder.defDatum(colIdx) + if err != nil { + return err + } + + chk.AppendDatum(colIdx, &defDatum) } return nil } -func (decoder *Decoder) decodeColData(colIdx int, colData []byte, chk *chunk.Chunk) error { - ft := decoder.requestTypes[colIdx] - switch ft.Tp { +func (decoder *ChunkDecoder) decodeColToChunk(colIdx int, col *ColInfo, colData []byte, chk *chunk.Chunk) error { + switch byte(col.Tp) { case mysql.TypeLonglong, mysql.TypeLong, mysql.TypeInt24, mysql.TypeShort, mysql.TypeTiny, mysql.TypeYear: - if mysql.HasUnsignedFlag(ft.Flag) { + if mysql.HasUnsignedFlag(uint(col.Flag)) { chk.AppendUint64(colIdx, decodeUint(colData)) } else { chk.AppendInt64(colIdx, decodeInt(colData)) } case mysql.TypeFloat: - chk.AppendFloat32(colIdx, float32(math.Float64frombits(decodeUint(colData)))) + _, fVal, err := codec.DecodeFloat(colData) + if err != nil { + return err + } + chk.AppendFloat32(colIdx, float32(fVal)) case mysql.TypeDouble: - chk.AppendFloat64(colIdx, math.Float64frombits(decodeUint(colData))) + _, fVal, err := codec.DecodeFloat(colData) + if err != nil { + return err + } + chk.AppendFloat64(colIdx, fVal) case mysql.TypeVarString, mysql.TypeVarchar, mysql.TypeString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: chk.AppendBytes(colIdx, colData) @@ -167,13 +273,13 @@ func (decoder *Decoder) decodeColData(colIdx int, colData []byte, chk *chunk.Chu chk.AppendMyDecimal(colIdx, dec) case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp: var t types.Time - t.Type = ft.Tp - t.Fsp = int8(ft.Decimal) + t.Type = uint8(col.Tp) + t.Fsp = int8(col.Decimal) err := t.FromPackedUint(decodeUint(colData)) if err != nil { return err } - if ft.Tp == mysql.TypeTimestamp && !t.IsZero() { + if byte(col.Tp) == mysql.TypeTimestamp && decoder.loc != nil && !t.IsZero() { err = t.ConvertTimeZone(time.UTC, decoder.loc) if err != nil { return err @@ -183,23 +289,23 @@ func (decoder *Decoder) decodeColData(colIdx int, colData []byte, chk *chunk.Chu case mysql.TypeDuration: var dur types.Duration dur.Duration = time.Duration(decodeInt(colData)) - dur.Fsp = int8(ft.Decimal) + dur.Fsp = int8(col.Decimal) chk.AppendDuration(colIdx, dur) case mysql.TypeEnum: // ignore error deliberately, to read empty enum value. - enum, err := types.ParseEnumValue(ft.Elems, decodeUint(colData)) + enum, err := types.ParseEnumValue(col.Elems, decodeUint(colData)) if err != nil { enum = types.Enum{} } chk.AppendEnum(colIdx, enum) case mysql.TypeSet: - set, err := types.ParseSetValue(ft.Elems, decodeUint(colData)) + set, err := types.ParseSetValue(col.Elems, decodeUint(colData)) if err != nil { return err } chk.AppendSet(colIdx, set) case mysql.TypeBit: - byteSize := (ft.Flen + 7) >> 3 + byteSize := (col.Flen + 7) >> 3 chk.AppendBytes(colIdx, types.NewBinaryLiteralFromUint(decodeUint(colData), byteSize)) case mysql.TypeJSON: var j json.BinaryJSON @@ -207,7 +313,132 @@ func (decoder *Decoder) decodeColData(colIdx int, colData []byte, chk *chunk.Chu j.Value = colData[1:] chk.AppendJSON(colIdx, j) default: - return errors.Errorf("unknown type %d", ft.Tp) + return errors.Errorf("unknown type %d", col.Tp) } return nil } + +// BytesDecoder decodes the row to old datums bytes. +type BytesDecoder struct { + decoder + defBytes func(i int) ([]byte, error) +} + +// NewByteDecoder creates a BytesDecoder. +// defBytes: provided default value bytes in old datum format(flag+colData). +func NewByteDecoder(columns []ColInfo, handleColID int64, defBytes func(i int) ([]byte, error), loc *time.Location) *BytesDecoder { + return &BytesDecoder{ + decoder: decoder{ + columns: columns, + handleColID: handleColID, + loc: loc, + }, + defBytes: defBytes, + } +} + +// DecodeToBytes decodes raw byte slice to row data. +func (decoder *BytesDecoder) DecodeToBytes(outputOffset map[int64]int, handle int64, value []byte, cacheBytes []byte) ([][]byte, error) { + var r row + err := r.fromBytes(value) + if err != nil { + return nil, err + } + values := make([][]byte, len(outputOffset)) + for i, col := range decoder.columns { + tp := fieldType2Flag(byte(col.Tp), uint(col.Flag)&mysql.UnsignedFlag == 0) + colID := col.ID + offset := outputOffset[colID] + if col.IsPKHandle || colID == model.ExtraHandleID { + handleData := cacheBytes + if mysql.HasUnsignedFlag(uint(col.Flag)) { + handleData = append(handleData, UintFlag) + handleData = codec.EncodeUint(handleData, uint64(handle)) + } else { + handleData = append(handleData, IntFlag) + handleData = codec.EncodeInt(handleData, handle) + } + values[offset] = handleData + continue + } + + idx, isNil, notFound := r.findColID(colID) + if !notFound && !isNil { + val := r.getData(idx) + values[offset] = decoder.encodeOldDatum(tp, val) + continue + } + + if isNil { + values[offset] = []byte{NilFlag} + continue + } + + if decoder.defBytes != nil { + defVal, err := decoder.defBytes(i) + if err != nil { + return nil, err + } + if len(defVal) > 0 { + values[offset] = defVal + continue + } + } + + values[offset] = []byte{NilFlag} + } + return values, nil +} + +func (decoder *BytesDecoder) encodeOldDatum(tp byte, val []byte) []byte { + var buf []byte + switch tp { + case BytesFlag: + buf = append(buf, CompactBytesFlag) + buf = codec.EncodeCompactBytes(buf, val) + case IntFlag: + buf = append(buf, VarintFlag) + buf = codec.EncodeVarint(buf, decodeInt(val)) + case UintFlag: + buf = append(buf, VaruintFlag) + buf = codec.EncodeUvarint(buf, decodeUint(val)) + default: + buf = append(buf, tp) + buf = append(buf, val...) + } + return buf +} + +// fieldType2Flag transforms field type into kv type flag. +func fieldType2Flag(tp byte, signed bool) (flag byte) { + switch tp { + case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong: + if signed { + flag = IntFlag + } else { + flag = UintFlag + } + case mysql.TypeFloat, mysql.TypeDouble: + flag = FloatFlag + case mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, + mysql.TypeString, mysql.TypeVarchar, mysql.TypeVarString: + flag = BytesFlag + case mysql.TypeDatetime, mysql.TypeDate, mysql.TypeTimestamp: + flag = UintFlag + case mysql.TypeDuration: + flag = IntFlag + case mysql.TypeNewDecimal: + flag = DecimalFlag + case mysql.TypeYear: + flag = IntFlag + case mysql.TypeEnum, mysql.TypeBit, mysql.TypeSet: + flag = UintFlag + case mysql.TypeJSON: + flag = JSONFlag + case mysql.TypeNull: + flag = NilFlag + default: + panic(fmt.Sprintf("unknown field type %d", tp)) + } + return +} diff --git a/util/rowcodec/encoder.go b/util/rowcodec/encoder.go index 87ad0b4712bf6..ad999d4a313ee 100644 --- a/util/rowcodec/encoder.go +++ b/util/rowcodec/encoder.go @@ -31,65 +31,70 @@ type Encoder struct { row tempColIDs []int64 values []types.Datum - sc *stmtctx.StatementContext } -// NewEncoder creates a new Encoder with column IDs. -func NewEncoder(colIDs []int64, sc *stmtctx.StatementContext) *Encoder { - return &Encoder{ - tempColIDs: colIDs, - sc: sc, +// Encode encodes a row from a datums slice. +func (encoder *Encoder) Encode(sc *stmtctx.StatementContext, colIDs []int64, values []types.Datum, buf []byte) ([]byte, error) { + encoder.reset() + encoder.appendColVals(colIDs, values) + numCols, notNullIdx := encoder.reformatCols() + err := encoder.encodeRowCols(sc, numCols, notNullIdx) + if err != nil { + return nil, err } + return encoder.row.toBytes(buf[:0]), nil } func (encoder *Encoder) reset() { - encoder.isLarge = false + encoder.large = false encoder.numNotNullCols = 0 encoder.numNullCols = 0 encoder.data = encoder.data[:0] + encoder.tempColIDs = encoder.tempColIDs[:0] encoder.values = encoder.values[:0] } -// Encode encodes a row from a datums slice. -func (encoder *Encoder) Encode(values []types.Datum, buf []byte) ([]byte, error) { - encoder.reset() - encoder.values = append(encoder.values, values...) - for i, colID := range encoder.tempColIDs { - if colID > 255 { - encoder.isLarge = true - } - if values[i].IsNull() { - encoder.numNullCols++ - } else { - encoder.numNotNullCols++ - } +func (encoder *Encoder) appendColVals(colIDs []int64, values []types.Datum) { + for i, colID := range colIDs { + encoder.appendColVal(colID, values[i]) + } +} + +func (encoder *Encoder) appendColVal(colID int64, d types.Datum) { + if colID > 255 { + encoder.large = true + } + if d.IsNull() { + encoder.numNullCols++ + } else { + encoder.numNotNullCols++ } - return encoder.build(buf) + encoder.tempColIDs = append(encoder.tempColIDs, colID) + encoder.values = append(encoder.values, d) } -func (encoder *Encoder) build(buf []byte) ([]byte, error) { +func (encoder *Encoder) reformatCols() (numCols, notNullIdx int) { r := &encoder.row - // Separate null and not-null column IDs. - numCols := len(encoder.tempColIDs) + numCols = len(encoder.tempColIDs) nullIdx := numCols - int(r.numNullCols) - notNullIdx := 0 - if r.isLarge { - encoder.initColIDs32() - encoder.initOffsets32() + notNullIdx = 0 + if r.large { + r.initColIDs32() + r.initOffsets32() } else { - encoder.initColIDs() - encoder.initOffsets() + r.initColIDs() + r.initOffsets() } for i, colID := range encoder.tempColIDs { if encoder.values[i].IsNull() { - if r.isLarge { + if r.large { r.colIDs32[nullIdx] = uint32(colID) } else { r.colIDs[nullIdx] = byte(colID) } nullIdx++ } else { - if r.isLarge { + if r.large { r.colIDs32[notNullIdx] = uint32(colID) } else { r.colIDs[notNullIdx] = byte(colID) @@ -98,7 +103,7 @@ func (encoder *Encoder) build(buf []byte) ([]byte, error) { notNullIdx++ } } - if r.isLarge { + if r.large { largeNotNullSorter := (*largeNotNullSorter)(encoder) sort.Sort(largeNotNullSorter) if r.numNullCols > 0 { @@ -113,203 +118,114 @@ func (encoder *Encoder) build(buf []byte) ([]byte, error) { sort.Sort(smallNullSorter) } } + return +} + +func (encoder *Encoder) encodeRowCols(sc *stmtctx.StatementContext, numCols, notNullIdx int) error { + r := &encoder.row for i := 0; i < notNullIdx; i++ { + d := encoder.values[i] var err error - r.data, err = encodeDatum(r.data, encoder.values[i], encoder.sc) + r.data, err = EncodeValueDatum(sc, d, r.data) if err != nil { - return nil, errors.Trace(err) + return err } - if len(r.data) > math.MaxUint16 && !r.isLarge { - // We need to convert the row to large row. - encoder.initColIDs32() + // handle convert to large + if len(r.data) > math.MaxUint16 && !r.large { + r.initColIDs32() for j := 0; j < numCols; j++ { r.colIDs32[j] = uint32(r.colIDs[j]) } - encoder.initOffsets32() + r.initOffsets32() for j := 0; j <= i; j++ { r.offsets32[j] = uint32(r.offsets[j]) } - r.isLarge = true + r.large = true } - if r.isLarge { + if r.large { r.offsets32[i] = uint32(len(r.data)) } else { r.offsets[i] = uint16(len(r.data)) } } - buf = append(buf, CodecVer) - flag := byte(0) - if r.isLarge { - flag = 1 - } - buf = append(buf, flag) - buf = append(buf, byte(r.numNotNullCols), byte(r.numNotNullCols>>8)) - buf = append(buf, byte(r.numNullCols), byte(r.numNullCols>>8)) - if r.isLarge { - buf = append(buf, u32SliceToBytes(r.colIDs32)...) - buf = append(buf, u32SliceToBytes(r.offsets32)...) - } else { - buf = append(buf, r.colIDs...) - buf = append(buf, u16SliceToBytes(r.offsets)...) + // handle convert to large + if !r.large { + if len(r.data) >= math.MaxUint16 { + r.large = true + r.initColIDs32() + for i, val := range r.colIDs { + r.colIDs32[i] = uint32(val) + } + } else { + r.initOffsets() + for i, val := range r.offsets32 { + r.offsets[i] = uint16(val) + } + } } - buf = append(buf, r.data...) - return buf, nil + return nil } -func encodeDatum(buf []byte, d types.Datum, sc *stmtctx.StatementContext) ([]byte, error) { +// EncodeValueDatum encodes one row datum entry into bytes. +// due to encode as value, this method will flatten value type like tablecodec.flatten +func EncodeValueDatum(sc *stmtctx.StatementContext, d types.Datum, buffer []byte) (nBuffer []byte, err error) { switch d.Kind() { case types.KindInt64: - buf = encodeInt(buf, d.GetInt64()) + buffer = encodeInt(buffer, d.GetInt64()) case types.KindUint64: - buf = encodeUint(buf, d.GetUint64()) + buffer = encodeUint(buffer, d.GetUint64()) case types.KindString, types.KindBytes: - buf = append(buf, d.GetBytes()...) - case types.KindFloat32, types.KindFloat64: - buf = encodeUint(buf, uint64(math.Float64bits(d.GetFloat64()))) - case types.KindMysqlDecimal: - var err error - buf, err = codec.EncodeDecimal(buf, d.GetMysqlDecimal(), d.Length(), d.Frac()) - if terror.ErrorEqual(err, types.ErrTruncated) { - err = sc.HandleTruncate(err) - } else if terror.ErrorEqual(err, types.ErrOverflow) { - err = sc.HandleOverflow(err, err) - } - if err != nil { - return nil, errors.Trace(err) - } + buffer = append(buffer, d.GetBytes()...) case types.KindMysqlTime: + // for mysql datetime, timestamp and date type t := d.GetMysqlTime() - // Encoding timestamp need to consider timezone. - // If it's not in UTC, transform to UTC first. - if t.Type == mysql.TypeTimestamp && sc.TimeZone != time.UTC { - err := t.ConvertTimeZone(sc.TimeZone, time.UTC) + if t.Type == mysql.TypeTimestamp && sc != nil && sc.TimeZone != time.UTC { + err = t.ConvertTimeZone(sc.TimeZone, time.UTC) if err != nil { - return nil, errors.Trace(err) + return } } - v, err := t.ToPackedUint() + var v uint64 + v, err = t.ToPackedUint() if err != nil { - return nil, errors.Trace(err) + return } - buf = encodeUint(buf, v) + buffer = encodeUint(buffer, v) case types.KindMysqlDuration: - buf = encodeInt(buf, int64(d.GetMysqlDuration().Duration)) + buffer = encodeInt(buffer, int64(d.GetMysqlDuration().Duration)) case types.KindMysqlEnum: - buf = encodeUint(buf, uint64(d.GetMysqlEnum().ToNumber())) + buffer = encodeUint(buffer, d.GetMysqlEnum().Value) case types.KindMysqlSet: - buf = encodeUint(buf, uint64(d.GetMysqlSet().ToNumber())) - case types.KindMysqlBit, types.KindBinaryLiteral: - val, err := types.BinaryLiteral(d.GetBytes()).ToInt(sc) + buffer = encodeUint(buffer, d.GetMysqlSet().Value) + case types.KindBinaryLiteral, types.KindMysqlBit: + // We don't need to handle errors here since the literal is ensured to be able to store in uint64 in convertToMysqlBit. + var val uint64 + val, err = d.GetBinaryLiteral().ToInt(sc) if err != nil { - terror.Log(errors.Trace(err)) + return + } + buffer = encodeUint(buffer, val) + case types.KindFloat32, types.KindFloat64: + buffer = codec.EncodeFloat(buffer, d.GetFloat64()) + case types.KindMysqlDecimal: + buffer, err = codec.EncodeDecimal(buffer, d.GetMysqlDecimal(), d.Length(), d.Frac()) + if sc != nil { + if terror.ErrorEqual(err, types.ErrTruncated) { + err = sc.HandleTruncate(err) + } else if terror.ErrorEqual(err, types.ErrOverflow) { + err = sc.HandleOverflow(err, err) + } } - buf = encodeUint(buf, val) case types.KindMysqlJSON: j := d.GetMysqlJSON() - buf = append(buf, j.TypeCode) - buf = append(buf, j.Value...) + buffer = append(buffer, j.TypeCode) + buffer = append(buffer, j.Value...) + case types.KindNull: + case types.KindMinNotNull: + case types.KindMaxValue: default: - return nil, errors.Errorf("unsupport encode type %d", d.Kind()) + err = errors.Errorf("unsupport encode type %d", d.Kind()) } - return buf, nil -} - -func (encoder *Encoder) initColIDs() { - numCols := int(encoder.numNotNullCols + encoder.numNullCols) - if cap(encoder.colIDs) >= numCols { - encoder.colIDs = encoder.colIDs[:numCols] - } else { - encoder.colIDs = make([]byte, numCols) - } -} - -func (encoder *Encoder) initColIDs32() { - numCols := int(encoder.numNotNullCols + encoder.numNullCols) - if cap(encoder.colIDs32) >= numCols { - encoder.colIDs32 = encoder.colIDs32[:numCols] - } else { - encoder.colIDs32 = make([]uint32, numCols) - } -} - -func (encoder *Encoder) initOffsets() { - if cap(encoder.offsets) >= int(encoder.numNotNullCols) { - encoder.offsets = encoder.offsets[:encoder.numNotNullCols] - } else { - encoder.offsets = make([]uint16, encoder.numNotNullCols) - } -} - -func (encoder *Encoder) initOffsets32() { - if cap(encoder.offsets32) >= int(encoder.numNotNullCols) { - encoder.offsets32 = encoder.offsets32[:encoder.numNotNullCols] - } else { - encoder.offsets32 = make([]uint32, encoder.numNotNullCols) - } -} - -/* - We define several sorters to avoid switch cost in sort functions. -*/ - -type largeNotNullSorter Encoder - -func (s *largeNotNullSorter) Less(i, j int) bool { - return s.colIDs32[i] < s.colIDs32[j] -} - -func (s *largeNotNullSorter) Len() int { - return int(s.numNotNullCols) -} - -func (s *largeNotNullSorter) Swap(i, j int) { - s.colIDs32[i], s.colIDs32[j] = s.colIDs32[j], s.colIDs32[i] - s.values[i], s.values[j] = s.values[j], s.values[i] -} - -type smallNotNullSorter Encoder - -func (s *smallNotNullSorter) Less(i, j int) bool { - return s.colIDs[i] < s.colIDs[j] -} - -func (s *smallNotNullSorter) Len() int { - return int(s.numNotNullCols) -} - -func (s *smallNotNullSorter) Swap(i, j int) { - s.colIDs[i], s.colIDs[j] = s.colIDs[j], s.colIDs[i] - s.values[i], s.values[j] = s.values[j], s.values[i] -} - -type smallNullSorter Encoder - -func (s *smallNullSorter) Less(i, j int) bool { - nullCols := s.colIDs[s.numNotNullCols:] - return nullCols[i] < nullCols[j] -} - -func (s *smallNullSorter) Len() int { - return int(s.numNullCols) -} - -func (s *smallNullSorter) Swap(i, j int) { - nullCols := s.colIDs[s.numNotNullCols:] - nullCols[i], nullCols[j] = nullCols[j], nullCols[i] -} - -type largeNullSorter Encoder - -func (s *largeNullSorter) Less(i, j int) bool { - nullCols := s.colIDs32[s.numNotNullCols:] - return nullCols[i] < nullCols[j] -} - -func (s *largeNullSorter) Len() int { - return int(s.numNullCols) -} - -func (s *largeNullSorter) Swap(i, j int) { - nullCols := s.colIDs32[s.numNotNullCols:] - nullCols[i], nullCols[j] = nullCols[j], nullCols[i] + nBuffer = buffer + return } diff --git a/util/rowcodec/export_test.go b/util/rowcodec/export_test.go new file mode 100644 index 0000000000000..323113ca64cc8 --- /dev/null +++ b/util/rowcodec/export_test.go @@ -0,0 +1,49 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package rowcodec + +import ( + "github.com/pingcap/tidb/sessionctx/stmtctx" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/codec" +) + +// EncodeFromOldRow encodes a row from an old-format row. +// this method will be used in test. +func EncodeFromOldRow(encoder *Encoder, sc *stmtctx.StatementContext, oldRow, buf []byte) ([]byte, error) { + if len(oldRow) > 0 && oldRow[0] == CodecVer { + return oldRow, nil + } + encoder.reset() + for len(oldRow) > 1 { + var d types.Datum + var err error + oldRow, d, err = codec.DecodeOne(oldRow) + if err != nil { + return nil, err + } + colID := d.GetInt64() + oldRow, d, err = codec.DecodeOne(oldRow) + if err != nil { + return nil, err + } + encoder.appendColVal(colID, d) + } + numCols, notNullIdx := encoder.reformatCols() + err := encoder.encodeRowCols(sc, numCols, notNullIdx) + if err != nil { + return nil, err + } + return encoder.row.toBytes(buf[:0]), nil +} diff --git a/util/rowcodec/row.go b/util/rowcodec/row.go new file mode 100644 index 0000000000000..8eb07dfbde9e9 --- /dev/null +++ b/util/rowcodec/row.go @@ -0,0 +1,192 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package rowcodec + +import ( + "encoding/binary" +) + +// row is the struct type used to access the a row. +type row struct { + // small: colID []byte, offsets []uint16, optimized for most cases. + // large: colID []uint32, offsets []uint32. + large bool + numNotNullCols uint16 + numNullCols uint16 + colIDs []byte + + offsets []uint16 + data []byte + + // for large row + colIDs32 []uint32 + offsets32 []uint32 +} + +func (r *row) getData(i int) []byte { + var start, end uint32 + if r.large { + if i > 0 { + start = r.offsets32[i-1] + } + end = r.offsets32[i] + } else { + if i > 0 { + start = uint32(r.offsets[i-1]) + } + end = uint32(r.offsets[i]) + } + return r.data[start:end] +} + +func (r *row) fromBytes(rowData []byte) error { + if rowData[0] != CodecVer { + return errInvalidCodecVer + } + r.large = rowData[1]&1 > 0 + r.numNotNullCols = binary.LittleEndian.Uint16(rowData[2:]) + r.numNullCols = binary.LittleEndian.Uint16(rowData[4:]) + cursor := 6 + if r.large { + colIDsLen := int(r.numNotNullCols+r.numNullCols) * 4 + r.colIDs32 = bytesToU32Slice(rowData[cursor : cursor+colIDsLen]) + cursor += colIDsLen + offsetsLen := int(r.numNotNullCols) * 4 + r.offsets32 = bytesToU32Slice(rowData[cursor : cursor+offsetsLen]) + cursor += offsetsLen + } else { + colIDsLen := int(r.numNotNullCols + r.numNullCols) + r.colIDs = rowData[cursor : cursor+colIDsLen] + cursor += colIDsLen + offsetsLen := int(r.numNotNullCols) * 2 + r.offsets = bytes2U16Slice(rowData[cursor : cursor+offsetsLen]) + cursor += offsetsLen + } + r.data = rowData[cursor:] + return nil +} + +func (r *row) toBytes(buf []byte) []byte { + buf = append(buf, CodecVer) + flag := byte(0) + if r.large { + flag = 1 + } + buf = append(buf, flag) + buf = append(buf, byte(r.numNotNullCols), byte(r.numNotNullCols>>8)) + buf = append(buf, byte(r.numNullCols), byte(r.numNullCols>>8)) + if r.large { + buf = append(buf, u32SliceToBytes(r.colIDs32)...) + buf = append(buf, u32SliceToBytes(r.offsets32)...) + } else { + buf = append(buf, r.colIDs...) + buf = append(buf, u16SliceToBytes(r.offsets)...) + } + buf = append(buf, r.data...) + return buf +} + +func (r *row) findColID(colID int64) (idx int, isNil, notFound bool) { + // Search the column in not-null columns array. + i, j := 0, int(r.numNotNullCols) + for i < j { + h := int(uint(i+j) >> 1) // avoid overflow when computing h + // i ≤ h < j + var v int64 + if r.large { + v = int64(r.colIDs32[h]) + } else { + v = int64(r.colIDs[h]) + } + if v < colID { + i = h + 1 + } else if v > colID { + j = h + } else { + idx = h + return + } + } + + // Search the column in null columns array. + i, j = int(r.numNotNullCols), int(r.numNotNullCols+r.numNullCols) + for i < j { + h := int(uint(i+j) >> 1) // avoid overflow when computing h + // i ≤ h < j + var v int64 + if r.large { + v = int64(r.colIDs32[h]) + } else { + v = int64(r.colIDs[h]) + } + if v < colID { + i = h + 1 + } else if v > colID { + j = h + } else { + isNil = true + return + } + } + notFound = true + return +} + +// ColumnIsNull returns if the column value is null. Mainly used for count column aggregation. +// this method will used in unistore. +func (r *row) ColumnIsNull(rowData []byte, colID int64, defaultVal []byte) (bool, error) { + err := r.fromBytes(rowData) + if err != nil { + return false, err + } + _, isNil, notFound := r.findColID(colID) + if notFound { + return defaultVal == nil, nil + } + return isNil, nil +} + +func (r *row) initColIDs() { + numCols := int(r.numNotNullCols + r.numNullCols) + if cap(r.colIDs) >= numCols { + r.colIDs = r.colIDs[:numCols] + } else { + r.colIDs = make([]byte, numCols) + } +} + +func (r *row) initColIDs32() { + numCols := int(r.numNotNullCols + r.numNullCols) + if cap(r.colIDs32) >= numCols { + r.colIDs32 = r.colIDs32[:numCols] + } else { + r.colIDs32 = make([]uint32, numCols) + } +} + +func (r *row) initOffsets() { + if cap(r.offsets) >= int(r.numNotNullCols) { + r.offsets = r.offsets[:r.numNotNullCols] + } else { + r.offsets = make([]uint16, r.numNotNullCols) + } +} + +func (r *row) initOffsets32() { + if cap(r.offsets32) >= int(r.numNotNullCols) { + r.offsets32 = r.offsets32[:r.numNotNullCols] + } else { + r.offsets32 = make([]uint32, r.numNotNullCols) + } +} diff --git a/util/rowcodec/rowcodec_test.go b/util/rowcodec/rowcodec_test.go index 4324404b3225a..8993c01bef102 100644 --- a/util/rowcodec/rowcodec_test.go +++ b/util/rowcodec/rowcodec_test.go @@ -11,20 +11,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -package rowcodec +package rowcodec_test import ( "math" + "strings" "testing" "time" . "github.com/pingcap/check" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/sessionctx/stmtctx" + "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/json" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/codec" + "github.com/pingcap/tidb/util/rowcodec" ) func TestT(t *testing.T) { @@ -35,166 +38,695 @@ var _ = Suite(&testSuite{}) type testSuite struct{} -func (s *testSuite) TestRowCodec(c *C) { - colIDs := []int64{1, 2, 3, 4, 5, 10} - rb := NewEncoder(colIDs, new(stmtctx.StatementContext)) - dt, err := types.ParseDatetime(rb.sc, "2018-01-19 03:14:07.999999") - c.Assert(err, IsNil) - datums := types.MakeDatums( - dt, - "abc", - nil, - 1, - types.NewDecFromInt(1), - "abc", - ) - newRow, err := rb.Encode(datums, nil) - c.Check(err, IsNil) - s.checkDecode(c, rb.sc, newRow) +type testData struct { + id int64 + ft *types.FieldType + dt types.Datum + bt types.Datum + def *types.Datum + handle bool +} - // Test large column ID - rb.tempColIDs = []int64{1, 2, 3, 4, 5, 512} - newRow, err = rb.Encode(datums, nil) - c.Check(err, IsNil) - s.checkDecode(c, rb.sc, newRow) +func (s *testSuite) TestDecodeRowWithHandle(c *C) { + handleID := int64(-1) + handleValue := int64(10000) - // Test large column value - rb.tempColIDs = []int64{1, 2, 3, 4, 5, 10} - datums[5] = types.NewBytesDatum(make([]byte, 65536)) - newRow, err = rb.Encode(datums, nil) - c.Check(err, IsNil) - s.checkDecode(c, rb.sc, newRow) + encodeAndDecodeHandle := func(c *C, testData []testData) { + // transform test data into input. + colIDs := make([]int64, 0, len(testData)) + dts := make([]types.Datum, 0, len(testData)) + fts := make([]*types.FieldType, 0, len(testData)) + cols := make([]rowcodec.ColInfo, 0, len(testData)) + for i := range testData { + t := testData[i] + if !t.handle { + colIDs = append(colIDs, t.id) + dts = append(dts, t.dt) + } + fts = append(fts, t.ft) + cols = append(cols, rowcodec.ColInfo{ + ID: t.id, + Tp: int32(t.ft.Tp), + Flag: int32(t.ft.Flag), + IsPKHandle: t.handle, + Flen: t.ft.Flen, + Decimal: t.ft.Decimal, + Elems: t.ft.Elems, + }) + } + + // test encode input. + var encoder rowcodec.Encoder + sc := new(stmtctx.StatementContext) + sc.TimeZone = time.UTC + newRow, err := encoder.Encode(sc, colIDs, dts, nil) + c.Assert(err, IsNil) + + // decode to datum map. + mDecoder := rowcodec.NewDatumMapDecoder(cols, -1, sc.TimeZone) + dm, err := mDecoder.DecodeToDatumMap(newRow, handleValue, nil) + c.Assert(err, IsNil) + for _, t := range testData { + d, exists := dm[t.id] + c.Assert(exists, IsTrue) + c.Assert(d, DeepEquals, t.dt) + } + + // decode to chunk. + cDecoder := rowcodec.NewChunkDecoder(cols, -1, nil, sc.TimeZone) + chk := chunk.New(fts, 1, 1) + err = cDecoder.DecodeToChunk(newRow, handleValue, chk) + c.Assert(err, IsNil) + chkRow := chk.GetRow(0) + cdt := chkRow.GetDatumRow(fts) + for i, t := range testData { + d := cdt[i] + if d.Kind() == types.KindMysqlDecimal { + c.Assert(d.GetMysqlDecimal(), DeepEquals, t.bt.GetMysqlDecimal()) + } else { + c.Assert(d, DeepEquals, t.bt) + } + } + + // decode to old row bytes. + colOffset := make(map[int64]int) + for i, t := range testData { + colOffset[t.id] = i + } + bDecoder := rowcodec.NewByteDecoder(cols, -1, nil, nil) + oldRow, err := bDecoder.DecodeToBytes(colOffset, handleValue, newRow, nil) + c.Assert(err, IsNil) + for i, t := range testData { + remain, d, err := codec.DecodeOne(oldRow[i]) + c.Assert(err, IsNil) + c.Assert(len(remain), Equals, 0) + if d.Kind() == types.KindMysqlDecimal { + c.Assert(d.GetMysqlDecimal(), DeepEquals, t.bt.GetMysqlDecimal()) + } else { + c.Assert(d, DeepEquals, t.bt) + } + } + } + + // encode & decode signed int. + testDataSigned := []testData{ + { + handleID, + types.NewFieldType(mysql.TypeLonglong), + types.NewIntDatum(handleValue), + types.NewIntDatum(handleValue), + nil, + true, + }, + { + 10, + types.NewFieldType(mysql.TypeLonglong), + types.NewIntDatum(1), + types.NewIntDatum(1), + nil, + false, + }, + } + encodeAndDecodeHandle(c, testDataSigned) + + // encode & decode unsigned int. + testDataUnsigned := []testData{ + { + handleID, + withUnsigned(types.NewFieldType(mysql.TypeLonglong)), + types.NewIntDatum(handleValue), // decode as chunk & map, always encode it as int + types.NewUintDatum(uint64(handleValue)), // decode as bytes will uint if unsigned. + nil, + true, + }, + { + 10, + types.NewFieldType(mysql.TypeLonglong), + types.NewIntDatum(1), + types.NewIntDatum(1), + nil, + false, + }, + } + encodeAndDecodeHandle(c, testDataUnsigned) +} + +func (s *testSuite) TestTypesNewRowCodec(c *C) { + getJSONDatum := func(value string) types.Datum { + j, err := json.ParseBinaryFromString(value) + c.Assert(err, IsNil) + var d types.Datum + d.SetMysqlJSON(j) + return d + } + getSetDatum := func(name string, value uint64) types.Datum { + var d types.Datum + d.SetMysqlSet(types.Set{Name: name, Value: value}) + return d + } + getTime := func(value string) types.Time { + t, err := types.ParseTime(&stmtctx.StatementContext{TimeZone: time.UTC}, value, mysql.TypeTimestamp, 6) + c.Assert(err, IsNil) + return t + } + + encodeAndDecode := func(c *C, testData []testData) { + // transform test data into input. + colIDs := make([]int64, 0, len(testData)) + dts := make([]types.Datum, 0, len(testData)) + fts := make([]*types.FieldType, 0, len(testData)) + cols := make([]rowcodec.ColInfo, 0, len(testData)) + for i := range testData { + t := testData[i] + colIDs = append(colIDs, t.id) + dts = append(dts, t.dt) + fts = append(fts, t.ft) + cols = append(cols, rowcodec.ColInfo{ + ID: t.id, + Tp: int32(t.ft.Tp), + Flag: int32(t.ft.Flag), + IsPKHandle: t.handle, + Flen: t.ft.Flen, + Decimal: t.ft.Decimal, + Elems: t.ft.Elems, + }) + } + + // test encode input. + var encoder rowcodec.Encoder + sc := new(stmtctx.StatementContext) + sc.TimeZone = time.UTC + newRow, err := encoder.Encode(sc, colIDs, dts, nil) + c.Assert(err, IsNil) + + // decode to datum map. + mDecoder := rowcodec.NewDatumMapDecoder(cols, -1, sc.TimeZone) + dm, err := mDecoder.DecodeToDatumMap(newRow, -1, nil) + c.Assert(err, IsNil) + for _, t := range testData { + d, exists := dm[t.id] + c.Assert(exists, IsTrue) + c.Assert(d, DeepEquals, t.dt) + } + + // decode to chunk. + cDecoder := rowcodec.NewChunkDecoder(cols, -1, nil, sc.TimeZone) + chk := chunk.New(fts, 1, 1) + err = cDecoder.DecodeToChunk(newRow, -1, chk) + c.Assert(err, IsNil) + chkRow := chk.GetRow(0) + cdt := chkRow.GetDatumRow(fts) + for i, t := range testData { + d := cdt[i] + if d.Kind() == types.KindMysqlDecimal { + c.Assert(d.GetMysqlDecimal(), DeepEquals, t.bt.GetMysqlDecimal()) + } else { + c.Assert(d, DeepEquals, t.dt) + } + } + + // decode to old row bytes. + colOffset := make(map[int64]int) + for i, t := range testData { + colOffset[t.id] = i + } + bDecoder := rowcodec.NewByteDecoder(cols, -1, nil, nil) + oldRow, err := bDecoder.DecodeToBytes(colOffset, -1, newRow, nil) + c.Assert(err, IsNil) + for i, t := range testData { + remain, d, err := codec.DecodeOne(oldRow[i]) + c.Assert(err, IsNil) + c.Assert(len(remain), Equals, 0) + if d.Kind() == types.KindMysqlDecimal { + c.Assert(d.GetMysqlDecimal(), DeepEquals, t.bt.GetMysqlDecimal()) + } else { + c.Assert(d, DeepEquals, t.bt) + } + } + } + + testData := []testData{ + { + 1, + types.NewFieldType(mysql.TypeLonglong), + types.NewIntDatum(1), + types.NewIntDatum(1), + nil, + false, + }, + { + 22, + withUnsigned(types.NewFieldType(mysql.TypeShort)), + types.NewUintDatum(1), + types.NewUintDatum(1), + nil, + false, + }, + { + 3, + types.NewFieldType(mysql.TypeDouble), + types.NewFloat64Datum(2), + types.NewFloat64Datum(2), + nil, + false, + }, + { + 24, + types.NewFieldType(mysql.TypeString), + types.NewBytesDatum([]byte("abc")), + types.NewBytesDatum([]byte("abc")), + nil, + false, + }, + { + 5, + withFsp(6)(types.NewFieldType(mysql.TypeTimestamp)), + types.NewTimeDatum(getTime("2011-11-10 11:11:11.999999")), + types.NewUintDatum(1840446893366133311), + nil, + false, + }, + { + 16, + withFsp(0)(types.NewFieldType(mysql.TypeDuration)), + types.NewDurationDatum(getDuration("4:00:00")), + types.NewIntDatum(14400000000000), + nil, + false, + }, + { + 8, + types.NewFieldType(mysql.TypeNewDecimal), + types.NewDecimalDatum(types.NewDecFromStringForTest("1.99")), + types.NewDecimalDatum(types.NewDecFromStringForTest("1.99")), + nil, + false, + }, + { + 12, + types.NewFieldType(mysql.TypeYear), + types.NewIntDatum(1999), + types.NewIntDatum(1999), + nil, + false, + }, + { + 9, + withEnumElems("y", "n")(types.NewFieldType(mysql.TypeEnum)), + types.NewMysqlEnumDatum(types.Enum{Name: "n", Value: 2}), + types.NewUintDatum(2), + nil, + false, + }, + { + 14, + types.NewFieldType(mysql.TypeJSON), + getJSONDatum(`{"a":2}`), + getJSONDatum(`{"a":2}`), + nil, + false, + }, + { + 11, + types.NewFieldType(mysql.TypeNull), + types.NewDatum(nil), + types.NewDatum(nil), + nil, + false, + }, + { + 2, + types.NewFieldType(mysql.TypeNull), + types.NewDatum(nil), + types.NewDatum(nil), + nil, + false, + }, + { + 100, + types.NewFieldType(mysql.TypeNull), + types.NewDatum(nil), + types.NewDatum(nil), + nil, + false, + }, + { + 116, + types.NewFieldType(mysql.TypeFloat), + types.NewFloat32Datum(6), + types.NewFloat64Datum(6), + nil, + false, + }, + { + 117, + withEnumElems("n1", "n2")(types.NewFieldType(mysql.TypeSet)), + getSetDatum("n1", 1), + types.NewUintDatum(1), + nil, + false, + }, + { + 118, + withFlen(24)(types.NewFieldType(mysql.TypeBit)), // 3 bit + types.NewMysqlBitDatum(types.NewBinaryLiteralFromUint(3223600, 3)), + types.NewUintDatum(3223600), + nil, + false, + }, + { + 119, + types.NewFieldType(mysql.TypeVarString), + types.NewBytesDatum([]byte("")), + types.NewBytesDatum([]byte("")), + nil, + false, + }, + } + + // test small + encodeAndDecode(c, testData) + + // test large colID + testData[0].id = 300 + encodeAndDecode(c, testData) + testData[0].id = 1 + + // test large data + testData[3].dt = types.NewBytesDatum([]byte(strings.Repeat("a", math.MaxUint16+1))) + testData[3].bt = types.NewBytesDatum([]byte(strings.Repeat("a", math.MaxUint16+1))) + encodeAndDecode(c, testData) +} + +func (s *testSuite) TestNilAndDefault(c *C) { + encodeAndDecode := func(c *C, testData []testData) { + // transform test data into input. + colIDs := make([]int64, 0, len(testData)) + dts := make([]types.Datum, 0, len(testData)) + cols := make([]rowcodec.ColInfo, 0, len(testData)) + fts := make([]*types.FieldType, 0, len(testData)) + for i := range testData { + t := testData[i] + if t.def == nil { + colIDs = append(colIDs, t.id) + dts = append(dts, t.dt) + } + fts = append(fts, t.ft) + cols = append(cols, rowcodec.ColInfo{ + ID: t.id, + Tp: int32(t.ft.Tp), + Flag: int32(t.ft.Flag), + IsPKHandle: t.handle, + Flen: t.ft.Flen, + Decimal: t.ft.Decimal, + Elems: t.ft.Elems, + }) + } + ddf := func(i int) (types.Datum, error) { + t := testData[i] + if t.def == nil { + var d types.Datum + d.SetNull() + return d, nil + } + return *t.def, nil + } + bdf := func(i int) ([]byte, error) { + t := testData[i] + if t.def == nil { + return nil, nil + } + return getOldDatumByte(*t.def), nil + } + // test encode input. + var encoder rowcodec.Encoder + sc := new(stmtctx.StatementContext) + sc.TimeZone = time.UTC + newRow, err := encoder.Encode(sc, colIDs, dts, nil) + c.Assert(err, IsNil) + + // decode to datum map. + mDecoder := rowcodec.NewDatumMapDecoder(cols, -1, sc.TimeZone) + dm, err := mDecoder.DecodeToDatumMap(newRow, -1, nil) + c.Assert(err, IsNil) + for _, t := range testData { + d, exists := dm[t.id] + if t.def != nil { + // for datum should not fill default value. + c.Assert(exists, IsFalse) + } else { + c.Assert(exists, IsTrue) + c.Assert(d, DeepEquals, t.bt) + } + } + + //decode to chunk. + chk := chunk.New(fts, 1, 1) + cDecoder := rowcodec.NewChunkDecoder(cols, -1, ddf, sc.TimeZone) + err = cDecoder.DecodeToChunk(newRow, -1, chk) + c.Assert(err, IsNil) + chkRow := chk.GetRow(0) + cdt := chkRow.GetDatumRow(fts) + for i, t := range testData { + d := cdt[i] + if d.Kind() == types.KindMysqlDecimal { + c.Assert(d.GetMysqlDecimal(), DeepEquals, t.bt.GetMysqlDecimal()) + } else { + c.Assert(d, DeepEquals, t.bt) + } + } + + // decode to old row bytes. + colOffset := make(map[int64]int) + for i, t := range testData { + colOffset[t.id] = i + } + bDecoder := rowcodec.NewByteDecoder(cols, -1, bdf, sc.TimeZone) + oldRow, err := bDecoder.DecodeToBytes(colOffset, -1, newRow, nil) + c.Assert(err, IsNil) + for i, t := range testData { + remain, d, err := codec.DecodeOne(oldRow[i]) + c.Assert(err, IsNil) + c.Assert(len(remain), Equals, 0) + if d.Kind() == types.KindMysqlDecimal { + c.Assert(d.GetMysqlDecimal(), DeepEquals, t.bt.GetMysqlDecimal()) + } else { + c.Assert(d, DeepEquals, t.bt) + } + } + } + dtNilData := []testData{ + { + 1, + types.NewFieldType(mysql.TypeLonglong), + types.NewIntDatum(1), + types.NewIntDatum(1), + nil, + false, + }, + { + 2, + withUnsigned(types.NewFieldType(mysql.TypeLonglong)), + types.NewUintDatum(1), + types.NewUintDatum(9), + getDatumPoint(types.NewUintDatum(9)), + false, + }, + } + encodeAndDecode(c, dtNilData) } -func (s *testSuite) TestIntCodec(c *C) { - uints := []uint64{255, math.MaxUint16, math.MaxUint32, math.MaxUint32 + 1} - sizes := []int{1, 2, 4, 8} - for i, v := range uints { - data := encodeUint(nil, v) - c.Assert(len(data), Equals, sizes[i]) - c.Assert(decodeUint(data), Equals, v) +func (s *testSuite) TestVarintCompatibility(c *C) { + encodeAndDecodeByte := func(c *C, testData []testData) { + // transform test data into input. + colIDs := make([]int64, 0, len(testData)) + dts := make([]types.Datum, 0, len(testData)) + fts := make([]*types.FieldType, 0, len(testData)) + cols := make([]rowcodec.ColInfo, 0, len(testData)) + for i := range testData { + t := testData[i] + colIDs = append(colIDs, t.id) + dts = append(dts, t.dt) + fts = append(fts, t.ft) + cols = append(cols, rowcodec.ColInfo{ + ID: t.id, + Tp: int32(t.ft.Tp), + Flag: int32(t.ft.Flag), + IsPKHandle: t.handle, + Flen: t.ft.Flen, + Decimal: t.ft.Decimal, + Elems: t.ft.Elems, + }) + } + + // test encode input. + var encoder rowcodec.Encoder + sc := new(stmtctx.StatementContext) + sc.TimeZone = time.UTC + newRow, err := encoder.Encode(sc, colIDs, dts, nil) + c.Assert(err, IsNil) + decoder := rowcodec.NewByteDecoder(cols, -1, nil, sc.TimeZone) + // decode to old row bytes. + colOffset := make(map[int64]int) + for i, t := range testData { + colOffset[t.id] = i + } + oldRow, err := decoder.DecodeToBytes(colOffset, 1, newRow, nil) + c.Assert(err, IsNil) + for i, t := range testData { + oldVarint, err := tablecodec.EncodeValue(nil, nil, t.bt) // tablecodec will encode as varint/varuint + c.Assert(err, IsNil) + c.Assert(oldVarint, DeepEquals, oldRow[i]) + } } - ints := []int64{127, math.MaxInt16, math.MaxInt32, math.MaxInt32 + 1} - for i, v := range ints { - data := encodeInt(nil, v) - c.Assert(len(data), Equals, sizes[i]) - c.Assert(decodeInt(data), Equals, v) + testDataValue := []testData{ + { + 1, + types.NewFieldType(mysql.TypeLonglong), + types.NewIntDatum(1), + types.NewIntDatum(1), + nil, + false, + }, + { + 2, + withUnsigned(types.NewFieldType(mysql.TypeLonglong)), + types.NewUintDatum(1), + types.NewUintDatum(1), + nil, + false, + }, } + encodeAndDecodeByte(c, testDataValue) } -func (s *testSuite) TestMoreTypes(c *C) { - colIDs := []int64{1, 2, 3, 4, 5, 6, 7, 8} +func (s *testSuite) TestCodecUtil(c *C) { + colIDs := []int64{1, 2, 3, 4} + tps := make([]*types.FieldType, 4) + for i := 0; i < 3; i++ { + tps[i] = types.NewFieldType(mysql.TypeLonglong) + } + tps[3] = types.NewFieldType(mysql.TypeNull) sc := new(stmtctx.StatementContext) - sc.TimeZone = time.Local - rb := NewEncoder(colIDs, sc) - ts, err := types.ParseTimestampFromNum(rb.sc, 20181111090909) - c.Assert(err, IsNil) - datums := types.MakeDatums( - float32(1.0), - float64(1.0), - ts, - types.Duration{Duration: time.Minute}, - types.Enum{Name: "a", Value: 1}, - types.Set{Name: "a", Value: 1}, - json.CreateBinary("abc"), - types.BitLiteral([]byte{101}), - ) - newRow, err := rb.Encode(datums, nil) + oldRow, err := tablecodec.EncodeRow(sc, types.MakeDatums(1, 2, 3, nil), colIDs, nil, nil) c.Check(err, IsNil) - fieldTypes := []*types.FieldType{ - types.NewFieldType(mysql.TypeFloat), - types.NewFieldType(mysql.TypeDouble), - {Tp: mysql.TypeTimestamp, Decimal: 0}, - types.NewFieldType(mysql.TypeDuration), - {Tp: mysql.TypeEnum, Elems: []string{"a"}}, - {Tp: mysql.TypeSet, Elems: []string{"a"}}, - types.NewFieldType(mysql.TypeJSON), - {Tp: mysql.TypeBit, Flen: 8}, - } - rd, err := NewDecoder(colIDs, 0, fieldTypes, make([][]byte, 8), rb.sc) - c.Assert(err, IsNil) - chk := chunk.NewChunkWithCapacity(fieldTypes, 1) - err = rd.Decode(newRow, 0, chk) + var ( + rb rowcodec.Encoder + newRow []byte + ) + newRow, err = rowcodec.EncodeFromOldRow(&rb, nil, oldRow, nil) c.Assert(err, IsNil) - row := chk.GetRow(0) - c.Assert(row.GetFloat32(0), Equals, float32(1.0)) - c.Assert(row.GetFloat64(1), Equals, float64(1.0)) - c.Assert(row.GetTime(2).String(), Equals, ts.String()) - c.Assert(row.GetDuration(3, 0), Equals, datums[3].GetMysqlDuration()) - c.Assert(row.GetEnum(4), Equals, datums[4].GetMysqlEnum()) - c.Assert(row.GetSet(5), Equals, datums[5].GetMysqlSet()) - c.Assert(row.GetJSON(6), DeepEquals, datums[6].GetMysqlJSON()) - c.Assert(row.GetBytes(7), DeepEquals, []byte(datums[7].GetMysqlBit())) -} + c.Assert(rowcodec.IsNewFormat(newRow), IsTrue) + c.Assert(rowcodec.IsNewFormat(oldRow), IsFalse) -func (s *testSuite) checkDecode(c *C, sc *stmtctx.StatementContext, newRow []byte) { - readRowTypes := []*types.FieldType{ - types.NewFieldType(mysql.TypeVarString), - types.NewFieldType(mysql.TypeLonglong), - types.NewFieldType(mysql.TypeLonglong), - types.NewFieldType(mysql.TypeNewDecimal), - types.NewFieldType(mysql.TypeLonglong), - types.NewFieldType(mysql.TypeLonglong), - } - readColIDS := []int64{2, 3, 4, 5, 6, 7} - defaultColVal, err := codec.EncodeValue(sc, nil, types.NewIntDatum(5)) - c.Assert(err, IsNil) - defaults := [][]byte{nil, defaultColVal, defaultColVal, nil, defaultColVal, nil} + // test stringer for decoder. + var cols []rowcodec.ColInfo + for i, ft := range tps { + cols = append(cols, rowcodec.ColInfo{ + ID: colIDs[i], + Tp: int32(ft.Tp), + Flag: int32(ft.Flag), + IsPKHandle: false, + Flen: ft.Flen, + Decimal: ft.Decimal, + Elems: ft.Elems, + }) + } + d := rowcodec.NewDecoder(cols, -1, nil) - rd, err := NewDecoder(readColIDS, 7, readRowTypes, defaults, sc) + // test ColumnIsNull + isNil, err := d.ColumnIsNull(newRow, 4, nil) c.Assert(err, IsNil) - chk := chunk.NewChunkWithCapacity(readRowTypes, 1) - err = rd.Decode(newRow, 1000, chk) + c.Assert(isNil, IsTrue) + isNil, err = d.ColumnIsNull(newRow, 1, nil) c.Assert(err, IsNil) - row := chk.GetRow(0) - c.Assert(row.GetString(0), Equals, "abc") - c.Assert(row.IsNull(1), IsTrue) - c.Assert(row.GetInt64(2), Equals, int64(1)) - c.Assert(row.GetMyDecimal(3).String(), Equals, "1") - c.Assert(row.GetInt64(4), Equals, int64(5)) - c.Assert(row.GetInt64(5), Equals, int64(1000)) + c.Assert(isNil, IsFalse) + isNil, err = d.ColumnIsNull(newRow, 5, nil) + c.Assert(err, IsNil) + c.Assert(isNil, IsTrue) + isNil, err = d.ColumnIsNull(newRow, 5, []byte{1}) + c.Assert(err, IsNil) + c.Assert(isNil, IsFalse) + + // test isRowKey + c.Assert(rowcodec.IsRowKey([]byte{'b', 't'}), IsFalse) + c.Assert(rowcodec.IsRowKey([]byte{'t', 'r'}), IsFalse) } -func BenchmarkEncode(b *testing.B) { - b.ReportAllocs() - oldRow := types.MakeDatums(1, "abc", 1.1) - xb := NewEncoder([]int64{1, 2, 3}, new(stmtctx.StatementContext)) - var buf []byte - var err error - for i := 0; i < b.N; i++ { - buf = buf[:0] - buf, err = xb.Encode(oldRow, buf) - if err != nil { - b.Fatal(err) +func (s *testSuite) TestOldRowCodec(c *C) { + colIDs := []int64{1, 2, 3, 4} + tps := make([]*types.FieldType, 4) + for i := 0; i < 3; i++ { + tps[i] = types.NewFieldType(mysql.TypeLonglong) + } + tps[3] = types.NewFieldType(mysql.TypeNull) + sc := new(stmtctx.StatementContext) + oldRow, err := tablecodec.EncodeRow(sc, types.MakeDatums(1, 2, 3, nil), colIDs, nil, nil) + c.Check(err, IsNil) + + var ( + rb rowcodec.Encoder + newRow []byte + ) + newRow, err = rowcodec.EncodeFromOldRow(&rb, nil, oldRow, nil) + c.Check(err, IsNil) + cols := make([]rowcodec.ColInfo, len(tps)) + for i, tp := range tps { + cols[i] = rowcodec.ColInfo{ + ID: colIDs[i], + Tp: int32(tp.Tp), + Flag: int32(tp.Flag), + Flen: tp.Flen, + Decimal: tp.Decimal, + Elems: tp.Elems, } } + rd := rowcodec.NewChunkDecoder(cols, 0, nil, time.Local) + chk := chunk.NewChunkWithCapacity(tps, 1) + err = rd.DecodeToChunk(newRow, -1, chk) + c.Assert(err, IsNil) + row := chk.GetRow(0) + for i := 0; i < 3; i++ { + c.Assert(row.GetInt64(i), Equals, int64(i)+1) + } } -func BenchmarkDecode(b *testing.B) { - b.ReportAllocs() - oldRow := types.MakeDatums(1, "abc", 1.1) - colIDs := []int64{-1, 2, 3} - tps := []*types.FieldType{ - types.NewFieldType(mysql.TypeLonglong), - types.NewFieldType(mysql.TypeString), - types.NewFieldType(mysql.TypeDouble), +var ( + withUnsigned = func(ft *types.FieldType) *types.FieldType { + ft.Flag = ft.Flag | mysql.UnsignedFlag + return ft } - xb := NewEncoder(colIDs, new(stmtctx.StatementContext)) - xRowData, err := xb.Encode(oldRow, nil) - if err != nil { - b.Fatal(err) + withEnumElems = func(elem ...string) func(ft *types.FieldType) *types.FieldType { + return func(ft *types.FieldType) *types.FieldType { + ft.Elems = elem + return ft + } } - decoder, err := NewDecoder(colIDs, -1, tps, make([][]byte, 3), xb.sc) - if err != nil { - b.Fatal(err) + withFsp = func(fsp int) func(ft *types.FieldType) *types.FieldType { + return func(ft *types.FieldType) *types.FieldType { + ft.Decimal = fsp + return ft + } } - chk := chunk.NewChunkWithCapacity(tps, 1) - for i := 0; i < b.N; i++ { - chk.Reset() - err = decoder.Decode(xRowData, 1, chk) + withFlen = func(flen int) func(ft *types.FieldType) *types.FieldType { + return func(ft *types.FieldType) *types.FieldType { + ft.Flen = flen + return ft + } + } + getDuration = func(value string) types.Duration { + dur, _ := types.ParseDuration(nil, value, 0) + return dur + } + getOldDatumByte = func(d types.Datum) []byte { + b, err := tablecodec.EncodeValue(nil, nil, d) if err != nil { - b.Fatal(err) + panic(err) } + return b } -} + getDatumPoint = func(d types.Datum) *types.Datum { + return &d + } +)