Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

distsql: Unify endian for Chunk encode format #13349

Merged
merged 18 commits into from
Nov 27, 2019
Merged

distsql: Unify endian for Chunk encode format #13349

Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
29765c2
tidb endian part 1
wshwsh12 Nov 11, 2019
839d811
mock tikv
wshwsh12 Nov 11, 2019
185510b
fix ci
wshwsh12 Nov 11, 2019
3b48057
update tipb
wshwsh12 Nov 11, 2019
41d9580
Merge remote-tracking branch 'upstream/master' into endian
wshwsh12 Nov 12, 2019
4190bc3
update tipb
wshwsh12 Nov 12, 2019
badacea
Merge remote-tracking branch 'upstream/master' into endian
wshwsh12 Nov 12, 2019
c6ab9c4
header
wshwsh12 Nov 12, 2019
f25f41e
Merge remote-tracking branch 'upstream/master' into endian
wshwsh12 Nov 18, 2019
45993a0
check align
wshwsh12 Nov 18, 2019
4206ec3
Merge remote-tracking branch 'upstream/master' into endian
wshwsh12 Nov 26, 2019
1bd76b2
update mod
wshwsh12 Nov 26, 2019
fa32a5d
address comments
wshwsh12 Nov 26, 2019
7ced633
address comments
wshwsh12 Nov 26, 2019
441a4fa
address comments
wshwsh12 Nov 26, 2019
e3b71d6
update tipb
wshwsh12 Nov 27, 2019
1157992
update go mod
wshwsh12 Nov 27, 2019
4f561ce
Merge branch 'master' into endian
sre-bot Nov 27, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions distsql/distsql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package distsql
import (
"context"
"fmt"
"unsafe"

"github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
Expand Down Expand Up @@ -146,6 +147,7 @@ func Checksum(ctx context.Context, client kv.Client, kvReq *kv.Request, vars *kv
func SetEncodeType(ctx sessionctx.Context, dagReq *tipb.DAGRequest) {
if canUseChunkRPC(ctx) {
dagReq.EncodeType = tipb.EncodeType_TypeChunk
setSystemEndian(dagReq)
} else {
dagReq.EncodeType = tipb.EncodeType_TypeDefault
}
Expand All @@ -158,5 +160,41 @@ func canUseChunkRPC(ctx sessionctx.Context) bool {
if ctx.GetSessionVars().EnableStreaming {
return false
}
if !checkAlignment() {
return false
}
return true
}

var supportedAlignment = !(unsafe.Sizeof(types.MysqlTime{}) != 16 ||
unsafe.Sizeof(types.Time{}) != 20 ||
unsafe.Sizeof(types.MyDecimal{}) != 40)

// checkAlignment checks the alignment in current system environment.
// The alignment is influenced by system, machine and Golang version.
// Using this function can guarantee the alignment is we want.
func checkAlignment() bool {
return supportedAlignment
}

var systemEndian tipb.Endian

// setSystemEndian sets the system endian for the DAGRequest.
func setSystemEndian(dagReq *tipb.DAGRequest) {
dagReq.TidbSystemEndian = GetSystemEndian()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We may check this when process starting and cache the result, instead of checking in each session.

}

// GetSystemEndian gets the system endian.
func GetSystemEndian() tipb.Endian {
return systemEndian
}

func init() {
var i int = 0x0100
ptr := unsafe.Pointer(&i)
if 0x01 == *(*byte)(ptr) {
systemEndian = tipb.Endian_BigEndian
} else {
systemEndian = tipb.Endian_LittleEndian
}
}
8 changes: 4 additions & 4 deletions distsql/request_builder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ func (s *testSuite) TestRequestBuilder1(c *C) {
expect := &kv.Request{
Tp: 103,
StartTs: 0x0,
Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0},
SunRunAway marked this conversation as resolved.
Show resolved Hide resolved
Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0, 0x70, 0x0},
KeyRanges: []kv.KeyRange{
{
StartKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},
Expand Down Expand Up @@ -360,7 +360,7 @@ func (s *testSuite) TestRequestBuilder2(c *C) {
expect := &kv.Request{
Tp: 103,
StartTs: 0x0,
Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0},
Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0, 0x70, 0x0},
KeyRanges: []kv.KeyRange{
{
StartKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},
Expand Down Expand Up @@ -409,7 +409,7 @@ func (s *testSuite) TestRequestBuilder3(c *C) {
expect := &kv.Request{
Tp: 103,
StartTs: 0x0,
Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0},
Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0, 0x70, 0x0},
KeyRanges: []kv.KeyRange{
{
StartKey: kv.Key{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
Expand Down Expand Up @@ -472,7 +472,7 @@ func (s *testSuite) TestRequestBuilder4(c *C) {
expect := &kv.Request{
Tp: 103,
StartTs: 0x0,
Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0},
Data: []uint8{0x8, 0x0, 0x18, 0x0, 0x20, 0x0, 0x40, 0x0, 0x5a, 0x0, 0x70, 0x0},
KeyRanges: keyRanges,
KeepOrder: false,
Desc: false,
Expand Down
2 changes: 2 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,5 @@ require (
go 1.13

replace github.com/pingcap/check => github.com/tiancaiamao/check v0.0.0-20191119042138-8e73d07b629d

replace github.com/pingcap/tipb => github.com/wshwsh12/tipb v0.0.0-20191126022620-fce96f6e0e24
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,6 @@ github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 h1:GIEq+wZfrl2bc
github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0/go.mod h1:G/6rJpnYwM0LKMec2rI82/5Kg6GaZMvlfB+e6/tvYmI=
github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible h1:H1jg0aDWz2SLRh3hNBo2HFtnuHtudIUvBumU7syRkic=
github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
github.com/pingcap/tipb v0.0.0-20191120020146-6161b015e21e h1:OWgXsJ2Zwa+q+sqi87fGuNda+ChJrclVd7wiGP5Epps=
github.com/pingcap/tipb v0.0.0-20191120020146-6161b015e21e/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down Expand Up @@ -264,6 +262,8 @@ github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d h1:ggUgChAeyge4NZ4
github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
github.com/wshwsh12/tipb v0.0.0-20191126022620-fce96f6e0e24 h1:GHqjILkzVElXbYgItjPVDsHVyLPhcqla71FFVE/iQfE=
github.com/wshwsh12/tipb v0.0.0-20191126022620-fce96f6e0e24/go.mod h1:UFKGXY+Ij4rifkYSXsZOrEQcV70dCog1faHm7QKJiWs=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE=
Expand Down
4 changes: 4 additions & 0 deletions store/mockstore/mocktikv/cop_handler_dag.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/mysql"
"github.com/pingcap/parser/terror"
"github.com/pingcap/tidb/distsql"
"github.com/pingcap/tidb/expression"
"github.com/pingcap/tidb/expression/aggregation"
"github.com/pingcap/tidb/kv"
Expand Down Expand Up @@ -574,6 +575,9 @@ func (h *rpcHandler) fillUpData4SelectResponse(selResp *tipb.SelectResponse, dag
case tipb.EncodeType_TypeDefault:
h.encodeDefault(selResp, rows, dagReq.OutputOffsets)
case tipb.EncodeType_TypeChunk:
if dagReq.TidbSystemEndian != distsql.GetSystemEndian() {
return errors.Errorf("Mocktikv endian is must be same as TiDB system endian.")
wshwsh12 marked this conversation as resolved.
Show resolved Hide resolved
}
colTypes := h.constructRespSchema(dagCtx)
loc := dagCtx.evalCtx.sc.TimeZone
err := h.encodeChunk(selResp, rows, colTypes, dagReq.OutputOffsets, loc)
Expand Down