Skip to content

Commit

Permalink
Add msgpack handler for smarter JSON encoding
Browse files Browse the repository at this point in the history
The msgpack handler correctly retains information about whether the
number is a float or an integer, unlike JSON. While the format is not
human-readable, it makes a good interchange format for applications that
don't necessarily care about a human readable output.

This uses `github.com/tinylib/msgp` to precompile the marshaling for
`models.Row`. Since we only use this library to marshal the one type,
this is a much more efficient method of encoding than using reflection.
  • Loading branch information
jsternberg committed May 30, 2017
1 parent 8432c2b commit 0ef3334
Show file tree
Hide file tree
Showing 8 changed files with 562 additions and 14 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ The admin UI is removed and unusable in this release. The `[admin]` configuratio
- [#8390](https://github.com/influxdata/influxdb/issues/8390): Add nanosecond duration literal support.
- [#8394](https://github.com/influxdata/influxdb/pull/8394): Optimize top() and bottom() using an incremental aggregator.
- [#7129](https://github.com/influxdata/influxdb/issues/7129): Maintain the tags of points selected by top() or bottom() when writing the results.
- [#7154](https://github.com/influxdata/influxdb/pull/7154): Add msgpack handler for smart JSON encoding.

### Bugfixes

Expand Down
2 changes: 2 additions & 0 deletions Godeps
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@ github.com/influxdata/usage-client 6d3895376368aa52a3a81d2a16e90f0f52371967
github.com/jwilder/encoding 27894731927e49b0a9023f00312be26733744815
github.com/paulbellamy/ratecounter 5a11f585a31379765c190c033b6ad39956584447
github.com/peterh/liner 88609521dc4b6c858fd4c98b628147da928ce4ac
github.com/philhofer/fwd 98c11a7a6ec829d672b03833c3d69a7fae1ca972
github.com/retailnext/hllpp 38a7bb71b483e855d35010808143beaf05b67f9d
github.com/spaolacci/murmur3 0d12bf811670bf6a1a63828dfbd003eded177fce
github.com/tinylib/msgp ad0ff2e232ad2e37faf67087fb24bf8d04a8ce20
github.com/uber-go/atomic 74ca5ec650841aee9f289dce76e928313a37cbc6
github.com/uber-go/zap fbae0281ffd546fa6d1959fec6075ac5da7fb577
golang.org/x/crypto 9477e0b78b9ac3d0b03822fd95422e2fe07627cd
2 changes: 2 additions & 0 deletions LICENSE_OF_DEPENDENCIES.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,12 @@
- github.com/jwilder/encoding [MIT LICENSE](https://github.com/jwilder/encoding/blob/master/LICENSE)
- github.com/paulbellamy/ratecounter [MIT LICENSE](https://github.com/paulbellamy/ratecounter/blob/master/LICENSE)
- github.com/peterh/liner [MIT LICENSE](https://github.com/peterh/liner/blob/master/COPYING)
- github.com/philhofer/fwd [MIT LICENSE](https://github.com/philhofer/fwd/blob/master/LICENSE.md)
- github.com/rakyll/statik [APACHE LICENSE](https://github.com/rakyll/statik/blob/master/LICENSE)
- github.com/retailnext/hllpp [BSD LICENSE](https://github.com/retailnext/hllpp/blob/master/LICENSE)
- github.com/uber-go/atomic [MIT LICENSE](https://github.com/uber-go/atomic/blob/master/LICENSE.txt)
- github.com/uber-go/zap [MIT LICENSE](https://github.com/uber-go/zap/blob/master/LICENSE.txt)
- github.com/tinylib/msgp [BSD LICENSE](https://github.com/tinylib/msgp/blob/master/LICENSE)
- glyphicons [LICENSE](http://glyphicons.com/license/)
- golang.org/x/crypto [BSD LICENSE](https://github.com/golang/crypto/blob/master/LICENSE)
- jquery 2.1.4 [MIT LICENSE](https://github.com/jquery/jquery/blob/master/LICENSE.txt)
Expand Down
286 changes: 286 additions & 0 deletions models/encode.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,286 @@
package models

// NOTE: THIS FILE WAS PRODUCED BY THE
// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
// DO NOT EDIT

import "github.com/tinylib/msgp/msgp"

// DecodeMsg implements msgp.Decodable
func (z *Row) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
var zwht uint32
zwht, err = dc.ReadMapHeader()
if err != nil {
return
}
for zwht > 0 {
zwht--
field, err = dc.ReadMapKeyPtr()
if err != nil {
return
}
switch msgp.UnsafeString(field) {
case "name":
z.Name, err = dc.ReadString()
if err != nil {
return
}
case "tags":
var zhct uint32
zhct, err = dc.ReadMapHeader()
if err != nil {
return
}
if z.Tags == nil && zhct > 0 {
z.Tags = make(map[string]string, zhct)
} else if len(z.Tags) > 0 {
for key, _ := range z.Tags {
delete(z.Tags, key)
}
}
for zhct > 0 {
zhct--
var zxvk string
var zbzg string
zxvk, err = dc.ReadString()
if err != nil {
return
}
zbzg, err = dc.ReadString()
if err != nil {
return
}
z.Tags[zxvk] = zbzg
}
case "columns":
var zcua uint32
zcua, err = dc.ReadArrayHeader()
if err != nil {
return
}
if cap(z.Columns) >= int(zcua) {
z.Columns = (z.Columns)[:zcua]
} else {
z.Columns = make([]string, zcua)
}
for zbai := range z.Columns {
z.Columns[zbai], err = dc.ReadString()
if err != nil {
return
}
}
case "values":
var zxhx uint32
zxhx, err = dc.ReadArrayHeader()
if err != nil {
return
}
if cap(z.Values) >= int(zxhx) {
z.Values = (z.Values)[:zxhx]
} else {
z.Values = make([][]interface{}, zxhx)
}
for zcmr := range z.Values {
var zlqf uint32
zlqf, err = dc.ReadArrayHeader()
if err != nil {
return
}
if cap(z.Values[zcmr]) >= int(zlqf) {
z.Values[zcmr] = (z.Values[zcmr])[:zlqf]
} else {
z.Values[zcmr] = make([]interface{}, zlqf)
}
for zajw := range z.Values[zcmr] {
z.Values[zcmr][zajw], err = dc.ReadIntf()
if err != nil {
return
}
}
}
case "partial":
z.Partial, err = dc.ReadBool()
if err != nil {
return
}
default:
err = dc.Skip()
if err != nil {
return
}
}
}
return
}

// EncodeMsg implements msgp.Encodable
func (z *Row) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 5
// write "name"
err = en.Append(0x85, 0xa4, 0x6e, 0x61, 0x6d, 0x65)
if err != nil {
return err
}
err = en.WriteString(z.Name)
if err != nil {
return
}
// write "tags"
err = en.Append(0xa4, 0x74, 0x61, 0x67, 0x73)
if err != nil {
return err
}
err = en.WriteMapHeader(uint32(len(z.Tags)))
if err != nil {
return
}
for zxvk, zbzg := range z.Tags {
err = en.WriteString(zxvk)
if err != nil {
return
}
err = en.WriteString(zbzg)
if err != nil {
return
}
}
// write "columns"
err = en.Append(0xa7, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73)
if err != nil {
return err
}
err = en.WriteArrayHeader(uint32(len(z.Columns)))
if err != nil {
return
}
for zbai := range z.Columns {
err = en.WriteString(z.Columns[zbai])
if err != nil {
return
}
}
// write "values"
err = en.Append(0xa6, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73)
if err != nil {
return err
}
err = en.WriteArrayHeader(uint32(len(z.Values)))
if err != nil {
return
}
for zcmr := range z.Values {
err = en.WriteArrayHeader(uint32(len(z.Values[zcmr])))
if err != nil {
return
}
for zajw := range z.Values[zcmr] {
err = en.WriteIntf(z.Values[zcmr][zajw])
if err != nil {
return
}
}
}
// write "partial"
err = en.Append(0xa7, 0x70, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c)
if err != nil {
return err
}
err = en.WriteBool(z.Partial)
if err != nil {
return
}
return
}

// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *Row) Msgsize() (s int) {
s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 5 + msgp.MapHeaderSize
if z.Tags != nil {
for zxvk, zbzg := range z.Tags {
_ = zbzg
s += msgp.StringPrefixSize + len(zxvk) + msgp.StringPrefixSize + len(zbzg)
}
}
s += 8 + msgp.ArrayHeaderSize
for zbai := range z.Columns {
s += msgp.StringPrefixSize + len(z.Columns[zbai])
}
s += 7 + msgp.ArrayHeaderSize
for zcmr := range z.Values {
s += msgp.ArrayHeaderSize
for zajw := range z.Values[zcmr] {
s += msgp.GuessSize(z.Values[zcmr][zajw])
}
}
s += 8 + msgp.BoolSize
return
}

// DecodeMsg implements msgp.Decodable
func (z *Rows) DecodeMsg(dc *msgp.Reader) (err error) {
var zjfb uint32
zjfb, err = dc.ReadArrayHeader()
if err != nil {
return
}
if cap((*z)) >= int(zjfb) {
(*z) = (*z)[:zjfb]
} else {
(*z) = make(Rows, zjfb)
}
for zpks := range *z {
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
return
}
(*z)[zpks] = nil
} else {
if (*z)[zpks] == nil {
(*z)[zpks] = new(Row)
}
err = (*z)[zpks].DecodeMsg(dc)
if err != nil {
return
}
}
}
return
}

// EncodeMsg implements msgp.Encodable
func (z Rows) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteArrayHeader(uint32(len(z)))
if err != nil {
return
}
for zcxo := range z {
if z[zcxo] == nil {
err = en.WriteNil()
if err != nil {
return
}
} else {
err = z[zcxo].EncodeMsg(en)
if err != nil {
return
}
}
}
return
}

// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z Rows) Msgsize() (s int) {
s = msgp.ArrayHeaderSize
for zcxo := range z {
if z[zcxo] == nil {
s += msgp.NilSize
} else {
s += z[zcxo].Msgsize()
}
}
return
}
Loading

0 comments on commit 0ef3334

Please sign in to comment.