Skip to content

Commit

Permalink
Add msgpack handler for smarter JSON encoding
Browse files Browse the repository at this point in the history
The msgpack handler correctly retains information about whether the
number is a float or an integer, unlike JSON. While the format is not
human-readable, it makes a good interchange format for applications that
don't necessarily care about a human readable output.

This uses `github.com/tinylib/msgp` to precompile the marshaling for
`models.Row`. Since we only use this library to marshal the one type,
this is a much more efficient method of encoding than using reflection.
  • Loading branch information
jsternberg committed Nov 29, 2016
1 parent e0c1908 commit f4d55fd
Show file tree
Hide file tree
Showing 8 changed files with 567 additions and 20 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
- [#7066](https://github.com/influxdata/influxdb/issues/7066): Add support for secure transmission via collectd.
- [#7554](https://github.com/influxdata/influxdb/pull/7554): update latest dependencies with Godeps.
- [#7368](https://github.com/influxdata/influxdb/pull/7368): Introduce syntax for marking a partial response with chunking.
- [#7154](https://github.com/influxdata/influxdb/pull/7154): Add msgpack handler for smart JSON encoding.

### Bugfixes

Expand Down Expand Up @@ -88,6 +89,9 @@ All Changes:
### Bugfixes

- [#7392](https://github.com/influxdata/influxdb/pull/7392): Enable https subscriptions to work with custom CA certificates.

### Bugfixes

- [#1834](https://github.com/influxdata/influxdb/issues/1834): Drop time when used as a tag or field key.
- [#7152](https://github.com/influxdata/influxdb/issues/7152): Decrement number of measurements only once when deleting the last series from a measurement.
- [#7177](https://github.com/influxdata/influxdb/issues/7177): Fix base64 encoding issue with /debug/vars stats.
Expand Down
2 changes: 2 additions & 0 deletions Godeps
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ github.com/influxdata/usage-client 6d3895376368aa52a3a81d2a16e90f0f52371967
github.com/jwilder/encoding 4dada27c33277820fe35c7ee71ed34fbc9477d00
github.com/paulbellamy/ratecounter 5a11f585a31379765c190c033b6ad39956584447
github.com/peterh/liner 8975875355a81d612fafb9f5a6037bdcc2d9b073
github.com/philhofer/fwd 98c11a7a6ec829d672b03833c3d69a7fae1ca972
github.com/rakyll/statik e383bbf6b2ec1a2fb8492dfd152d945fb88919b6
github.com/retailnext/hllpp 38a7bb71b483e855d35010808143beaf05b67f9d
github.com/tinylib/msgp ad0ff2e232ad2e37faf67087fb24bf8d04a8ce20
golang.org/x/crypto 9477e0b78b9ac3d0b03822fd95422e2fe07627cd
2 changes: 2 additions & 0 deletions LICENSE_OF_DEPENDENCIES.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@
- github.com/jwilder/encoding [MIT LICENSE](https://github.com/jwilder/encoding/blob/master/LICENSE)
- github.com/paulbellamy/ratecounter [MIT LICENSE](https://github.com/paulbellamy/ratecounter/blob/master/LICENSE)
- github.com/peterh/liner [MIT LICENSE](https://github.com/peterh/liner/blob/master/COPYING)
- github.com/philhofer/fwd [MIT LICENSE](https://github.com/philhofer/fwd/blob/master/LICENSE.md)
- github.com/rakyll/statik [APACHE LICENSE](https://github.com/rakyll/statik/blob/master/LICENSE)
- github.com/retailnext/hllpp [BSD LICENSE](https://github.com/retailnext/hllpp/blob/master/LICENSE)
- github.com/tinylib/msgp [BSD LICENSE](https://github.com/tinylib/msgp/blob/master/LICENSE)
- glyphicons [LICENSE](http://glyphicons.com/license/)
- golang.org/x/crypto [BSD LICENSE](https://github.com/golang/crypto/blob/master/LICENSE)
- jquery 2.1.4 [MIT LICENSE](https://github.com/jquery/jquery/blob/master/LICENSE.txt)
Expand Down
271 changes: 271 additions & 0 deletions models/encode.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,271 @@
package models

// NOTE: THIS FILE WAS PRODUCED BY THE
// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
// DO NOT EDIT

import "github.com/tinylib/msgp/msgp"

// DecodeMsg implements msgp.Decodable
func (z *Row) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
var zwht uint32
zwht, err = dc.ReadMapHeader()
if err != nil {
return
}
for zwht > 0 {
zwht--
field, err = dc.ReadMapKeyPtr()
if err != nil {
return
}
switch msgp.UnsafeString(field) {
case "name":
z.Name, err = dc.ReadString()
if err != nil {
return
}
case "tags":
var zhct uint32
zhct, err = dc.ReadMapHeader()
if err != nil {
return
}
if z.Tags == nil && zhct > 0 {
z.Tags = make(map[string]string, zhct)
} else if len(z.Tags) > 0 {
for key, _ := range z.Tags {
delete(z.Tags, key)
}
}
for zhct > 0 {
zhct--
var zxvk string
var zbzg string
zxvk, err = dc.ReadString()
if err != nil {
return
}
zbzg, err = dc.ReadString()
if err != nil {
return
}
z.Tags[zxvk] = zbzg
}
case "columns":
var zcua uint32
zcua, err = dc.ReadArrayHeader()
if err != nil {
return
}
if cap(z.Columns) >= int(zcua) {
z.Columns = (z.Columns)[:zcua]
} else {
z.Columns = make([]string, zcua)
}
for zbai := range z.Columns {
z.Columns[zbai], err = dc.ReadString()
if err != nil {
return
}
}
case "values":
var zxhx uint32
zxhx, err = dc.ReadArrayHeader()
if err != nil {
return
}
if cap(z.Values) >= int(zxhx) {
z.Values = (z.Values)[:zxhx]
} else {
z.Values = make([][]interface{}, zxhx)
}
for zcmr := range z.Values {
var zlqf uint32
zlqf, err = dc.ReadArrayHeader()
if err != nil {
return
}
if cap(z.Values[zcmr]) >= int(zlqf) {
z.Values[zcmr] = (z.Values[zcmr])[:zlqf]
} else {
z.Values[zcmr] = make([]interface{}, zlqf)
}
for zajw := range z.Values[zcmr] {
z.Values[zcmr][zajw], err = dc.ReadIntf()
if err != nil {
return
}
}
}
default:
err = dc.Skip()
if err != nil {
return
}
}
}
return
}

// EncodeMsg implements msgp.Encodable
func (z *Row) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 4
// write "name"
err = en.Append(0x84, 0xa4, 0x6e, 0x61, 0x6d, 0x65)
if err != nil {
return err
}
err = en.WriteString(z.Name)
if err != nil {
return
}
// write "tags"
err = en.Append(0xa4, 0x74, 0x61, 0x67, 0x73)
if err != nil {
return err
}
err = en.WriteMapHeader(uint32(len(z.Tags)))
if err != nil {
return
}
for zxvk, zbzg := range z.Tags {
err = en.WriteString(zxvk)
if err != nil {
return
}
err = en.WriteString(zbzg)
if err != nil {
return
}
}
// write "columns"
err = en.Append(0xa7, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73)
if err != nil {
return err
}
err = en.WriteArrayHeader(uint32(len(z.Columns)))
if err != nil {
return
}
for zbai := range z.Columns {
err = en.WriteString(z.Columns[zbai])
if err != nil {
return
}
}
// write "values"
err = en.Append(0xa6, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73)
if err != nil {
return err
}
err = en.WriteArrayHeader(uint32(len(z.Values)))
if err != nil {
return
}
for zcmr := range z.Values {
err = en.WriteArrayHeader(uint32(len(z.Values[zcmr])))
if err != nil {
return
}
for zajw := range z.Values[zcmr] {
err = en.WriteIntf(z.Values[zcmr][zajw])
if err != nil {
return
}
}
}
return
}

// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *Row) Msgsize() (s int) {
s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 5 + msgp.MapHeaderSize
if z.Tags != nil {
for zxvk, zbzg := range z.Tags {
_ = zbzg
s += msgp.StringPrefixSize + len(zxvk) + msgp.StringPrefixSize + len(zbzg)
}
}
s += 8 + msgp.ArrayHeaderSize
for zbai := range z.Columns {
s += msgp.StringPrefixSize + len(z.Columns[zbai])
}
s += 7 + msgp.ArrayHeaderSize
for zcmr := range z.Values {
s += msgp.ArrayHeaderSize
for zajw := range z.Values[zcmr] {
s += msgp.GuessSize(z.Values[zcmr][zajw])
}
}
return
}

// DecodeMsg implements msgp.Decodable
func (z *Rows) DecodeMsg(dc *msgp.Reader) (err error) {
var zjfb uint32
zjfb, err = dc.ReadArrayHeader()
if err != nil {
return
}
if cap((*z)) >= int(zjfb) {
(*z) = (*z)[:zjfb]
} else {
(*z) = make(Rows, zjfb)
}
for zpks := range *z {
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
return
}
(*z)[zpks] = nil
} else {
if (*z)[zpks] == nil {
(*z)[zpks] = new(Row)
}
err = (*z)[zpks].DecodeMsg(dc)
if err != nil {
return
}
}
}
return
}

// EncodeMsg implements msgp.Encodable
func (z Rows) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteArrayHeader(uint32(len(z)))
if err != nil {
return
}
for zcxo := range z {
if z[zcxo] == nil {
err = en.WriteNil()
if err != nil {
return
}
} else {
err = z[zcxo].EncodeMsg(en)
if err != nil {
return
}
}
}
return
}

// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z Rows) Msgsize() (s int) {
s = msgp.ArrayHeaderSize
for zcxo := range z {
if z[zcxo] == nil {
s += msgp.NilSize
} else {
s += z[zcxo].Msgsize()
}
}
return
}
Loading

0 comments on commit f4d55fd

Please sign in to comment.