From 0e41c793a83669e21972a939bfb59897537810c7 Mon Sep 17 00:00:00 2001 From: Marcel van Lohuizen Date: Thu, 25 Mar 2021 08:11:02 +0100 Subject: [PATCH] encoding/protobuf/textproto: add decoder implementation Note that this is incredibly buggy. Not much I can do, as there just doesn't seem to be any good textproto parser for Go, and this one is the recommended "gold standard". Issue #5 Change-Id: Ieab0910dc4ea6072d9dc50e2947d8a7fb33ba7ef --- encoding/protobuf/pbinternal/attribute.go | 139 ++++++ encoding/protobuf/textproto/decoder.go | 439 ++++++++++++++++++ encoding/protobuf/textproto/decoder_test.go | 80 ++++ encoding/protobuf/textproto/doc.go | 27 ++ .../textproto/testdata/decoder/comments.txtar | 116 +++++ .../textproto/testdata/decoder/errors.txtar | 12 + .../textproto/testdata/decoder/list.txtar | 47 ++ .../textproto/testdata/decoder/map.txtar | 51 ++ .../textproto/testdata/decoder/scalar.txtar | 19 + .../textproto/testdata/decoder/simple.txtar | 63 +++ go.mod | 1 + go.sum | 3 + internal/attrs_test.go | 8 +- 13 files changed, 1001 insertions(+), 4 deletions(-) create mode 100644 encoding/protobuf/pbinternal/attribute.go create mode 100644 encoding/protobuf/textproto/decoder.go create mode 100644 encoding/protobuf/textproto/decoder_test.go create mode 100644 encoding/protobuf/textproto/doc.go create mode 100644 encoding/protobuf/textproto/testdata/decoder/comments.txtar create mode 100644 encoding/protobuf/textproto/testdata/decoder/errors.txtar create mode 100644 encoding/protobuf/textproto/testdata/decoder/list.txtar create mode 100644 encoding/protobuf/textproto/testdata/decoder/map.txtar create mode 100644 encoding/protobuf/textproto/testdata/decoder/scalar.txtar create mode 100644 encoding/protobuf/textproto/testdata/decoder/simple.txtar diff --git a/encoding/protobuf/pbinternal/attribute.go b/encoding/protobuf/pbinternal/attribute.go new file mode 100644 index 000000000..de4245f66 --- /dev/null +++ b/encoding/protobuf/pbinternal/attribute.go @@ -0,0 +1,139 @@ +// Copyright 2021 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pbinternal + +import ( + "strings" + + "cuelang.org/go/cue" +) + +type CompositeType int + +const ( + Normal CompositeType = iota + List + Map +) + +type ValueType int + +const ( + Unknown ValueType = iota + Message + Int + Float + String + Bytes + Bool +) + +type Info struct { + Name string + CUEName string + Attr cue.Attribute + + CompositeType CompositeType + ValueType ValueType + Type string + + // For maps only + KeyType ValueType // only for maps + KeyTypeString string +} + +func FromIter(i *cue.Iterator) (info Info, err error) { + return FromValue(i.Label(), i.Value()) +} + +func FromValue(name string, v cue.Value) (info Info, err error) { + a := v.Attribute("protobuf") + + info.Name = name + info.CUEName = name + + if a.Err() == nil { + info.Attr = a + + s, ok, err := a.Lookup(1, "name") + if err != nil { + return info, err + } + if ok { + info.Name = strings.TrimSpace(s) + } + + info.Type, err = a.String(1) + if err != nil { + return info, err + } + } + + switch v.IncompleteKind() { + case cue.ListKind: + info.CompositeType = List + e, _ := v.Elem() + if e.Exists() { + v = e + } else { + for i, _ := v.List(); i.Next(); { + v = i.Value() + break + } + } + + case cue.StructKind: + if strings.HasPrefix(info.Type, "map[") { + a := strings.SplitN(info.Type[len("map["):], ",", 2) + info.KeyTypeString = strings.TrimSpace(a[0]) + switch info.KeyTypeString { + case "string": + info.KeyType = String + case "bytes": + info.KeyType = Bytes + case "double", "float": + info.KeyType = Float + case "bool": + info.KeyType = Bool + default: + info.KeyType = Int // Assuming + } + info.CompositeType = Map + v, _ = v.Elem() + } + } + + switch v.IncompleteKind() { + case cue.StructKind: + info.ValueType = Message + + case cue.StringKind: + info.ValueType = String + + case cue.BytesKind: + info.ValueType = Bytes + + case cue.BoolKind: + info.ValueType = Bool + + case cue.IntKind: + info.ValueType = Int + + case cue.FloatKind, cue.NumberKind: + info.ValueType = Float + } + + return info, nil +} diff --git a/encoding/protobuf/textproto/decoder.go b/encoding/protobuf/textproto/decoder.go new file mode 100644 index 000000000..e32556bcd --- /dev/null +++ b/encoding/protobuf/textproto/decoder.go @@ -0,0 +1,439 @@ +// Copyright 2021 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package textproto + +import ( + "fmt" + "strings" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" + "cuelang.org/go/cue/errors" + "cuelang.org/go/cue/literal" + "cuelang.org/go/cue/token" + "cuelang.org/go/encoding/protobuf/pbinternal" + "cuelang.org/go/internal/core/adt" + "cuelang.org/go/internal/value" + + pbast "github.com/protocolbuffers/txtpbfmt/ast" + "github.com/protocolbuffers/txtpbfmt/parser" + "github.com/protocolbuffers/txtpbfmt/unquote" +) + +// Option defines options for the decoder. +// There are currently no options. +type Option func(*options) + +type options struct { +} + +// NewDecoder returns a new Decoder +func NewDecoder(option ...Option) *Decoder { + d := &Decoder{} + _ = d.m // work around linter bug. + return d +} + +// A Decoder caches conversions of cue.Value between calls to its methods. +type Decoder struct { + m map[*adt.Vertex]*mapping +} + +type decoder struct { + *Decoder + + // Reset on each call + errs errors.Error + file *token.File +} + +// Parse parses the given textproto bytes and converts them to a CUE expression, +// using schema as the guideline for conversion using the following rules: +// +// - the @protobuf attribute is optional, but is necessary for: +// - interpreting protobuf maps +// - using a name different from the CUE name +// - fields in the textproto that have no corresponding field in +// schema are ignored +// +// NOTE: the filename is used for associating position information. However, +// currently no position information is associated with the text proto because +// the position information of github.com/protocolbuffers/txtpbfmt is too +// unreliable to be useful. +func (d *Decoder) Parse(schema cue.Value, filename string, b []byte) (ast.Expr, error) { + dec := decoder{Decoder: d} + + // dec.errs = nil + + f := token.NewFile(filename, 0, len(b)) + f.SetLinesForContent(b) + dec.file = f + + cfg := parser.Config{} + nodes, err := parser.ParseWithConfig(b, cfg) + if err != nil { + return nil, errors.Newf(token.NoPos, "textproto: %v", err) + } + + m := dec.parseSchema(schema) + if dec.errs != nil { + return nil, dec.errs + } + + n := dec.decodeMsg(m, nodes) + if dec.errs != nil { + return nil, dec.errs + } + + return n, nil +} + +// Don't expose until the protobuf APIs settle down. +// func (d *decoder) Decode(schema cue.Value, textpbfmt) (cue.Value, error) { +// } + +type mapping struct { + children map[string]*fieldInfo +} + +type fieldInfo struct { + pbinternal.Info + msg *mapping + // keytype, for now +} + +func (d *decoder) addErr(err error) { + d.errs = errors.Append(d.errs, errors.Promote(err, "textproto")) +} + +func (d *decoder) addErrf(pos pbast.Position, format string, args ...interface{}) { + err := errors.Newf(d.protoPos(pos), "textproto: "+format, args...) + d.errs = errors.Append(d.errs, err) +} + +func (d *decoder) protoPos(p pbast.Position) token.Pos { + return d.file.Pos(int(p.Byte), token.NoRelPos) +} + +// parseSchema walks over a CUE "type", converts it to an internal data +// structure that is used for parsing text proto, and writes it to +// +func (d *decoder) parseSchema(schema cue.Value) *mapping { + _, v := value.ToInternal(schema) + if v == nil { + return nil + } + + if d.m == nil { + d.m = map[*adt.Vertex]*mapping{} + } else if m := d.m[v]; m != nil { + return m + } + + m := &mapping{children: map[string]*fieldInfo{}} + + i, err := schema.Fields() + if err != nil { + d.addErr(err) + return nil + } + + for i.Next() { + info, err := pbinternal.FromIter(i) + if err != nil { + d.addErr(err) + continue + } + + var msg *mapping + + switch info.CompositeType { + case pbinternal.Normal: + switch info.ValueType { + case pbinternal.Message: + msg = d.parseSchema(i.Value()) + } + + case pbinternal.List, pbinternal.Map: + e, _ := i.Value().Elem() + if e.IncompleteKind() == cue.StructKind { + msg = d.parseSchema(e) + } + } + + m.children[info.Name] = &fieldInfo{ + Info: info, + msg: msg, + } + } + + d.m[v] = m + return m +} + +func (d *decoder) decodeMsg(m *mapping, n []*pbast.Node) ast.Expr { + st := &ast.StructLit{} + + var listMap map[string]*ast.ListLit + + for _, x := range n { + if x.Values == nil && x.Children == nil { + if cg := addComments(x.PreComments...); cg != nil { + ast.SetRelPos(cg, token.NewSection) + st.Elts = append(st.Elts, cg) + continue + } + } + if m == nil { + continue + } + f, ok := m.children[x.Name] + if !ok { + continue // ignore unknown fields + } + + var value ast.Expr + + switch f.CompositeType { + default: + value = d.decodeValue(f, x) + + case pbinternal.List: + if listMap == nil { + listMap = make(map[string]*ast.ListLit) + } + + list := listMap[f.CUEName] + if list == nil { + list = &ast.ListLit{} + listMap[f.CUEName] = list + value = list + } + + if len(x.Values) == 1 || f.ValueType == pbinternal.Message { + v := d.decodeValue(f, x) + if value == nil { + if cg := addComments(x.PreComments...); cg != nil { + cg.Doc = true + ast.AddComment(v, cg) + } + } + if cg := addComments(x.PostValuesComments...); cg != nil { + cg.Position = 4 + ast.AddComment(v, cg) + } + list.Elts = append(list.Elts, v) + break + } + + var last ast.Expr + // Handle [1, 2, 3] + for _, v := range x.Values { + if v.Value == "" { + if cg := addComments(v.PreComments...); cg != nil { + if last != nil { + cg.Position = 4 + ast.AddComment(last, cg) + } else { + cg.Position = 1 + ast.AddComment(list, cg) + } + } + continue + } + y := *x + y.Values = []*pbast.Value{v} + last = d.decodeValue(f, &y) + list.Elts = append(list.Elts, last) + } + if cg := addComments(x.PostValuesComments...); cg != nil { + if last != nil { + cg.Position = 4 + ast.AddComment(last, cg) + } else { + cg.Position = 1 + ast.AddComment(list, cg) + } + } + if cg := addComments(x.ClosingBraceComment); cg != nil { + cg.Position = 4 + ast.AddComment(list, cg) + } + + case pbinternal.Map: + // mapValue: { + // key: 123 + // value: "string" + // } + if k := len(x.Values); k > 0 { + d.addErrf(x.Start, "values not allowed for Message type; found %d", k) + } + + var ( + key ast.Label + val ast.Expr + ) + + for _, c := range x.Children { + if len(c.Values) != 1 { + d.addErrf(x.Start, "expected 1 value, found %d", len(c.Values)) + continue + } + s := c.Values[0].Value + + switch c.Name { + case "key": + if strings.HasPrefix(s, `"`) { + key = &ast.BasicLit{Kind: token.STRING, Value: s} + } else { + key = ast.NewString(s) + } + + case "value": + val = d.decodeValue(f, c) + + if cg := addComments(x.ClosingBraceComment); cg != nil { + cg.Line = true + ast.AddComment(val, cg) + } + + default: + d.addErrf(c.Start, "unsupported key name %q in map", c.Name) + continue + } + } + + if key != nil && val != nil { + value = ast.NewStruct(key, val) + } + } + + if value != nil { + var label ast.Label + if s := f.CUEName; ast.IsValidIdent(s) { + label = ast.NewIdent(s) + } else { + label = ast.NewString(s) + + } + // TODO: convert line number information. However, position + // information in textpbfmt packages is too wonky to be useful + f := &ast.Field{ + Label: label, + Value: value, + // Attrs: []*ast.Attribute{{Text: f.attr.}}, + } + if cg := addComments(x.PreComments...); cg != nil { + cg.Doc = true + ast.AddComment(f, cg) + } + st.Elts = append(st.Elts, f) + } + } + + return st +} + +func addComments(lines ...string) (cg *ast.CommentGroup) { + var a []*ast.Comment + for _, c := range lines { + if !strings.HasPrefix(c, "#") { + continue + } + a = append(a, &ast.Comment{Text: "//" + c[1:]}) + } + if a != nil { + cg = &ast.CommentGroup{List: a} + } + return cg +} + +func (d *decoder) decodeValue(f *fieldInfo, n *pbast.Node) (x ast.Expr) { + if f.ValueType == pbinternal.Message { + if k := len(n.Values); k > 0 { + d.addErrf(n.Start, "values not allowed for Message type; found %d", k) + } + x = d.decodeMsg(f.msg, n.Children) + if cg := addComments(n.ClosingBraceComment); cg != nil { + cg.Line = true + cg.Position = 4 + ast.AddComment(x, cg) + } + return x + } + + if len(n.Values) != 1 { + d.addErrf(n.Start, "expected 1 value, found %d", len(n.Values)) + return nil + } + v := n.Values[0] + + defer func() { + if cg := addComments(v.PreComments...); cg != nil { + cg.Doc = true + ast.AddComment(x, cg) + } + if cg := addComments(v.InlineComment); cg != nil { + cg.Line = true + cg.Position = 2 + ast.AddComment(x, cg) + } + }() + + switch f.ValueType { + case pbinternal.String, pbinternal.Bytes: + s, err := unquote.Unquote(n) + if err != nil { + d.addErrf(n.Start, "invalid string or bytes: %v", err) + } + if f.ValueType == pbinternal.String { + s = literal.String.Quote(s) + } else { + s = literal.Bytes.Quote(s) + } + return &ast.BasicLit{Kind: token.STRING, Value: s} + + case pbinternal.Bool: + switch v.Value { + case "true": + return ast.NewBool(true) + + case "false": + default: + d.addErrf(n.Start, "invalid bool %s", v.Value) + } + return ast.NewBool(false) + + case pbinternal.Int, pbinternal.Float: + s := v.Value + switch s { + case "inf", "nan": + // TODO: include message. + return &ast.BottomLit{} + } + + var info literal.NumInfo + if err := literal.ParseNum(s, &info); err != nil { + d.addErrf(n.Start, "invalid number %s", s) + } + if !info.IsInt() { + return &ast.BasicLit{Kind: token.FLOAT, Value: s} + } + return &ast.BasicLit{Kind: token.INT, Value: info.String()} + + default: + panic(fmt.Sprintf("unexpected type %v", f.ValueType)) + } +} diff --git a/encoding/protobuf/textproto/decoder_test.go b/encoding/protobuf/textproto/decoder_test.go new file mode 100644 index 000000000..4e5dd56c5 --- /dev/null +++ b/encoding/protobuf/textproto/decoder_test.go @@ -0,0 +1,80 @@ +// Copyright 2021 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package textproto_test + +import ( + "strings" + "testing" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast/astutil" + "cuelang.org/go/cue/errors" + "cuelang.org/go/cue/format" + "cuelang.org/go/encoding/protobuf/textproto" + "cuelang.org/go/internal/cuetest" + "cuelang.org/go/internal/cuetxtar" +) + +func TestParse(t *testing.T) { + test := cuetxtar.TxTarTest{ + Root: "./testdata/decoder", + Name: "decode", + Update: cuetest.UpdateGoldenFiles, + } + + r := cue.Runtime{} + + d := textproto.NewDecoder() + + test.Run(t, func(t *cuetxtar.Test) { + // TODO: use high-level API. + + var schema cue.Value + var filename string + var b []byte + + for _, f := range t.Archive.Files { + switch { + case strings.HasSuffix(f.Name, ".cue"): + inst, err := r.Compile(f.Name, f.Data) + if err != nil { + t.WriteErrors(errors.Promote(err, "test")) + return + } + schema = inst.Value() + + case strings.HasSuffix(f.Name, ".textproto"): + filename = f.Name + b = f.Data + } + } + + x, err := d.Parse(schema, filename, b) + if err != nil { + t.WriteErrors(errors.Promote(err, "test")) + return + } + + f, err := astutil.ToFile(x) + if err != nil { + t.WriteErrors(errors.Promote(err, "test")) + } + b, err = format.Node(f) + if err != nil { + t.WriteErrors(errors.Promote(err, "test")) + } + _, _ = t.Write(b) + }) +} diff --git a/encoding/protobuf/textproto/doc.go b/encoding/protobuf/textproto/doc.go new file mode 100644 index 000000000..75fb6e57d --- /dev/null +++ b/encoding/protobuf/textproto/doc.go @@ -0,0 +1,27 @@ +// Copyright 2021 CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package textproto converts text protobuffer files to and from CUE. +// +// Note that textproto is an unofficial standard and that there are no +// specifications: the recommended packages are the de facto standard for the +// relevant programming languages and the recommended implementations may vary +// considerably between them. +// +// Also, the standard text proto parsing libraries are rather buggy. Please +// verify that a parsing issues is not related these libraries before filing +// bugs with CUE. +// +// API Status: DRAFT: API may change without notice. +package textproto diff --git a/encoding/protobuf/textproto/testdata/decoder/comments.txtar b/encoding/protobuf/textproto/testdata/decoder/comments.txtar new file mode 100644 index 000000000..f4fbc7493 --- /dev/null +++ b/encoding/protobuf/textproto/testdata/decoder/comments.txtar @@ -0,0 +1,116 @@ +// TODO: there are many missing comments, but these really are consequences +// of the buggy textpbfmt library. + +-- foo.cue -- +a: string +b: [...int] +c: [...int32] +m: [...#Msg] +#Msg: { + x: string + y: string +} + +-- input.textproto -- +# file comment + +# doc comment a +a: "dsfadsafsaf" # line comment + +# floating comment a-b + +# doc comment b +b: [ + # pre-elem comment + + # doc elem 0 + 123, # elem 0 line comment + # trailing elem 0 + + # inbetween comment 1 + + # inbetween comment 2 + + # doc elem 1 + 456 # elem 1 line comment + # trailing elem 1 + + # final floating 1 + + # final floating 2 +] +# floating end + +c: 2342134 # line elem 0 +c: 2342135 # line elem 1 +# inbetween elems +c: 2342136 # line elem 2 +# after list c + +# floating + +m { + x: "sdfff" # inner line comment + y: "q\"qq\\q\n" + # after last value +} # after elem line + # after elem separate +m { + x: " sdfff2 \321\202\320\265\321\201\321\202 " + y: "q\tqq<>q2&\001\377" +} # after list line +# after list + +# floating end + +-- out/decode -- +// file comment + +// doc comment a +a: "dsfadsafsaf" // line comment + +// floating comment a-b + +// doc comment b +b: [ + // pre-elem comment + 123, // elem 0 line comment + + // trailing elem 0 + // inbetween comment 2 + 456, // elem 1 line comment + + // trailing elem 1 + + // final floating 2 +] + +// floating end + +c: [2342134, // line elem 0 + 2342135, // line elem 1 + // inbetween elems + 2342136, // line elem 2 +] + +// after list c + +// floating + +m: [{ + x: "sdfff" // inner line comment + y: "q\"qq\\q\n" + + // after last value + +}, // after elem line + // after elem separate + { + x: " sdfff2 тест " + y: "q\tqq<>q2&\u0001�" + }, // after list line +] + +// after list + +// floating end diff --git a/encoding/protobuf/textproto/testdata/decoder/errors.txtar b/encoding/protobuf/textproto/testdata/decoder/errors.txtar new file mode 100644 index 000000000..3f173d40e --- /dev/null +++ b/encoding/protobuf/textproto/testdata/decoder/errors.txtar @@ -0,0 +1,12 @@ +# The error in input.textproto (wrong comment style) is going undetected. +# This is a protobuf bug. Can't do much about it. +-- errors.cue -- +a: int + +-- input.textproto -- + +// Silent nights +a: 1 + +-- out/decode -- + diff --git a/encoding/protobuf/textproto/testdata/decoder/list.txtar b/encoding/protobuf/textproto/testdata/decoder/list.txtar new file mode 100644 index 000000000..5a0969ffa --- /dev/null +++ b/encoding/protobuf/textproto/testdata/decoder/list.txtar @@ -0,0 +1,47 @@ +// test of non-standard list + +// TODO: there are many missing comments, but these really are consequences +// of the buggy textpbfmt library. + +-- list.cue -- +empty1: [...int] +empty2: [...int] + +int1: [...int] +int2: [...int] +int3: [...int] + +string1: [...string] + +float1: [...number] + + +-- input.textproto -- +empty1: [] +empty2: [ # foo +] + +int1: [1, 2] +int2: [1 2] # omitting commas okay +int3: [ + 1 # omitting comma okay + 2 +] + +string1: [ + "a", # omitting comma NOT supported + "b" +] + +float1: [ 1e+2 1. 0] +-- out/decode -- +empty1: [] +empty2: [ // foo +] +int1: [1, 2] +int2: [1, 2] // omitting commas okay +int3: [1, // omitting comma okay + 2] +string1: ["a", // omitting comma NOT supported + "b"] +float1: [1e+2, 1.0, 0] diff --git a/encoding/protobuf/textproto/testdata/decoder/map.txtar b/encoding/protobuf/textproto/testdata/decoder/map.txtar new file mode 100644 index 000000000..0dbf2b50c --- /dev/null +++ b/encoding/protobuf/textproto/testdata/decoder/map.txtar @@ -0,0 +1,51 @@ +-- map.cue -- +map: {[string]: int} @protobuf(1,map[string]int) + + +implicit: [string]: string + +intMap: {[string]: int} @protobuf(1,map[int]int) + +-- input.textproto -- +map: { + key: "foo" + value: 2 +} +map: { + key: "bar" + value: 3 +} + +implicit: { + key: "foo" + value: 2 +} +implicit: { + key: "bar" + value: 3 +} + +intMap: { + key: 100 + value: 2 +} +intMap: { + key: 102 + value: 3 +} + +-- out/decode -- +map: { + "foo": 2 +} +map: { + "bar": 3 +} +implicit: {} +implicit: {} +intMap: { + "100": 2 +} +intMap: { + "102": 3 +} diff --git a/encoding/protobuf/textproto/testdata/decoder/scalar.txtar b/encoding/protobuf/textproto/testdata/decoder/scalar.txtar new file mode 100644 index 000000000..e39c3f873 --- /dev/null +++ b/encoding/protobuf/textproto/testdata/decoder/scalar.txtar @@ -0,0 +1,19 @@ +-- map.cue -- +inf: number +nan: number + +t: bool +f: bool + + +-- input.textproto -- +inf: inf +nan: nan +t: true +f: false + +-- out/decode -- +inf: _|_ +nan: _|_ +t: true +f: false diff --git a/encoding/protobuf/textproto/testdata/decoder/simple.txtar b/encoding/protobuf/textproto/testdata/decoder/simple.txtar new file mode 100644 index 000000000..fb4645e01 --- /dev/null +++ b/encoding/protobuf/textproto/testdata/decoder/simple.txtar @@ -0,0 +1,63 @@ +// From: https://stackoverflow.com/questions/18873924/what-does-the-protobuf-text-format-look-like +-- foo.cue -- +#MyEnum: "Default" | "Variant1" | "Variant100" + +f1: string +f2: int64 +fa: [...uint64] +fb: [...int32] +fc: [...number] +pairs: [...#Pair] +bbbb: bytes // optional + +// extensions 100 to max; + +#Pair: { + key: string + value: string +} + +-- input.textproto -- +f1: "dsfadsafsaf" +f2: 234 # value comment + +fa: 2342134 +fa: 2342135 +fa: 2342136 +# Mix of list and single elements. +fb: [ -2342134, -2342135, -2342136 ] +fb: -1000 + +fc: 4 +fc: 7 +fc: -12 +fc: 4 +fc: 7 +fc: -3 +fc: 4 +fc: 7 +fc: 0 +pairs { + key: "sdfff" + value: "q\"qq\\q\n" +} +pairs { + key: " sdfff2 \321\202\320\265\321\201\321\202 " + value: "q\tqq<>q2&\001\377" +} +bbbb: "\000\001\002\377\376\375" +-- out/decode -- +f1: "dsfadsafsaf" +f2: 234 // value comment +fa: [2342134, 2342135, 2342136] +// Mix of list and single elements. +fb: [-2342134, -2342135, -2342136, -1000] +fc: [4, 7, -12, 4, 7, -3, 4, 7, 0] +pairs: [{ + key: "sdfff" + value: "q\"qq\\q\n" +}, { + key: " sdfff2 тест " + value: "q\tqq<>q2&\u0001�" +}] +bbbb: '\x00\x01\x02\xff\xfe\xfd' diff --git a/go.mod b/go.mod index d838caf68..81af4658e 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/lib/pq v1.0.0 // indirect github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de github.com/pkg/errors v0.8.1 // indirect + github.com/protocolbuffers/txtpbfmt v0.0.0-20201118171849-f6a6b3f636fc github.com/rogpeppe/go-internal v1.8.0 github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.3 diff --git a/go.sum b/go.sum index ef6affb4a..de875ad49 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,7 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -91,6 +92,8 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/protocolbuffers/txtpbfmt v0.0.0-20201118171849-f6a6b3f636fc h1:gSVONBi2HWMFXCa9jFdYvYk7IwW/mTLxWOF7rXS4LO0= +github.com/protocolbuffers/txtpbfmt v0.0.0-20201118171849-f6a6b3f636fc/go.mod h1:KbKfKPy2I6ecOIGA9apfheFv14+P3RSmmQvshofQyMY= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= diff --git a/internal/attrs_test.go b/internal/attrs_test.go index 8256ab352..38ba0d627 100644 --- a/internal/attrs_test.go +++ b/internal/attrs_test.go @@ -48,11 +48,11 @@ func TestAttributeBody(t *testing.T) { in: `foo,bar,baz`, out: "[{foo 0} {bar 0} {baz 0}]", }, { - in: `1,"map"`, - out: "[{1 0} {map 0}]", + in: `1,map[int]string`, + out: "[{1 0} {map[int]string 0}]", }, { - in: `1, "map"`, - out: "[{1 0} {map 0}]", + in: `1,map[int]string`, + out: "[{1 0} {map[int]string 0}]", }, { in: `bar=str`, out: "[{bar=str 3}]",