diff --git a/go.mod b/go.mod index 7a21115..e1a9789 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,5 @@ go 1.15 require ( github.com/ipfs/go-cid v0.0.7 github.com/ipld/go-ipld-prime v0.9.0 - github.com/mr-tron/base58 v1.2.0 // indirect - github.com/multiformats/go-varint v0.0.6 // indirect - github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 + google.golang.org/protobuf v1.26.0 ) diff --git a/go.sum b/go.sum index dcd79d4..b8d4d10 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,12 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipld/go-ipld-prime v0.7.0 h1:eigF1ZpaL1prbsKYVMqPLoPJqD/pzkQOe2j1uzvVg7w= -github.com/ipld/go-ipld-prime v0.7.0/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM= -github.com/ipld/go-ipld-prime v0.7.1-0.20210225173718-8fef5312eb12 h1:O9VMUYa2ktly9ql6W0LG0k8lXqg3bqz2ZfbaHXN3law= -github.com/ipld/go-ipld-prime v0.7.1-0.20210225173718-8fef5312eb12/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM= github.com/ipld/go-ipld-prime v0.9.0 h1:N2OjJMb+fhyFPwPnVvJcWU/NsumP8etal+d2v3G4eww= github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -21,8 +19,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= -github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= @@ -39,29 +35,22 @@ github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77 github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I= -github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.15 h1:hWOPdrNqDjwHDx82vsYGSDZNyktOJJ2dzZJzFkOV1jM= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 h1:CskT+S6Ay54OwxBGB0R3Rsx4Muto6UnEYTyKJbyRIAI= github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= -github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= -github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -69,14 +58,12 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201223074533-0d417f636930 h1:vRgIt+nup/B/BwIS0g2oC0haq0iqbV3ZA+u6+0TlNCo= -golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= diff --git a/marshal.go b/marshal.go index fe78e10..291ff97 100644 --- a/marshal.go +++ b/marshal.go @@ -1,14 +1,14 @@ package dagpb import ( + "fmt" "io" - math_bits "math/bits" "sort" "github.com/ipfs/go-cid" ipld "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "golang.org/x/xerrors" + "google.golang.org/protobuf/encoding/protowire" ) type pbLink struct { @@ -19,23 +19,42 @@ type pbLink struct { hasTsize bool } -// Marshal provides an IPLD codec encode interface for DAG-PB data. Provide a +// Encode provides an IPLD codec encode interface for DAG-PB data. Provide a // conforming Node and a destination for bytes to marshal a DAG-PB IPLD Node. // The Node must strictly conform to the DAG-PB schema // (https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-pb.md). // For safest use, build Nodes using the Type.PBNode type. -func Marshal(inNode ipld.Node, out io.Writer) error { +// This function is registered via the go-ipld-prime link loader for multicodec +// code 0x70 when this package is invoked via init. +func Encode(node ipld.Node, w io.Writer) error { + // 1KiB can be allocated on the stack, and covers most small nodes + // without having to grow the buffer and cause allocations. + enc := make([]byte, 0, 1024) + + enc, err := AppendEncode(enc, node) + if err != nil { + return err + } + _, err = w.Write(enc) + return err +} + +// AppendEncode is like Encode, but it uses a destination buffer directly. +// This means less copying of bytes, and if the destination has enough capacity, +// fewer allocations. +func AppendEncode(enc []byte, inNode ipld.Node) ([]byte, error) { // Wrap in a typed node for some basic schema form checking builder := Type.PBNode.NewBuilder() if err := builder.AssignNode(inNode); err != nil { - return err + return enc, err } node := builder.Build() links, err := node.LookupByString("Links") if err != nil { - return err + return enc, err } + if links.Length() > 0 { // collect links into a slice so we can properly sort for encoding pbLinks := make([]pbLink, links.Length()) @@ -44,26 +63,26 @@ func Marshal(inNode ipld.Node, out io.Writer) error { for !linksIter.Done() { ii, link, err := linksIter.Next() if err != nil { - return err + return enc, err } { // Hash (required) d, err := link.LookupByString("Hash") if err != nil { - return err + return enc, err } l, err := d.AsLink() if err != nil { - return err + return enc, err } if err != nil { - return err + return enc, err } cl, ok := l.(cidlink.Link) if !ok { // this _should_ be taken care of by the Typed conversion above with // "missing required fields: Hash" - return xerrors.Errorf("invalid DAG-PB form (link must have a Hash)") + return enc, fmt.Errorf("invalid DAG-PB form (link must have a Hash)") } pbLinks[ii].hash = cl.Cid } @@ -71,12 +90,12 @@ func Marshal(inNode ipld.Node, out io.Writer) error { { // Name (optional) nameNode, err := link.LookupByString("Name") if err != nil { - return err + return enc, err } if !nameNode.IsAbsent() { name, err := nameNode.AsString() if err != nil { - return err + return enc, err } pbLinks[ii].name = name pbLinks[ii].hasName = true @@ -86,15 +105,15 @@ func Marshal(inNode ipld.Node, out io.Writer) error { { // Tsize (optional) tsizeNode, err := link.LookupByString("Tsize") if err != nil { - return err + return enc, err } if !tsizeNode.IsAbsent() { tsize, err := tsizeNode.AsInt() if err != nil { - return err + return enc, err } if tsize < 0 { - return xerrors.Errorf("Link has negative Tsize value [%v]", tsize) + return enc, fmt.Errorf("Link has negative Tsize value [%v]", tsize) } utsize := uint64(tsize) pbLinks[ii].tsize = utsize @@ -105,98 +124,53 @@ func Marshal(inNode ipld.Node, out io.Writer) error { // links must be strictly sorted by Name before encoding, leaving stable // ordering where the names are the same (or absent) - sortLinks(pbLinks) + sort.Stable(pbLinkSlice(pbLinks)) for _, link := range pbLinks { - size := link.encodedSize() - chunk := make([]byte, size+sizeOfVarint(uint64(size))+1) - chunk[0] = 0x12 // field & wire type for Links - offset := encodeVarint(chunk, 1, uint64(size)) - wrote, err := link.marshal(chunk, offset) - if err != nil { - return err + hash := link.hash.Bytes() + + size := 0 + size += protowire.SizeTag(2) + size += protowire.SizeBytes(len(hash)) + if link.hasName { + size += protowire.SizeTag(2) + size += protowire.SizeBytes(len(link.name)) + } + if link.hasTsize { + size += protowire.SizeTag(3) + size += protowire.SizeVarint(uint64(link.tsize)) } - if wrote != size { - return xerrors.Errorf("bad PBLink marshal, wrote wrong number of bytes") + + enc = protowire.AppendTag(enc, 2, 2) // field & wire type for Links + enc = protowire.AppendVarint(enc, uint64(size)) + + enc = protowire.AppendTag(enc, 1, 2) // field & wire type for Hash + enc = protowire.AppendBytes(enc, hash) + if link.hasName { + enc = protowire.AppendTag(enc, 2, 2) // field & wire type for Name + enc = protowire.AppendString(enc, link.name) + } + if link.hasTsize { + enc = protowire.AppendTag(enc, 3, 0) // field & wire type for Tsize + enc = protowire.AppendVarint(enc, uint64(link.tsize)) } - out.Write(chunk) } } // if links // Data (optional) data, err := node.LookupByString("Data") if err != nil { - return err + return enc, err } if !data.IsAbsent() { byts, err := data.AsBytes() if err != nil { - return err + return enc, err } - size := uint64(len(byts)) - lead := make([]byte, sizeOfVarint(size)+1) - lead[0] = 0xa // field and wireType for Data - encodeVarint(lead, 1, size) - out.Write(lead) - out.Write(byts) + enc = protowire.AppendTag(enc, 1, 2) // field & wire type for Data + enc = protowire.AppendBytes(enc, byts) } - return nil -} - -// predict the byte size of the encoded Link -func (link pbLink) encodedSize() (n int) { - l := link.hash.ByteLen() - n += 1 + l + sizeOfVarint(uint64(l)) - if link.hasName { - l = len(link.name) - n += 1 + l + sizeOfVarint(uint64(l)) - } - if link.hasTsize { - n += 1 + sizeOfVarint(uint64(link.tsize)) - } - return n -} - -// encode a Link to PB -func (link pbLink) marshal(data []byte, offset int) (int, error) { - base := offset - data[offset] = 0xa // field and wireType for Hash - byts := link.hash.Bytes() - offset = encodeVarint(data, offset+1, uint64(len(byts))) - copy(data[offset:], byts) - offset += len(byts) - if link.hasName { - data[offset] = 0x12 // field and wireType for Name - offset = encodeVarint(data, offset+1, uint64(len(link.name))) - copy(data[offset:], link.name) - offset += len(link.name) - } - if link.hasTsize { - data[offset] = 0x18 // field and wireType for Tsize - offset = encodeVarint(data, offset+1, uint64(link.tsize)) - } - return offset - base, nil -} - -// predict the size of a varint for PB before creating it -func sizeOfVarint(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} - -// encode a varint to a PB chunk -func encodeVarint(data []byte, offset int, v uint64) int { - for v >= 1<<7 { - data[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - data[offset] = uint8(v) - return offset + 1 -} - -// stable sorting of Links using the strict sorting rules -func sortLinks(links []pbLink) { - sort.Stable(pbLinkSlice(links)) + return enc, err } type pbLinkSlice []pbLink diff --git a/multicodec.go b/multicodec.go index e222e45..568c777 100644 --- a/multicodec.go +++ b/multicodec.go @@ -19,27 +19,6 @@ func init() { multicodec.RegisterEncoder(0x70, Encode) } -// Decode provides an IPLD codec decode interface for DAG-PB data. Provide a -// compatible NodeAssembler and a byte source to unmarshal a DAG-PB IPLD Node. -// Use the NodeAssembler from the PBNode type for safest construction -// (Type.PBNode.NewBuilder()). A Map assembler will also work. -// This function is registered via the go-ipld-prime link loader for multicodec -// code 0x70 when this package is invoked via init. -func Decode(na ipld.NodeAssembler, r io.Reader) error { - return Unmarshal(na, r) -} - -// Encode provides an IPLD codec encode interface for DAG-PB data. Provide a -// conforming Node and a destination for bytes to marshal a DAG-PB IPLD Node. -// The Node must strictly conform to the DAG-PB schema -// (https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-pb.md). -// For safest use, build Nodes using the Type.PBNode type. -// This function is registered via the go-ipld-prime link loader for multicodec -// code 0x70 when this package is invoked via init. -func Encode(n ipld.Node, w io.Writer) error { - return Marshal(n, w) -} - // AddSupportToChooser takes an existing node prototype chooser and subs in // PBNode for the dag-pb multicodec code. func AddSupportToChooser(existing traversal.LinkTargetNodePrototypeChooser) traversal.LinkTargetNodePrototypeChooser { @@ -57,11 +36,13 @@ func AddSupportToChooser(existing traversal.LinkTargetNodePrototypeChooser) trav // unnecessary to have two supported names for each API. // Deprecated: use Decode instead. -func Decoder(na ipld.NodeAssembler, r io.Reader) error { - return Unmarshal(na, r) -} +func Decoder(na ipld.NodeAssembler, r io.Reader) error { return Decode(na, r) } + +// Deprecated: use Decode instead. +func Unmarshal(na ipld.NodeAssembler, r io.Reader) error { return Decode(na, r) } // Deprecated: use Encode instead. -func Encoder(n ipld.Node, w io.Writer) error { - return Marshal(n, w) -} +func Encoder(inNode ipld.Node, w io.Writer) error { return Encode(inNode, w) } + +// Deprecated: use Encode instead. +func Marshal(inNode ipld.Node, w io.Writer) error { return Encode(inNode, w) } diff --git a/unmarshal.go b/unmarshal.go index 90f4a8a..c8b6a60 100644 --- a/unmarshal.go +++ b/unmarshal.go @@ -1,30 +1,52 @@ package dagpb import ( + "fmt" "io" + "io/ioutil" "github.com/ipfs/go-cid" ipld "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/polydawn/refmt/shared" - "golang.org/x/xerrors" + "google.golang.org/protobuf/encoding/protowire" ) // ErrIntOverflow is returned a varint overflows during decode, it indicates // malformed data -var ErrIntOverflow = xerrors.Errorf("protobuf: varint overflow") +var ErrIntOverflow = fmt.Errorf("protobuf: varint overflow") -// Unmarshal provides an IPLD codec decode interface for DAG-PB data. Provide -// a compatible NodeAssembler and a byte source to unmarshal a DAG-PB IPLD -// Node. Use the NodeAssembler from the PBNode type for safest construction +// Decode provides an IPLD codec decode interface for DAG-PB data. Provide a +// compatible NodeAssembler and a byte source to unmarshal a DAG-PB IPLD Node. +// Use the NodeAssembler from the PBNode type for safest construction // (Type.PBNode.NewBuilder()). A Map assembler will also work. -func Unmarshal(na ipld.NodeAssembler, in io.Reader) error { +// This function is registered via the go-ipld-prime link loader for multicodec +// code 0x70 when this package is invoked via init. +func Decode(na ipld.NodeAssembler, in io.Reader) error { + var src []byte + if buf, ok := in.(interface{ Bytes() []byte }); ok { + src = buf.Bytes() + } else { + var err error + src, err = ioutil.ReadAll(in) + if err != nil { + return err + } + } + return DecodeBytes(na, src) +} + +// DecodeBytes is like Decode, but it uses an input buffer directly. +// Decode will grab or read all the bytes from an io.Reader anyway, so this can +// save having to copy the bytes or create a bytes.Buffer. +func DecodeBytes(na ipld.NodeAssembler, src []byte) error { + remaining := src + ma, err := na.BeginMap(2) if err != nil { return err } // always make "Links", even if we don't use it - if err = ma.AssembleKey().AssignString("Links"); err != nil { + if err := ma.AssembleKey().AssignString("Links"); err != nil { return err } links, err := ma.AssembleValue().BeginList(0) @@ -33,30 +55,33 @@ func Unmarshal(na ipld.NodeAssembler, in io.Reader) error { } haveData := false - reader := shared.NewReader(in) for { - _, err := reader.Readn1() - if err == io.EOF { + if len(remaining) == 0 { break } - reader.Unreadn1() - fieldNum, wireType, err := decodeKey(reader) - if err != nil { - return err + fieldNum, wireType, n := protowire.ConsumeTag(remaining) + if n < 0 { + return protowire.ParseError(n) } + remaining = remaining[n:] + if wireType != 2 { - return xerrors.Errorf("protobuf: (PBNode) invalid wireType, expected 2, got %d", wireType) + return fmt.Errorf("protobuf: (PBNode) invalid wireType, expected 2, got %d", wireType) } - if fieldNum == 1 { + switch fieldNum { + case 1: if haveData { - return xerrors.Errorf("protobuf: (PBNode) duplicate Data section") + return fmt.Errorf("protobuf: (PBNode) duplicate Data section") } - var chunk []byte - if chunk, err = decodeBytes(reader); err != nil { - return err + + chunk, n := protowire.ConsumeBytes(remaining) + if n < 0 { + return protowire.ParseError(n) } + remaining = remaining[n:] + // Data must come after Links, so it's safe to close this here even if we // didn't use it if err := links.Finish(); err != nil { @@ -70,27 +95,32 @@ func Unmarshal(na ipld.NodeAssembler, in io.Reader) error { return err } haveData = true - } else if fieldNum == 2 { + + case 2: if haveData { - return xerrors.Errorf("protobuf: (PBNode) invalid order, found Data before Links content") + return fmt.Errorf("protobuf: (PBNode) invalid order, found Data before Links content") } - bytesLen, err := decodeVarint(reader) - if err != nil { - return err + bytesLen, n := protowire.ConsumeVarint(remaining) + if n < 0 { + return protowire.ParseError(n) } + remaining = remaining[n:] + curLink, err := links.AssembleValue().BeginMap(3) if err != nil { return err } - if err = unmarshalLink(reader, int(bytesLen), curLink); err != nil { + if err := unmarshalLink(remaining[:bytesLen], curLink); err != nil { return err } + remaining = remaining[bytesLen:] if err := curLink.Finish(); err != nil { return err } - } else { - return xerrors.Errorf("protobuf: (PBNode) invalid fieldNumber, expected 1 or 2, got %d", fieldNum) + + default: + return fmt.Errorf("protobuf: (PBNode) invalid fieldNumber, expected 1 or 2, got %d", fieldNum) } } @@ -102,44 +132,45 @@ func Unmarshal(na ipld.NodeAssembler, in io.Reader) error { return ma.Finish() } -func unmarshalLink(reader shared.SlickReader, length int, ma ipld.MapAssembler) error { +func unmarshalLink(remaining []byte, ma ipld.MapAssembler) error { haveHash := false haveName := false haveTsize := false - startOffset := reader.NumRead() for { - readBytes := reader.NumRead() - startOffset - if readBytes == length { + if len(remaining) == 0 { break - } else if readBytes > length { - return xerrors.Errorf("protobuf: (PBLink) bad length for link") } - fieldNum, wireType, err := decodeKey(reader) - if err != nil { - return err + + fieldNum, wireType, n := protowire.ConsumeTag(remaining) + if n < 0 { + return protowire.ParseError(n) } + remaining = remaining[n:] - if fieldNum == 1 { + switch fieldNum { + case 1: if haveHash { - return xerrors.Errorf("protobuf: (PBLink) duplicate Hash section") + return fmt.Errorf("protobuf: (PBLink) duplicate Hash section") } if haveName { - return xerrors.Errorf("protobuf: (PBLink) invalid order, found Name before Hash") + return fmt.Errorf("protobuf: (PBLink) invalid order, found Name before Hash") } if haveTsize { - return xerrors.Errorf("protobuf: (PBLink) invalid order, found Tsize before Hash") + return fmt.Errorf("protobuf: (PBLink) invalid order, found Tsize before Hash") } if wireType != 2 { - return xerrors.Errorf("protobuf: (PBLink) wrong wireType (%d) for Hash", wireType) + return fmt.Errorf("protobuf: (PBLink) wrong wireType (%d) for Hash", wireType) } - var chunk []byte - if chunk, err = decodeBytes(reader); err != nil { - return err + chunk, n := protowire.ConsumeBytes(remaining) + if n < 0 { + return protowire.ParseError(n) } - var c cid.Cid - if _, c, err = cid.CidFromBytes(chunk); err != nil { - return xerrors.Errorf("invalid Hash field found in link, expected CID (%v)", err) + remaining = remaining[n:] + + _, c, err := cid.CidFromBytes(chunk) + if err != nil { + return fmt.Errorf("invalid Hash field found in link, expected CID (%v)", err) } if err := ma.AssembleKey().AssignString("Hash"); err != nil { return err @@ -148,21 +179,24 @@ func unmarshalLink(reader shared.SlickReader, length int, ma ipld.MapAssembler) return err } haveHash = true - } else if fieldNum == 2 { + + case 2: if haveName { - return xerrors.Errorf("protobuf: (PBLink) duplicate Name section") + return fmt.Errorf("protobuf: (PBLink) duplicate Name section") } if haveTsize { - return xerrors.Errorf("protobuf: (PBLink) invalid order, found Tsize before Name") + return fmt.Errorf("protobuf: (PBLink) invalid order, found Tsize before Name") } if wireType != 2 { - return xerrors.Errorf("protobuf: (PBLink) wrong wireType (%d) for Name", wireType) + return fmt.Errorf("protobuf: (PBLink) wrong wireType (%d) for Name", wireType) } - var chunk []byte - if chunk, err = decodeBytes(reader); err != nil { - return err + chunk, n := protowire.ConsumeBytes(remaining) + if n < 0 { + return protowire.ParseError(n) } + remaining = remaining[n:] + if err := ma.AssembleKey().AssignString("Name"); err != nil { return err } @@ -170,18 +204,21 @@ func unmarshalLink(reader shared.SlickReader, length int, ma ipld.MapAssembler) return err } haveName = true - } else if fieldNum == 3 { + + case 3: if haveTsize { - return xerrors.Errorf("protobuf: (PBLink) duplicate Tsize section") + return fmt.Errorf("protobuf: (PBLink) duplicate Tsize section") } if wireType != 0 { - return xerrors.Errorf("protobuf: (PBLink) wrong wireType (%d) for Tsize", wireType) + return fmt.Errorf("protobuf: (PBLink) wrong wireType (%d) for Tsize", wireType) } - var v uint64 - if v, err = decodeVarint(reader); err != nil { - return err + v, n := protowire.ConsumeVarint(remaining) + if n < 0 { + return protowire.ParseError(n) } + remaining = remaining[n:] + if err := ma.AssembleKey().AssignString("Tsize"); err != nil { return err } @@ -189,59 +226,15 @@ func unmarshalLink(reader shared.SlickReader, length int, ma ipld.MapAssembler) return err } haveTsize = true - } else { - return xerrors.Errorf("protobuf: (PBLink) invalid fieldNumber, expected 1, 2 or 3, got %d", fieldNum) + + default: + return fmt.Errorf("protobuf: (PBLink) invalid fieldNumber, expected 1, 2 or 3, got %d", fieldNum) } } if !haveHash { - return xerrors.Errorf("invalid Hash field found in link, expected CID") + return fmt.Errorf("invalid Hash field found in link, expected CID") } return nil } - -// decode the lead for a PB chunk, fieldNum & wireType, that tells us which -// field in the schema we're looking at and what data type it is -func decodeKey(reader shared.SlickReader) (int, int, error) { - var wire uint64 - var err error - if wire, err = decodeVarint(reader); err != nil { - return 0, 0, err - } - fieldNum := int(wire >> 3) - wireType := int(wire & 0x7) - return fieldNum, wireType, nil -} - -// decode a byte string from PB -func decodeBytes(reader shared.SlickReader) ([]byte, error) { - bytesLen, err := decodeVarint(reader) - if err != nil { - return nil, err - } - byts, err := reader.Readn(int(bytesLen)) - if err != nil { - return nil, xerrors.Errorf("protobuf: unexpected read error: %w", err) - } - return byts, nil -} - -// decode a varint from PB -func decodeVarint(reader shared.SlickReader) (uint64, error) { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflow - } - b, err := reader.Readn1() - if err != nil { - return 0, xerrors.Errorf("protobuf: unexpected read error: %w", err) - } - v |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - return v, nil -}