Add ability to read a file through ProofReader.

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/ipfs/go-ipfs-util v0.0.1
 	github.com/ipfs/go-ipld-format v0.0.1
 	github.com/ipfs/go-merkledag v0.0.3
+	github.com/ipfs/interface-go-ipfs-core v0.0.6-0.20190326010311-c4e66131e60c
 	github.com/multiformats/go-multihash v0.0.1
 	github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72
 )

go.sum

@@ -14,6 +14,7 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0=
 github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis=
 github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
@@ -52,14 +53,11 @@ github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324 h1:PV190X5/DzQ/tbFFG5Y
 github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag=
 github.com/ipfs/bbloom v0.0.1 h1:s7KkiBPfxCeDVo47KySjK0ACPc5GJRUxFpdyWEuDjhw=
 github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI=
-github.com/ipfs/go-bitswap v0.0.1 h1:Xx1ma7TWy9ISOx5zFq5YVQyrTHzUP4GkRPMsZokHxAg=
-github.com/ipfs/go-bitswap v0.0.1/go.mod h1:z+tP3h+HTJ810n1R5yMy2ccKFffJ2F6Vqm/5Bf7vs2c=
+github.com/ipfs/go-bitswap v0.0.3 h1:uFcSI9dkjUn67S7IM60vr2wA27aAvn8o9xYjaQCug3o=
 github.com/ipfs/go-bitswap v0.0.3/go.mod h1:jadAZYsP/tcRMl47ZhFxhaNuDQoXawT8iHMg+iFoQbg=
 github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc=
 github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
-github.com/ipfs/go-blockservice v0.0.1 h1:l6g1hwYDV6vb4bAvTqia6Cvo+zLMOPKel/n0zUX48bc=
-github.com/ipfs/go-blockservice v0.0.1/go.mod h1:2Ao89U7jV1KIqqNk5EdhSTBG/Pgc1vMFr0bhkx376j4=
 github.com/ipfs/go-blockservice v0.0.3 h1:40OvwrxeudTAlUGUAKNYnNPcwQeLtXedjzTWecnUinQ=
 github.com/ipfs/go-blockservice v0.0.3/go.mod h1:/NNihwTi6V2Yr6g8wBI+BSwPuURpBRMtYNGrlxZ8KuI=
 github.com/ipfs/go-cid v0.0.1 h1:GBjWPktLnNyX0JiQCNFpUuUSoMw5KMyqrsejHYlILBE=
@@ -85,8 +83,6 @@ github.com/ipfs/go-ipfs-exchange-interface v0.0.1 h1:LJXIo9W7CAmugqI+uofioIpRb6r
 github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM=
 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C81I1NSHW1FxGew=
 github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0=
-github.com/ipfs/go-ipfs-files v0.0.1 h1:OroTsI58plHGX70HPLKy6LQhPR3HZJ5ip61fYlo6POM=
-github.com/ipfs/go-ipfs-files v0.0.1/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
 github.com/ipfs/go-ipfs-files v0.0.2 h1:fEEjF4H+1t8SFOHqUGp0KqcwgIRlbD2bu8CAS2sIggE=
 github.com/ipfs/go-ipfs-files v0.0.2/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
 github.com/ipfs/go-ipfs-flags v0.0.1 h1:OH5cEkJYL0QgA+bvD55TNG9ud8HA2Nqaav47b2c/UJk=
@@ -105,14 +101,17 @@ github.com/ipfs/go-ipld-format v0.0.1 h1:HCu4eB/Gh+KD/Q0M8u888RFkorTWNIL3da4oc5d
 github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms=
 github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
-github.com/ipfs/go-merkledag v0.0.1 h1:HqvQsqqLvNOgItOy80Sd4T4rHvq6cXtAtrbEoWAON+I=
-github.com/ipfs/go-merkledag v0.0.1/go.mod h1:CRdtHMROECqaehAGeJ0Wd9TtlmWv/ta5cUnvbTnniEI=
 github.com/ipfs/go-merkledag v0.0.3 h1:A5DlOMzqTRDVmdgkf3dzCKCFmVWH4Zqwb0cbYXUs+Ro=
 github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA=
 github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
+github.com/ipfs/go-path v0.0.3 h1:G/VFcCMXtp36JUPPyytYQ1I3UsBUBf47M//uSdTLnFg=
+github.com/ipfs/go-path v0.0.3/go.mod h1:zIRQUez3LuQIU25zFjC2hpBTHimWx7VK5bjZgRLbbdo=
+github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8=
 github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
+github.com/ipfs/interface-go-ipfs-core v0.0.6-0.20190326010311-c4e66131e60c h1:caBRZYmSPNCrAoFjlg0pEQyVzsWEc2bY3lQddV7Hz7c=
+github.com/ipfs/interface-go-ipfs-core v0.0.6-0.20190326010311-c4e66131e60c/go.mod h1:VceUOYu+kPEy8Ev/gAhzXFTIfc/7xILKnL4fgZg8tZM=
 github.com/jackpal/gateway v1.0.4 h1:LS5EHkLuQ6jzaHwULi0vL+JO0mU/n4yUtK8oUjHHOlM=
 github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
 github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA=
@@ -145,10 +144,8 @@ github.com/libp2p/go-conn-security-multistream v0.0.1 h1:XefjAQRHcnUaxKb26RGupTo
 github.com/libp2p/go-conn-security-multistream v0.0.1/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE=
 github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s=
 github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8=
-github.com/libp2p/go-libp2p v0.0.1 h1:oEjzhRzymY7NILTMzOKNVOpNZGpcQHrHriWw91pbJNM=
-github.com/libp2p/go-libp2p v0.0.1/go.mod h1:bmRs8I0vwn6iRaVssZnJx/epY6WPSKiLoK1vyle4EX0=
+github.com/libp2p/go-libp2p v0.0.2 h1:+jvgi0Zy3y4TKXJKApchCk3pCBPZf1T54z3+vKie3gw=
 github.com/libp2p/go-libp2p v0.0.2/go.mod h1:Qu8bWqFXiocPloabFGUcVG4kk94fLvfC8mWTDdFC9wE=
-github.com/libp2p/go-libp2p-autonat v0.0.1/go.mod h1:fs71q5Xk+pdnKU014o2iq1RhMs9/PMaG5zXRFNnIIT4=
 github.com/libp2p/go-libp2p-autonat v0.0.2/go.mod h1:fs71q5Xk+pdnKU014o2iq1RhMs9/PMaG5zXRFNnIIT4=
 github.com/libp2p/go-libp2p-blankhost v0.0.1 h1:/mZuuiwntNR8RywnCFlGHLKrKLYne+qciBpQXWqp5fk=
 github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc=
@@ -166,8 +163,7 @@ github.com/libp2p/go-libp2p-loggables v0.0.1 h1:HVww9oAnINIxbt69LJNkxD8lnbfgteXR
 github.com/libp2p/go-libp2p-loggables v0.0.1/go.mod h1:lDipDlBNYbpyqyPX/KcoO+eq0sJYEVR2JgOexcivchg=
 github.com/libp2p/go-libp2p-metrics v0.0.1 h1:yumdPC/P2VzINdmcKZd0pciSUCpou+s0lwYCjBbzQZU=
 github.com/libp2p/go-libp2p-metrics v0.0.1/go.mod h1:jQJ95SXXA/K1VZi13h52WZMa9ja78zjyy5rspMsC/08=
-github.com/libp2p/go-libp2p-nat v0.0.1 h1:on/zju7XE+JXc8gH+vTKmIh2UJFC1K8kGnJYluQrlz4=
-github.com/libp2p/go-libp2p-nat v0.0.1/go.mod h1:4L6ajyUIlJvx1Cbh5pc6Ma6vMDpKXf3GgLO5u7W0oQ4=
+github.com/libp2p/go-libp2p-nat v0.0.2 h1:sKI5hiCsGFhuEKdXMsF9mywQu2qhfoIGX6a+VG6zelE=
 github.com/libp2p/go-libp2p-nat v0.0.2/go.mod h1:QrjXQSD5Dj4IJOdEcjHRkWTSomyxRo6HnUkf/TfQpLQ=
 github.com/libp2p/go-libp2p-net v0.0.1 h1:xJ4Vh4yKF/XKb8fd1Ev0ebAGzVjMxXzrxG2kjtU+F5Q=
 github.com/libp2p/go-libp2p-net v0.0.1/go.mod h1:Yt3zgmlsHOgUWSXmt5V/Jpz9upuJBE8EgNU9DrCcR8c=
@@ -273,6 +269,7 @@ github.com/whyrusleeping/go-smux-multistream v2.0.2+incompatible h1:BdYHctE9HJZL
 github.com/whyrusleeping/go-smux-multistream v2.0.2+incompatible/go.mod h1:dRWHHvc4HDQSHh9gbKEBbUZ+f2Q8iZTPG3UOGYODxSQ=
 github.com/whyrusleeping/go-smux-yamux v2.0.8+incompatible h1:IGm/UP/JpEFS6D787sZnZg7RA6fZIR9c/Ms9DeAVNuk=
 github.com/whyrusleeping/go-smux-yamux v2.0.8+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI=
+github.com/whyrusleeping/go-smux-yamux v2.0.9+incompatible h1:nVkExQ7pYlN9e45LcqTCOiDD0904fjtm0flnHZGbXkw=
 github.com/whyrusleeping/go-smux-yamux v2.0.9+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI=
 github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA=
 github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA=

hamt/hamt.go

@@ -30,6 +30,7 @@ import (
 	ipld "github.com/ipfs/go-ipld-format"
 	dag "github.com/ipfs/go-merkledag"
 	format "github.com/ipfs/go-unixfs"
+	coreiface "github.com/ipfs/interface-go-ipfs-core"
 )
 
 const (
@@ -61,6 +62,8 @@ type Shard struct {
 	// leaf node
 	key string
 	val *ipld.Link
+
+	rawData []byte
 }
 
 // NewShard creates a new, empty HAMT shard with the given size.
@@ -126,6 +129,7 @@ func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*Shard, error) {
 	ds.cid = pbnd.Cid()
 	ds.hashFunc = fsn.HashType()
 	ds.builder = pbnd.CidBuilder()
+	ds.rawData = pbnd.RawData()
 
 	return ds, nil
 }
@@ -280,17 +284,33 @@ func (ds *Shard) Link() (*ipld.Link, error) {
 	return ipld.MakeLink(nd)
 }
 
+func hamtChunk(name string, rawData []byte) []byte {
+	buff := make([]byte, 4+len(name)+len(rawData))
+	buff[0] = 1
+	buff[1] = byte(len(name) >> 16)
+	buff[2] = byte(len(name) >> 8)
+	buff[3] = byte(len(name))
+	copy(buff[4:], []byte(name))
+	copy(buff[4+len(name):], rawData)
+
+	return buff
+}
+
 func (ds *Shard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*Shard) error) error {
 	childIndex, err := hv.Next(ds.tableSizeLg2)
 	if err != nil {
 		return err
 	}
+	sliceIndex := ds.childer.sliceIndex(childIndex)
 
 	if ds.childer.has(childIndex) {
-		child, err := ds.childer.get(ctx, ds.childer.sliceIndex(childIndex))
+		child, err := ds.childer.get(ctx, sliceIndex)
 		if err != nil {
 			return err
 		}
+		if pw, ok := ctx.Value("proxy-preamble").(coreiface.ProofWriter); ok {
+			pw.WriteChunk(hamtChunk(ds.childer.link(sliceIndex).Name, ds.rawData))
+		}
 
 		if child.isValueNode() {
 			if child.key == key {

io/proofreader.go

@@ -0,0 +1,99 @@
+package io
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	ipld "github.com/ipfs/go-ipld-format"
+	dag "github.com/ipfs/go-merkledag"
+	mdag "github.com/ipfs/go-merkledag"
+	unixfs "github.com/ipfs/go-unixfs"
+	pb "github.com/ipfs/go-unixfs/pb"
+	coreiface "github.com/ipfs/interface-go-ipfs-core"
+)
+
+// NewDagReaderWithProof creates a new proof reader object that reads the data
+// represented by the given node, using the passed in DAGService for data
+// retrieval.
+func NewDagReaderWithProof(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (coreiface.ProofReader, error) {
+	switch n := n.(type) {
+	case *mdag.RawNode:
+	case *mdag.ProtoNode:
+		fsNode, err := unixfs.FSNodeFromBytes(n.Data())
+		if err != nil {
+			return nil, err
+		}
+
+		switch fsNode.Type() {
+		case unixfs.TFile, unixfs.TRaw:
+		case unixfs.TDirectory, unixfs.THAMTShard:
+			return nil, ErrIsDir
+		case unixfs.TSymlink:
+			return nil, ErrCantReadSymlinks
+		default:
+			return nil, unixfs.ErrUnrecognizedType
+		}
+	default:
+		return nil, ErrUnkownNodeType
+	}
+
+	ctxWithCancel, cancel := context.WithCancel(ctx)
+
+	return &proofReader{
+		cancel:    cancel,
+		dagWalker: ipld.NewWalker(ctxWithCancel, ipld.NewNavigableIPLDNode(n, serv)),
+	}, nil
+}
+
+// proofReader provides a way to easily read the data contained in a dag, in a
+// way that it can be presented to an untrusting client.
+type proofReader struct {
+	dagWalker *ipld.Walker
+	cancel    func()
+}
+
+func (pr *proofReader) ReadChunk() ([]byte, error) {
+	var out []byte
+
+	err := pr.dagWalker.Iterate(func(visitedNode ipld.NavigableNode) error {
+		node := ipld.ExtractIPLDNode(visitedNode)
+
+		switch node := node.(type) {
+		case *dag.RawNode:
+			out = append([]byte{0}, node.RawData()...)
+
+		case *dag.ProtoNode:
+			fsNode, err := unixfs.FSNodeFromBytes(node.Data())
+			if err != nil {
+				return fmt.Errorf("incorrectly formatted protobuf: %s", err)
+			}
+
+			switch fsNode.Type() {
+			case pb.Data_File, pb.Data_Raw:
+				out = append([]byte{1}, node.RawData()...)
+			default:
+				return fmt.Errorf("found %s node in unexpected place",
+					fsNode.Type().String())
+			}
+
+		default:
+			return unixfs.ErrUnrecognizedType
+		}
+
+		pr.dagWalker.Pause()
+		return nil
+	})
+
+	if err == ipld.EndOfDag {
+		return nil, io.EOF
+	} else if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (pr *proofReader) Close() error {
+	pr.cancel()
+	return nil
+}

io/resolve.go

@@ -8,6 +8,7 @@ import (
 	hamt "github.com/ipfs/go-unixfs/hamt"
 
 	ipld "github.com/ipfs/go-ipld-format"
+	coreiface "github.com/ipfs/interface-go-ipfs-core"
 )
 
 // ResolveUnixfsOnce resolves a single hop of a path through a graph in a
@@ -16,12 +17,7 @@ func ResolveUnixfsOnce(ctx context.Context, ds ipld.NodeGetter, nd ipld.Node, na
 	pn, ok := nd.(*dag.ProtoNode)
 	if ok {
 		fsn, err := ft.FSNodeFromBytes(pn.Data())
-		if err != nil {
-			// Not a unixfs node, use standard object traversal code
-			return nd.ResolveLink(names)
-		}
-
-		if fsn.Type() == ft.THAMTShard {
+		if err == nil && fsn.Type() == ft.THAMTShard {
 			rods := dag.NewReadOnlyDagService(ds)
 			s, err := hamt.NewHamtFromDag(rods, nd)
 			if err != nil {
@@ -37,5 +33,8 @@ func ResolveUnixfsOnce(ctx context.Context, ds ipld.NodeGetter, nd ipld.Node, na
 		}
 	}
 
+	if pw, ok := ctx.Value("proxy-preamble").(coreiface.ProofWriter); ok {
+		pw.WriteChunk(append([]byte{0}, nd.RawData()...))
+	}
 	return nd.ResolveLink(names)
 }