From 96439c930ea8614fc21df053431f76f82e36a797 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 1 Apr 2021 12:46:06 -0700 Subject: [PATCH] feat(fetcher): add on demand prototype chooser augmentation --- README.md | 2 +- fetcher.go | 16 +++++++- fetcher_test.go | 102 ++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 2 + 5 files changed, 121 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7039f39..71def50 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ go-fetcher [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) -Go-fetcher is a library to retrieve IPLD prime nodes from IPFS using Bitswap. +Go-fetcher is a library to retrieve IPLD prime nodes from IPFS using data exchange protocols ## Contribute diff --git a/fetcher.go b/fetcher.go index a4ba0e7..d1765fa 100644 --- a/fetcher.go +++ b/fetcher.go @@ -17,11 +17,19 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector/builder" ) +// AugmentChooserFunc is a function that can augment a prototype chooser at the time the Fetcher is initialized, +// which is given the linksystem the fetcher itself will use +type AugmentChooserFunc func(*ipld.LinkSystem, traversal.LinkTargetNodePrototypeChooser) traversal.LinkTargetNodePrototypeChooser + +// FetcherConfig defines a configuration object from which Fetcher instances are constructed type FetcherConfig struct { blockService blockservice.BlockService + AugmentChooser AugmentChooserFunc PrototypeChooser traversal.LinkTargetNodePrototypeChooser } +// Fetcher is an interface for reading from a dag. Reads may be local or remote, and may employ data exchange +// protocols like graphsync and bitswap type Fetcher interface { // NodeMatching traverses a node graph starting with the provided node using the given selector and possibly crossing // block boundaries. Each matched node is passed as FetchResult to the callback. Errors returned from callback will @@ -48,6 +56,7 @@ type fetcherSession struct { protoChooser traversal.LinkTargetNodePrototypeChooser } +// FetchResult is a single node read as part of a dag operation called on a fetcher type FetchResult struct { Node ipld.Node Path ipld.Path @@ -55,6 +64,7 @@ type FetchResult struct { LastBlockLink ipld.Link } +// FetchCallback is called for each node traversed during a fetch type FetchCallback func(result FetchResult) error // NewFetcherConfig creates a FetchConfig from which session may be created and nodes retrieved. @@ -70,7 +80,11 @@ func NewFetcherConfig(blockService blockservice.BlockService) FetcherConfig { func (fc FetcherConfig) NewSession(ctx context.Context) Fetcher { ls := cidlink.DefaultLinkSystem() ls.StorageReadOpener = blockOpener(ctx, blockservice.NewSession(ctx, fc.blockService)) - return &fetcherSession{linkSystem: ls, protoChooser: fc.PrototypeChooser} + protoChooser := fc.PrototypeChooser + if fc.AugmentChooser != nil { + protoChooser = fc.AugmentChooser(&ls, protoChooser) + } + return &fetcherSession{linkSystem: ls, protoChooser: protoChooser} } // BlockOfType fetches a node graph of the provided type corresponding to single block by link. diff --git a/fetcher_test.go b/fetcher_test.go index 5287ef6..83205ce 100644 --- a/fetcher_test.go +++ b/fetcher_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/ipld/go-ipld-prime/traversal" "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" @@ -290,3 +291,104 @@ func assertNodesInOrder(t *testing.T, results []fetcher.FetchResult, nodeCount i assert.Equal(t, nodeCount, len(results)) } + +type selfLoader struct { + ipld.Node + ctx context.Context + ls *ipld.LinkSystem +} + +func (sl *selfLoader) LookupByString(key string) (ipld.Node, error) { + nd, err := sl.Node.LookupByString(key) + if err != nil { + return nd, err + } + if nd.Kind() == ipld.Kind_Link { + lnk, _ := nd.AsLink() + nd, err = sl.ls.Load(ipld.LinkContext{Ctx: sl.ctx}, lnk, basicnode.Prototype.Any) + } + return nd, err +} + +type selfLoadPrototype struct { + ctx context.Context + ls *ipld.LinkSystem + basePrototype ipld.NodePrototype +} + +func (slp *selfLoadPrototype) NewBuilder() ipld.NodeBuilder { + return &selfLoadBuilder{ctx: slp.ctx, NodeBuilder: slp.basePrototype.NewBuilder(), ls: slp.ls} +} + +type selfLoadBuilder struct { + ctx context.Context + ipld.NodeBuilder + ls *ipld.LinkSystem +} + +func (slb *selfLoadBuilder) Build() ipld.Node { + nd := slb.NodeBuilder.Build() + return &selfLoader{nd, slb.ctx, slb.ls} +} + +func TestChooserAugmentation(t *testing.T) { + // demonstrates how to use the augment chooser to build an ADL that self loads its own nodes + block3, node3, link3 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("three").AssignBool(true) + })) + block4, node4, link4 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) { + na.AssembleEntry("four").AssignBool(true) + })) + block2, _, _ := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) { + na.AssembleEntry("link3").AssignLink(link3) + na.AssembleEntry("link4").AssignLink(link4) + })) + + net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0*time.Millisecond)) + ig := testinstance.NewTestInstanceGenerator(net, nil, nil) + defer ig.Close() + + peers := ig.Instances(2) + hasBlock := peers[0] + defer hasBlock.Exchange.Close() + + err := hasBlock.Exchange.HasBlock(block2) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(block3) + require.NoError(t, err) + err = hasBlock.Exchange.HasBlock(block4) + require.NoError(t, err) + + wantsBlock := peers[1] + defer wantsBlock.Exchange.Close() + + wantsGetter := blockservice.New(wantsBlock.Blockstore(), wantsBlock.Exchange) + fetcherConfig := fetcher.NewFetcherConfig(wantsGetter) + augmentChooser := func(ls *ipld.LinkSystem, base traversal.LinkTargetNodePrototypeChooser) traversal.LinkTargetNodePrototypeChooser { + return func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + np, err := base(lnk, lnkCtx) + if err != nil { + return np, err + } + return &selfLoadPrototype{ctx: lnkCtx.Ctx, ls: ls, basePrototype: np}, nil + } + } + fetcherConfig.AugmentChooser = augmentChooser + session := fetcherConfig.NewSession(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + retrievedNode, err := fetcher.Block(ctx, session, cidlink.Link{Cid: block2.Cid()}) + require.NoError(t, err) + + // instead of getting links back, we automatically load the nodes + + retrievedNode3, err := retrievedNode.LookupByString("link3") + require.NoError(t, err) + assert.Equal(t, node3, retrievedNode3) + + retrievedNode4, err := retrievedNode.LookupByString("link4") + require.NoError(t, err) + assert.Equal(t, node4, retrievedNode4) + +} diff --git a/go.mod b/go.mod index 296b803..23dab07 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,6 @@ require ( github.com/ipfs/go-ipfs-delay v0.0.1 github.com/ipfs/go-ipfs-routing v0.1.0 github.com/ipld/go-codec-dagpb v1.2.0 - github.com/ipld/go-ipld-prime v0.9.0 + github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db github.com/stretchr/testify v1.6.1 ) diff --git a/go.sum b/go.sum index ec5919e..4958bc1 100644 --- a/go.sum +++ b/go.sum @@ -193,6 +193,8 @@ github.com/ipld/go-codec-dagpb v1.2.0 h1:2umV7ud8HBMkRuJgd8gXw95cLhwmcYrihS3cQEy github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-ipld-prime v0.9.0 h1:N2OjJMb+fhyFPwPnVvJcWU/NsumP8etal+d2v3G4eww= github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db h1:kFwGn8rXa/Z31ev1OFNQsYeNKNCdifnTPl/NvPy5L38= +github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=