feat: update go-libipfs and switch gateway code to use go-libipfs imp…

…lementation
ipfs · Mar 9, 2023 · 80d8ea2 · 80d8ea2
1 parent 40d7f2f
commit 80d8ea2
Show file tree

Hide file tree

Showing 5 changed files with 110 additions and 68 deletions.
diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go
@@ -2,20 +2,25 @@ package corehttp
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 
-	cid "github.com/ipfs/go-cid"
-	"github.com/ipfs/go-libipfs/blocks"
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	offline "github.com/ipfs/go-ipfs-exchange-offline"
+	offlineroute "github.com/ipfs/go-ipfs-routing/offline"
 	"github.com/ipfs/go-libipfs/files"
-	"github.com/ipfs/go-libipfs/gateway"
 	"github.com/ipfs/go-namesys"
 	iface "github.com/ipfs/interface-go-ipfs-core"
-	options "github.com/ipfs/interface-go-ipfs-core/options"
-	nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys"
 	"github.com/ipfs/interface-go-ipfs-core/path"
+	"github.com/ipfs/kubo/core/node"
+	"github.com/libp2p/go-libp2p/core/routing"
+
+	"github.com/ipfs/go-libipfs/gateway"
+	options "github.com/ipfs/interface-go-ipfs-core/options"
 	version "github.com/ipfs/kubo"
 	config "github.com/ipfs/kubo/config"
 	core "github.com/ipfs/kubo/core"
@@ -47,7 +52,7 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
 			Headers: headers,
 		}
 
-		gwAPI, err := newGatewayAPI(n)
+		gwAPI, err := newGatewayBackend(n)
 		if err != nil {
 			return nil, err
 		}
@@ -95,7 +100,7 @@ func HostnameOption() ServeOption {
 			return nil, err
 		}
 
-		gwAPI, err := newGatewayAPI(n)
+		gwAPI, err := newGatewayBackend(n)
 		if err != nil {
 			return nil, err
 		}
@@ -118,83 +123,120 @@ func VersionOption() ServeOption {
 	}
 }
 
-type gatewayAPI struct {
-	ns         namesys.NameSystem
-	api        iface.CoreAPI
-	offlineAPI iface.CoreAPI
-}
-
-func newGatewayAPI(n *core.IpfsNode) (*gatewayAPI, error) {
+func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
 	cfg, err := n.Repo.Config()
 	if err != nil {
 		return nil, err
 	}
 
-	api, err := coreapi.NewCoreAPI(n, options.Api.FetchBlocks(!cfg.Gateway.NoFetch))
-	if err != nil {
-		return nil, err
+	bserv := n.Blocks
+	var vsRouting routing.ValueStore = n.Routing
+	nsys := n.Namesys
+	if cfg.Gateway.NoFetch {
+		bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore()))
+
+		cs := cfg.Ipns.ResolveCacheSize
+		if cs == 0 {
+			cs = node.DefaultIpnsCacheSize
+		}
+		if cs < 0 {
+			return nil, fmt.Errorf("cannot specify negative resolve cache size")
+		}
+
+		vsRouting = offlineroute.NewOfflineRouter(n.Repo.Datastore(), n.RecordValidator)
+		nsys, err = namesys.NewNameSystem(vsRouting,
+			namesys.WithDatastore(n.Repo.Datastore()),
+			namesys.WithDNSResolver(n.DNSResolver),
+			namesys.WithCache(cs))
+		if err != nil {
+			return nil, fmt.Errorf("error constructing namesys: %w", err)
+		}
 	}
-	offlineAPI, err := api.WithOptions(options.Api.Offline(true))
+
+	gw, err := gateway.NewBlocksGateway(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
 	if err != nil {
 		return nil, err
 	}
+	return &offlineGatewayErrWrapper{gwimpl: gw}, nil
+}
+
+type offlineGatewayErrWrapper struct {
+	gwimpl gateway.IPFSBackend
+}
+
+func offlineErrWrap(err error) error {
+	if errors.Is(err, iface.ErrOffline) {
+		return fmt.Errorf("%s : %w", err.Error(), gateway.ErrServiceUnavailable)
+	}
+	return err
+}
 
-	return &gatewayAPI{
-		ns:         n.Namesys,
-		api:        api,
-		offlineAPI: offlineAPI,
-	}, nil
+func (o *offlineGatewayErrWrapper) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
+	md, n, err := o.gwimpl.Get(ctx, path)
+	err = offlineErrWrap(err)
+	return md, n, err
 }
 
-func (gw *gatewayAPI) GetUnixFsNode(ctx context.Context, pth path.Resolved) (files.Node, error) {
-	return gw.api.Unixfs().Get(ctx, pth)
+func (o *offlineGatewayErrWrapper) GetRange(ctx context.Context, path gateway.ImmutablePath, ranges ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) {
+	md, n, err := o.gwimpl.GetRange(ctx, path, ranges...)
+	err = offlineErrWrap(err)
+	return md, n, err
 }
 
-func (gw *gatewayAPI) LsUnixFsDir(ctx context.Context, pth path.Resolved) (<-chan iface.DirEntry, error) {
-	// Optimization: use Unixfs.Ls without resolving children, but using the
-	// cumulative DAG size as the file size. This allows for a fast listing
-	// while keeping a good enough Size field.
-	return gw.api.Unixfs().Ls(ctx, pth,
-		options.Unixfs.ResolveChildren(false),
-		options.Unixfs.UseCumulativeSize(true),
-	)
+func (o *offlineGatewayErrWrapper) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
+	md, n, err := o.gwimpl.GetAll(ctx, path)
+	err = offlineErrWrap(err)
+	return md, n, err
 }
 
-func (gw *gatewayAPI) GetBlock(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
-	r, err := gw.api.Block().Get(ctx, path.IpfsPath(cid))
-	if err != nil {
-		return nil, err
-	}
+func (o *offlineGatewayErrWrapper) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) {
+	md, n, err := o.gwimpl.GetBlock(ctx, path)
+	err = offlineErrWrap(err)
+	return md, n, err
+}
 
-	data, err := io.ReadAll(r)
-	if err != nil {
-		return nil, err
-	}
+func (o *offlineGatewayErrWrapper) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
+	md, n, err := o.gwimpl.Head(ctx, path)
+	err = offlineErrWrap(err)
+	return md, n, err
+}
 
-	return blocks.NewBlockWithCid(data, cid)
+func (o *offlineGatewayErrWrapper) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) {
+	md, err := o.gwimpl.ResolvePath(ctx, path)
+	err = offlineErrWrap(err)
+	return md, err
 }
 
-func (gw *gatewayAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
-	return gw.api.Routing().Get(ctx, "/ipns/"+c.String())
+func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) {
+	md, data, errCh, err := o.gwimpl.GetCAR(ctx, path)
+	err = offlineErrWrap(err)
+	return md, data, errCh, err
 }
 
-func (gw *gatewayAPI) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) {
-	p, err := gw.ns.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1))
-	if err == namesys.ErrResolveRecursion {
-		err = nil
-	}
-	return path.New(p.String()), err
+func (o *offlineGatewayErrWrapper) IsCached(ctx context.Context, path path.Path) bool {
+	return o.gwimpl.IsCached(ctx, path)
+}
+
+func (o *offlineGatewayErrWrapper) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
+	rec, err := o.gwimpl.GetIPNSRecord(ctx, c)
+	err = offlineErrWrap(err)
+	return rec, err
 }
 
-func (gw *gatewayAPI) IsCached(ctx context.Context, pth path.Path) bool {
-	_, err := gw.offlineAPI.Block().Stat(ctx, pth)
-	return err == nil
+func (o *offlineGatewayErrWrapper) ResolveMutable(ctx context.Context, path path.Path) (gateway.ImmutablePath, error) {
+	imPath, err := o.gwimpl.ResolveMutable(ctx, path)
+	err = offlineErrWrap(err)
+	return imPath, err
 }
 
-func (gw *gatewayAPI) ResolvePath(ctx context.Context, pth path.Path) (path.Resolved, error) {
-	return gw.api.ResolvePath(ctx, pth)
+func (o *offlineGatewayErrWrapper) GetDNSLinkRecord(ctx context.Context, s string) (path.Path, error) {
+	p, err := o.gwimpl.GetDNSLinkRecord(ctx, s)
+	err = offlineErrWrap(err)
+	return p, err
 }
 
+var _ gateway.IPFSBackend = (*offlineGatewayErrWrapper)(nil)
+
 var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/"}
 
 var subdomainGatewaySpec = &gateway.Specification{

diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod
@@ -7,7 +7,7 @@ go 1.18
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/go-libipfs v0.6.1
+	github.com/ipfs/go-libipfs v0.6.2-0.20230309100404-db8b79eb52d9
 	github.com/ipfs/interface-go-ipfs-core v0.11.0
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
 	github.com/libp2p/go-libp2p v0.26.2
@@ -103,7 +103,7 @@ require (
 	github.com/ipfs/go-namesys v0.7.0 // indirect
 	github.com/ipfs/go-path v0.3.1 // indirect
 	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
-	github.com/ipfs/go-unixfs v0.4.4 // indirect
+	github.com/ipfs/go-unixfs v0.4.5-0.20230309101008-2acd046af3c6 // indirect
 	github.com/ipfs/go-unixfsnode v1.5.2 // indirect
 	github.com/ipfs/go-verifcid v0.0.2 // indirect
 	github.com/ipld/edelweiss v0.2.0 // indirect

diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum
@@ -569,8 +569,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2
 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg=
 github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A=
 github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24=
-github.com/ipfs/go-libipfs v0.6.1 h1:OSO9cm1H3r4OXfP0MP1Q5UhTnhd2fByGl6CVYyz/Rhk=
-github.com/ipfs/go-libipfs v0.6.1/go.mod h1:FmhKgxMOQA572TK5DA3MZ5GL44ZqsMHIrkgK4gLn4A8=
+github.com/ipfs/go-libipfs v0.6.2-0.20230309100404-db8b79eb52d9 h1:e7GyGBmss/k/stuzpawdBtVJl6fmcTNK/zZ9wMFT9W8=
+github.com/ipfs/go-libipfs v0.6.2-0.20230309100404-db8b79eb52d9/go.mod h1:uC9VqJdOglX/UMYH4vX3R64ab0TqEj1qVsT4ncEQTFM=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
 github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
 github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
@@ -606,8 +606,8 @@ github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVzte
 github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
 github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
 github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o=
-github.com/ipfs/go-unixfs v0.4.4 h1:D/dLBOJgny5ZLIur2vIXVQVW0EyDHdOMBDEhgHrt6rY=
-github.com/ipfs/go-unixfs v0.4.4/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM=
+github.com/ipfs/go-unixfs v0.4.5-0.20230309101008-2acd046af3c6 h1:YEerKN2HOZ59r9v24UttrIRyUBBAeZyaFYpjm6PsTMc=
+github.com/ipfs/go-unixfs v0.4.5-0.20230309101008-2acd046af3c6/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM=
 github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s=
 github.com/ipfs/go-unixfsnode v1.5.2 h1:CvsiTt58W2uR5dD8bqQv+aAY0c1qolmXmSyNbPHYiew=
 github.com/ipfs/go-unixfsnode v1.5.2/go.mod h1:NlOebRwYx8lMCNMdhAhEspYPBD3obp7TE0LvBqHY+ks=

diff --git a/go.mod b/go.mod
@@ -45,7 +45,7 @@ require (
 	github.com/ipfs/go-ipld-git v0.1.1
 	github.com/ipfs/go-ipld-legacy v0.1.1
 	github.com/ipfs/go-ipns v0.3.0
-	github.com/ipfs/go-libipfs v0.6.1
+	github.com/ipfs/go-libipfs v0.6.2-0.20230309100404-db8b79eb52d9
 	github.com/ipfs/go-log v1.0.5
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipfs/go-merkledag v0.9.0
@@ -55,7 +55,7 @@ require (
 	github.com/ipfs/go-namesys v0.7.0
 	github.com/ipfs/go-path v0.3.1
 	github.com/ipfs/go-pinning-service-http-client v0.1.2
-	github.com/ipfs/go-unixfs v0.4.4
+	github.com/ipfs/go-unixfs v0.4.5-0.20230309101008-2acd046af3c6
 	github.com/ipfs/go-unixfsnode v1.5.2
 	github.com/ipfs/go-verifcid v0.0.2
 	github.com/ipfs/interface-go-ipfs-core v0.11.0

diff --git a/go.sum b/go.sum
@@ -591,8 +591,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2
 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg=
 github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A=
 github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24=
-github.com/ipfs/go-libipfs v0.6.1 h1:OSO9cm1H3r4OXfP0MP1Q5UhTnhd2fByGl6CVYyz/Rhk=
-github.com/ipfs/go-libipfs v0.6.1/go.mod h1:FmhKgxMOQA572TK5DA3MZ5GL44ZqsMHIrkgK4gLn4A8=
+github.com/ipfs/go-libipfs v0.6.2-0.20230309100404-db8b79eb52d9 h1:e7GyGBmss/k/stuzpawdBtVJl6fmcTNK/zZ9wMFT9W8=
+github.com/ipfs/go-libipfs v0.6.2-0.20230309100404-db8b79eb52d9/go.mod h1:uC9VqJdOglX/UMYH4vX3R64ab0TqEj1qVsT4ncEQTFM=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
 github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
 github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
@@ -632,8 +632,8 @@ github.com/ipfs/go-pinning-service-http-client v0.1.2 h1:jdr7KelhL9gNHTU8jbqPMwI
 github.com/ipfs/go-pinning-service-http-client v0.1.2/go.mod h1:6wd5mjYhXJTiWU8b4RSWPpWdlzE5/csoXV0dWWMjun4=
 github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
 github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o=
-github.com/ipfs/go-unixfs v0.4.4 h1:D/dLBOJgny5ZLIur2vIXVQVW0EyDHdOMBDEhgHrt6rY=
-github.com/ipfs/go-unixfs v0.4.4/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM=
+github.com/ipfs/go-unixfs v0.4.5-0.20230309101008-2acd046af3c6 h1:YEerKN2HOZ59r9v24UttrIRyUBBAeZyaFYpjm6PsTMc=
+github.com/ipfs/go-unixfs v0.4.5-0.20230309101008-2acd046af3c6/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM=
 github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s=
 github.com/ipfs/go-unixfsnode v1.5.2 h1:CvsiTt58W2uR5dD8bqQv+aAY0c1qolmXmSyNbPHYiew=
 github.com/ipfs/go-unixfsnode v1.5.2/go.mod h1:NlOebRwYx8lMCNMdhAhEspYPBD3obp7TE0LvBqHY+ks=