feat[gateway]: sync latest ipfs gateway

README.md

@@ -14,20 +14,28 @@ These are exactly what BTFS aims to clear up. Besides, underpinned by BTTC, BTFS
 ![Architecture Diagram](https://files.readme.io/a21e9fb--min.png)
 
 ## Table of Contents
-- [Faucet](#Faucet)
-- [Install](#install)
-  - [System Requirements](#system-requirements)
-- [Build from Source](#build-from-source)
-  - [MacOS](#macos)
-  - [Linux VM](#linux-vm)
-  - [Docker](#docker)
-- [Getting Started](#getting-started)
-  - [Some things to try](#some-things-to-try)
-  - [Usage](#usage)
-- [Development](#development)
-  - [Development Dependencies](#development-dependencies)
-  - [BTFS Gateway](#btfs-gateway)
-- [License](#license)
+- [go-btfs](#go-btfs)
+  - [What is BTFS 2.0?](#what-is-btfs-20)
+  - [BTFS 2.0 Architecture Diagram](#btfs-20-architecture-diagram)
+  - [Table of Contents](#table-of-contents)
+  - [Faucet](#faucet)
+  - [Install BTFS](#install-btfs)
+    - [System Requirements](#system-requirements)
+    - [Install Pre-Built Packages](#install-pre-built-packages)
+      - [Initialize a BTFS Daemon](#initialize-a-btfs-daemon)
+      - [Start the Daemon](#start-the-daemon)
+    - [Build from Source](#build-from-source)
+      - [Requires](#requires)
+      - [Install Go](#install-go)
+    - [Docker](#docker)
+    - [Notices](#notices)
+  - [Getting Started](#getting-started)
+    - [Some things to try](#some-things-to-try)
+    - [Usage](#usage)
+  - [Development](#development)
+    - [Development Dependencies](#development-dependencies)
+    - [BTFS Gateway](#btfs-gateway)
+  - [License](#license)
 
 ## Faucet
 
@@ -223,6 +231,14 @@ Execute commands within the docker container:
 docker exec CONTAINER btfs add FILE
 ```
 
+### Notices
+After upgrade to go-btfs v2.3.1, if you find that your node connectivity has deteriorated, you can execute the following command:
+
+```
+btfs config Swarm.ResourceMgr.Limits.System --json '{"ConnsInbound":0}'
+```
+Then restart your btfs node.
+This command will help you unblock the inbound connections to improve connectivity and thus increase the chances of getting a contract.
 ## Getting Started
 
 ### Some things to try

bigint/bigint.go

@@ -38,7 +38,7 @@ func (i *BigInt) UnmarshalJSON(b []byte) error {
 	return nil
 }
 
-//Wrap wraps big.Int pointer into BigInt struct.
+// Wrap wraps big.Int pointer into BigInt struct.
 func Wrap(i *big.Int) *BigInt {
 	return &BigInt{Int: i}
 }

bindata/bindata.go

@@ -156,11 +156,13 @@ var _bindata = map[string]func() (*asset, error){
 // directory embedded in the file by go-bindata.
 // For example if you run go-bindata on data/... and data contains the
 // following hierarchy:
-//     data/
-//       foo.txt
-//       img/
-//         a.png
-//         b.png
+//
+//	data/
+//	  foo.txt
+//	  img/
+//	    a.png
+//	    b.png
+//
 // then AssetDir("data") would return []string{"foo.txt", "img"}
 // AssetDir("data/img") would return []string{"a.png", "b.png"}
 // AssetDir("foo.txt") and AssetDir("notexist") would return an error

cmd/btfs/daemon.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	_ "expvar"
 	"fmt"
-	"github.com/bittorrent/go-btfs/chain/tokencfg"
 	"io/ioutil"
 	"math/rand"
 	"net"
@@ -22,6 +21,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/bittorrent/go-btfs/chain/tokencfg"
+
 	"github.com/bittorrent/go-btfs/guide"
 
 	config "github.com/TRON-US/go-btfs-config"
@@ -809,9 +810,9 @@ func serveHTTPApi(req *cmds.Request, cctx *oldcmds.Context, SimpleMode bool) (<-
 	// only the webui objects are allowed.
 	// if you know what you're doing, go ahead and pass --unrestricted-api.
 	unrestricted, _ := req.Options[unrestrictedApiAccessKwd].(bool)
-	gatewayOpt := corehttp.GatewayOption(false, corehttp.WebUIPaths...)
+	gatewayOpt := corehttp.GatewayOption(corehttp.WebUIPaths...)
 	if unrestricted {
-		gatewayOpt = corehttp.GatewayOption(true, "/btfs", "/btns")
+		gatewayOpt = corehttp.GatewayOption("/btfs", "/btns")
 	}
 
 	var opts = []corehttp.ServeOption{
@@ -825,7 +826,9 @@ func serveHTTPApi(req *cmds.Request, cctx *oldcmds.Context, SimpleMode bool) (<-
 		corehttp.VersionOption(),
 		defaultMux("/debug/vars"),
 		defaultMux("/debug/pprof/"),
+		defaultMux("/debug/stack"),
 		corehttp.MutexFractionOption("/debug/pprof-mutex/"),
+		corehttp.BlockProfileRateOption("/debug/pprof-block/"),
 		corehttp.MetricsScrapingOption("/debug/metrics/prometheus"),
 		corehttp.LogOption(),
 	}
@@ -962,7 +965,9 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e
 	if !writableOptionFound {
 		writable = cfg.Gateway.Writable
 	}
-
+	if writable {
+		log.Errorf("Support for Gateway.Writable and --writable has been REMOVED. Please remove it from your config file or CLI.")
+	}
 	listeners, err := sockets.TakeListeners("io.ipfs.gateway")
 	if err != nil {
 		return nil, fmt.Errorf("serveHTTPGateway: socket activation failed: %s", err)
@@ -992,14 +997,8 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e
 		listeners = append(listeners, gwLis)
 	}
 
-	// we might have listened to /tcp/0 - let's see what we are listing on
-	gwType := "readonly"
-	if writable {
-		gwType = "writable"
-	}
-
 	for _, listener := range listeners {
-		fmt.Printf("Gateway (%s) server listening on %s\n", gwType, listener.Multiaddr())
+		fmt.Printf("Gateway server listening on %s\n", listener.Multiaddr())
 	}
 
 	cmdctx := *cctx
@@ -1008,7 +1007,8 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e
 	var opts = []corehttp.ServeOption{
 		corehttp.MetricsCollectionOption("gateway"),
 		corehttp.HostnameOption(),
-		corehttp.GatewayOption(writable, "/btfs", "/btns"),
+		// TODO: rm writable
+		corehttp.GatewayOption("/btfs", "/btns"),
 		corehttp.VersionOption(),
 		corehttp.CheckVersionOption(),
 		corehttp.CommandsROOption(cmdctx),
@@ -1026,7 +1026,9 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e
 	if err != nil {
 		return nil, fmt.Errorf("serveHTTPGateway: ConstructNode() failed: %s", err)
 	}
-
+	if len(cfg.Gateway.PathPrefixes) > 0 {
+		log.Errorf("Support for custom Gateway.PathPrefixes was removed")
+	}
 	errc := make(chan error)
 	var wg sync.WaitGroup
 	for _, lis := range listeners {

cmd/ipfswatch/main.go

@@ -92,7 +92,7 @@ func run(ipfsPath, watchPath string) error {
 	if *http {
 		addr := "/ip4/127.0.0.1/tcp/5001"
 		var opts = []corehttp.ServeOption{
-			corehttp.GatewayOption(true, "/btfs", "/btns"),
+			corehttp.GatewayOption("/btfs", "/btns"),
 			corehttp.WebUIOption,
 			corehttp.DashboardOption,
 			corehttp.HostUIOption,

core/commands/dns.go

@@ -5,6 +5,7 @@ import (
 	"io"
 
 	nsopts "github.com/TRON-US/interface-go-btfs-core/options/namesys"
+	"github.com/bittorrent/go-btfs/core/commands/cmdenv"
 	ncmd "github.com/bittorrent/go-btfs/core/commands/name"
 	namesys "github.com/bittorrent/go-btfs/namesys"
 
@@ -60,9 +61,13 @@ The resolver can recursively resolve:
 		cmds.BoolOption(dnsRecursiveOptionName, "r", "Resolve until the result is not a DNS link.").WithDefault(true),
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
+		node, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
 		recursive, _ := req.Options[dnsRecursiveOptionName].(bool)
 		name := req.Arguments[0]
-		resolver := namesys.NewDNSResolver()
+		resolver := namesys.NewDNSResolver(node.DNSResolver.LookupTXT)
 
 		var routing []nsopts.ResolveOpt
 		if !recursive {

core/commands/storage/path/path.go

@@ -3,7 +3,6 @@ package path
 import (
 	"errors"
 	"fmt"
-	"github.com/bittorrent/go-btfs/utils"
 	"io"
 	"io/ioutil"
 	"os"
@@ -15,6 +14,8 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/bittorrent/go-btfs/utils"
+
 	cmds "github.com/bittorrent/go-btfs-cmds"
 
 	"github.com/dustin/go-humanize"
@@ -35,7 +36,9 @@ var (
 	srcProperties      string
 )
 
-/* can be dir of `btfs` or path like `/private/var/folders/q0/lc8cmwd93gv50ygrsy3bwfyc0000gn/T`,
+/*
+	can be dir of `btfs` or path like `/private/var/folders/q0/lc8cmwd93gv50ygrsy3bwfyc0000gn/T`,
+
 depends on how `btfs` is called
 */
 func init() {

core/core.go

@@ -16,6 +16,7 @@ import (
 	"github.com/bittorrent/go-btfs/peering"
 	irouting "github.com/bittorrent/go-btfs/routing"
 
+	mfs "github.com/TRON-US/go-mfs"
 	"github.com/bittorrent/go-btfs/core/bootstrap"
 	"github.com/bittorrent/go-btfs/core/node"
 	"github.com/bittorrent/go-btfs/core/node/libp2p"
@@ -24,9 +25,8 @@ import (
 	ipnsrp "github.com/bittorrent/go-btfs/namesys/republisher"
 	"github.com/bittorrent/go-btfs/p2p"
 	"github.com/bittorrent/go-btfs/repo"
-
-	mfs "github.com/TRON-US/go-mfs"
 	bserv "github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-fetcher"
 	"github.com/ipfs/go-filestore"
 	"github.com/ipfs/go-graphsync"
 	bstore "github.com/ipfs/go-ipfs-blockstore"
@@ -35,7 +35,6 @@ import (
 	provider "github.com/ipfs/go-ipfs-provider"
 	ipld "github.com/ipfs/go-ipld-format"
 	logging "github.com/ipfs/go-log"
-	resolver "github.com/ipfs/go-path/resolver"
 	goprocess "github.com/jbenet/goprocess"
 	ddht "github.com/libp2p/go-libp2p-kad-dht/dual"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
@@ -50,6 +49,7 @@ import (
 	discovery "github.com/libp2p/go-libp2p/p2p/discovery/mdns"
 	p2pbhost "github.com/libp2p/go-libp2p/p2p/host/basic"
 	ma "github.com/multiformats/go-multiaddr"
+	madns "github.com/multiformats/go-multiaddr-dns"
 )
 
 var log = logging.Logger("core")
@@ -69,18 +69,19 @@ type IpfsNode struct {
 	PNetFingerprint libp2p.PNetFingerprint `optional:"true"` // fingerprint of private network
 
 	// Services
-	Peerstore       pstore.Peerstore          `optional:"true"` // storage for other Peer instances
-	Blockstore      bstore.GCBlockstore       // the block store (lower level)
-	Filestore       *filestore.Filestore      `optional:"true"` // the filestore blockstore
-	BaseBlocks      node.BaseBlocks           // the raw blockstore, no filestore wrapping
-	GCLocker        bstore.GCLocker           // the locker used to protect the blockstore during gc
-	Blocks          bserv.BlockService        // the block service, get/add blocks.
-	DAG             ipld.DAGService           // the merkle dag service, get/add objects.
-	Resolver        *resolver.Resolver        // the path resolution system
-	Reporter        *metrics.BandwidthCounter `optional:"true"`
-	Discovery       discovery.Service         `optional:"true"`
-	FilesRoot       *mfs.Root
-	RecordValidator record.Validator
+	Peerstore            pstore.Peerstore          `optional:"true"` // storage for other Peer instances
+	Blockstore           bstore.GCBlockstore       // the block store (lower level)
+	Filestore            *filestore.Filestore      `optional:"true"` // the filestore blockstore
+	BaseBlocks           node.BaseBlocks           // the raw blockstore, no filestore wrapping
+	GCLocker             bstore.GCLocker           // the locker used to protect the blockstore during gc
+	Blocks               bserv.BlockService        // the block service, get/add blocks.
+	DAG                  ipld.DAGService           // the merkle dag service, get/add objects.
+	IPLDFetcherFactory   fetcher.Factory           `name:"ipldFetcher"`   // fetcher that paths over the IPLD data model
+	UnixFSFetcherFactory fetcher.Factory           `name:"unixfsFetcher"` // fetcher that interprets UnixFS data
+	Reporter             *metrics.BandwidthCounter `optional:"true"`
+	Discovery            discovery.Service         `optional:"true"`
+	FilesRoot            *mfs.Root
+	RecordValidator      record.Validator
 	//Statestore      storage.StateStorer
 
 	// Online
@@ -89,6 +90,7 @@ type IpfsNode struct {
 	Filters       *ma.Filters                `optional:"true"`
 	Bootstrapper  io.Closer                  `optional:"true"` // the periodic bootstrapper
 	Routing       irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht
+	DNSResolver   *madns.Resolver            // the DNS resolver
 	Exchange      exchange.Interface         // the block exchange + strategy (bitswap)
 	Namesys       namesys.NameSystem         // the name system, resolves paths to hashes
 	Provider      provider.System            // the value provider system

core/coreapi/coreapi.go

@@ -26,6 +26,7 @@ import (
 	coreiface "github.com/TRON-US/interface-go-btfs-core"
 	"github.com/TRON-US/interface-go-btfs-core/options"
 	bserv "github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-fetcher"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	exchange "github.com/ipfs/go-ipfs-exchange-interface"
 	offlinexch "github.com/ipfs/go-ipfs-exchange-offline"
@@ -54,13 +55,14 @@ type CoreAPI struct {
 	baseBlocks blockstore.Blockstore
 	pinning    pin.Pinner
 
-	blocks bserv.BlockService
-	dag    ipld.DAGService
-
-	peerstore       pstore.Peerstore
-	peerHost        p2phost.Host
-	recordValidator record.Validator
-	exchange        exchange.Interface
+	blocks               bserv.BlockService
+	dag                  ipld.DAGService
+	ipldFetcherFactory   fetcher.Factory
+	unixFSFetcherFactory fetcher.Factory
+	peerstore            pstore.Peerstore
+	peerHost             p2phost.Host
+	recordValidator      record.Validator
+	exchange             exchange.Interface
 
 	namesys namesys.NameSystem
 	routing routing.Routing
@@ -165,8 +167,10 @@ func (api *CoreAPI) WithOptions(opts ...options.ApiOption) (coreiface.CoreAPI, e
 		baseBlocks: n.BaseBlocks,
 		pinning:    n.Pinning,
 
-		blocks: n.Blocks,
-		dag:    n.DAG,
+		blocks:               n.Blocks,
+		dag:                  n.DAG,
+		ipldFetcherFactory:   n.IPLDFetcherFactory,
+		unixFSFetcherFactory: n.UnixFSFetcherFactory,
 
 		peerstore:       n.Peerstore,
 		peerHost:        n.PeerHost,
@@ -212,7 +216,12 @@ func (api *CoreAPI) WithOptions(opts ...options.ApiOption) (coreiface.CoreAPI, e
 		}
 
 		subApi.routing = offlineroute.NewOfflineRouter(subApi.repo.Datastore(), subApi.recordValidator)
-		subApi.namesys = namesys.NewNameSystem(subApi.routing, subApi.repo.Datastore(), cs)
+		subApi.namesys, err = namesys.NewNameSystem(subApi.routing,
+			namesys.WithDatastore(subApi.repo.Datastore()),
+			namesys.WithCache(cs))
+		if err != nil {
+			return nil, fmt.Errorf("error constructing namesys: %w", err)
+		}
 		subApi.provider = provider.NewOfflineProvider()
 
 		subApi.peerstore = nil