Merge branch 'main' into feat/gateway-refactor

cmd/migrate/go.mod

@@ -0,0 +1,11 @@
+module github.com/ipfs/boxo/cmd/migrate
+
+go 1.19
+
+require github.com/urfave/cli/v2 v2.25.1
+
+require (
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+)

cmd/migrate/go.sum

@@ -0,0 +1,8 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/urfave/cli/v2 v2.25.1 h1:zw8dSP7ghX0Gmm8vugrs6q9Ku0wzweqPyshy+syu9Gw=
+github.com/urfave/cli/v2 v2.25.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=

cmd/migrate/internal/config.go

@@ -0,0 +1,114 @@
+package migrate
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+)
+
+type Config struct {
+	ImportPaths map[string]string
+	Modules     []string
+}
+
+var DefaultConfig = Config{
+	ImportPaths: map[string]string{
+		"github.com/ipfs/go-bitswap":                     "github.com/ipfs/boxo/bitswap",
+		"github.com/ipfs/go-ipfs-files":                  "github.com/ipfs/boxo/files",
+		"github.com/ipfs/tar-utils":                      "github.com/ipfs/boxo/tar",
+		"github.com/ipfs/interface-go-ipfs-core":         "github.com/ipfs/boxo/coreiface",
+		"github.com/ipfs/go-unixfs":                      "github.com/ipfs/boxo/ipld/unixfs",
+		"github.com/ipfs/go-pinning-service-http-client": "github.com/ipfs/boxo/pinning/remote/client",
+		"github.com/ipfs/go-path":                        "github.com/ipfs/boxo/path",
+		"github.com/ipfs/go-namesys":                     "github.com/ipfs/boxo/namesys",
+		"github.com/ipfs/go-mfs":                         "github.com/ipfs/boxo/mfs",
+		"github.com/ipfs/go-ipfs-provider":               "github.com/ipfs/boxo/provider",
+		"github.com/ipfs/go-ipfs-pinner":                 "github.com/ipfs/boxo/pinning/pinner",
+		"github.com/ipfs/go-ipfs-keystore":               "github.com/ipfs/boxo/keystore",
+		"github.com/ipfs/go-filestore":                   "github.com/ipfs/boxo/filestore",
+		"github.com/ipfs/go-ipns":                        "github.com/ipfs/boxo/ipns",
+		"github.com/ipfs/go-blockservice":                "github.com/ipfs/boxo/blockservice",
+		"github.com/ipfs/go-ipfs-chunker":                "github.com/ipfs/boxo/chunker",
+		"github.com/ipfs/go-fetcher":                     "github.com/ipfs/boxo/fetcher",
+		"github.com/ipfs/go-ipfs-blockstore":             "github.com/ipfs/boxo/blockstore",
+		"github.com/ipfs/go-ipfs-posinfo":                "github.com/ipfs/boxo/filestore/posinfo",
+		"github.com/ipfs/go-ipfs-util":                   "github.com/ipfs/boxo/util",
+		"github.com/ipfs/go-ipfs-ds-help":                "github.com/ipfs/boxo/datastore/dshelp",
+		"github.com/ipfs/go-verifcid":                    "github.com/ipfs/boxo/verifcid",
+		"github.com/ipfs/go-ipfs-exchange-offline":       "github.com/ipfs/boxo/exchange/offline",
+		"github.com/ipfs/go-ipfs-routing":                "github.com/ipfs/boxo/routing",
+		"github.com/ipfs/go-ipfs-exchange-interface":     "github.com/ipfs/boxo/exchange",
+		"github.com/ipfs/go-merkledag":                   "github.com/ipfs/boxo/ipld/merkledag",
+		"github.com/ipld/go-car":                         "github.com/ipfs/boxo/ipld/car",
+
+		// Pre Boxo rename
+		"github.com/ipfs/go-libipfs/gateway":               "github.com/ipfs/boxo/gateway",
+		"github.com/ipfs/go-libipfs/bitswap":               "github.com/ipfs/boxo/bitswap",
+		"github.com/ipfs/go-libipfs/files":                 "github.com/ipfs/boxo/files",
+		"github.com/ipfs/go-libipfs/tar":                   "github.com/ipfs/boxo/tar",
+		"github.com/ipfs/go-libipfs/coreiface":             "github.com/ipfs/boxo/coreiface",
+		"github.com/ipfs/go-libipfs/unixfs":                "github.com/ipfs/boxo/ipld/unixfs",
+		"github.com/ipfs/go-libipfs/pinning/remote/client": "github.com/ipfs/boxo/pinning/remote/client",
+		"github.com/ipfs/go-libipfs/path":                  "github.com/ipfs/boxo/path",
+		"github.com/ipfs/go-libipfs/namesys":               "github.com/ipfs/boxo/namesys",
+		"github.com/ipfs/go-libipfs/mfs":                   "github.com/ipfs/boxo/mfs",
+		"github.com/ipfs/go-libipfs/provider":              "github.com/ipfs/boxo/provider",
+		"github.com/ipfs/go-libipfs/pinning/pinner":        "github.com/ipfs/boxo/pinning/pinner",
+		"github.com/ipfs/go-libipfs/keystore":              "github.com/ipfs/boxo/keystore",
+		"github.com/ipfs/go-libipfs/filestore":             "github.com/ipfs/boxo/filestore",
+		"github.com/ipfs/go-libipfs/ipns":                  "github.com/ipfs/boxo/ipns",
+		"github.com/ipfs/go-libipfs/blockservice":          "github.com/ipfs/boxo/blockservice",
+		"github.com/ipfs/go-libipfs/chunker":               "github.com/ipfs/boxo/chunker",
+		"github.com/ipfs/go-libipfs/fetcher":               "github.com/ipfs/boxo/fetcher",
+		"github.com/ipfs/go-libipfs/blockstore":            "github.com/ipfs/boxo/blockstore",
+		"github.com/ipfs/go-libipfs/filestore/posinfo":     "github.com/ipfs/boxo/filestore/posinfo",
+		"github.com/ipfs/go-libipfs/util":                  "github.com/ipfs/boxo/util",
+		"github.com/ipfs/go-libipfs/datastore/dshelp":      "github.com/ipfs/boxo/datastore/dshelp",
+		"github.com/ipfs/go-libipfs/verifcid":              "github.com/ipfs/boxo/verifcid",
+		"github.com/ipfs/go-libipfs/exchange/offline":      "github.com/ipfs/boxo/exchange/offline",
+		"github.com/ipfs/go-libipfs/routing":               "github.com/ipfs/boxo/routing",
+		"github.com/ipfs/go-libipfs/exchange":              "github.com/ipfs/boxo/exchange",
+
+		// Unmigrated things
+		"github.com/ipfs/go-libipfs/blocks": "github.com/ipfs/go-block-format",
+		"github.com/ipfs/boxo/blocks":       "github.com/ipfs/go-block-format",
+	},
+	Modules: []string{
+		"github.com/ipfs/go-bitswap",
+		"github.com/ipfs/go-ipfs-files",
+		"github.com/ipfs/tar-utils",
+		"gihtub.com/ipfs/go-block-format",
+		"github.com/ipfs/interface-go-ipfs-core",
+		"github.com/ipfs/go-unixfs",
+		"github.com/ipfs/go-pinning-service-http-client",
+		"github.com/ipfs/go-path",
+		"github.com/ipfs/go-namesys",
+		"github.com/ipfs/go-mfs",
+		"github.com/ipfs/go-ipfs-provider",
+		"github.com/ipfs/go-ipfs-pinner",
+		"github.com/ipfs/go-ipfs-keystore",
+		"github.com/ipfs/go-filestore",
+		"github.com/ipfs/go-ipns",
+		"github.com/ipfs/go-blockservice",
+		"github.com/ipfs/go-ipfs-chunker",
+		"github.com/ipfs/go-fetcher",
+		"github.com/ipfs/go-ipfs-blockstore",
+		"github.com/ipfs/go-ipfs-posinfo",
+		"github.com/ipfs/go-ipfs-util",
+		"github.com/ipfs/go-ipfs-ds-help",
+		"github.com/ipfs/go-verifcid",
+		"github.com/ipfs/go-ipfs-exchange-offline",
+		"github.com/ipfs/go-ipfs-routing",
+		"github.com/ipfs/go-ipfs-exchange-interface",
+		"github.com/ipfs/go-libipfs",
+	},
+}
+
+func ReadConfig(r io.Reader) (Config, error) {
+	var config Config
+	err := json.NewDecoder(r).Decode(&config)
+	if err != nil {
+		return Config{}, fmt.Errorf("reading and decoding config: %w", err)
+	}
+	return config, nil
+}

cmd/migrate/internal/json.go

@@ -0,0 +1,23 @@
+package migrate
+
+import "path/filepath"
+
+type pkgJSON struct {
+	Dir            string
+	GoFiles        []string
+	IgnoredGoFiles []string
+	TestGoFiles    []string
+	XTestGoFiles   []string
+	CgoFiles       []string
+}
+
+func (p *pkgJSON) allSourceFiles() []string {
+	var files []string
+	lists := [...][]string{p.GoFiles, p.IgnoredGoFiles, p.TestGoFiles, p.CgoFiles, p.XTestGoFiles}
+	for _, l := range lists {
+		for _, f := range l {
+			files = append(files, filepath.Join(p.Dir, f))
+		}
+	}
+	return files
+}

cmd/migrate/internal/migrator.go

@@ -0,0 +1,168 @@
+package migrate
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"go/ast"
+	"go/format"
+	"go/parser"
+	"go/token"
+	"io"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
+type Migrator struct {
+	DryRun bool
+	Dir    string
+	Config Config
+}
+
+func (m *Migrator) updateFileImports(filePath string) error {
+	fset := token.NewFileSet()
+	astFile, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments)
+	if err != nil {
+		return fmt.Errorf("parsing %q: %w", filePath, err)
+	}
+
+	var fileChanged bool
+
+	var errr error
+	ast.Inspect(astFile, func(n ast.Node) bool {
+		switch x := n.(type) {
+		case *ast.ImportSpec:
+			val, err := strconv.Unquote(x.Path.Value)
+			if err != nil {
+				errr = err
+				return false
+			}
+			// we take the first matching prefix, so you need to make sure you don't have ambiguous mappings
+			for from, to := range m.Config.ImportPaths {
+				if strings.HasPrefix(val, from) {
+					var newVal string
+					switch {
+					case len(val) == len(from):
+						newVal = to
+					case val[len(from)] != '/':
+						continue
+					default:
+						newVal = to + val[len(from):]
+					}
+					fmt.Printf("changing %s => %s in %s\n", x.Path.Value, newVal, filePath)
+					if !m.DryRun {
+						x.Path.Value = strconv.Quote(newVal)
+						fileChanged = true
+					}
+				}
+			}
+		}
+		return true
+	})
+	if errr != nil {
+		return errr
+	}
+
+	if !fileChanged {
+		return nil
+	}
+
+	f, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	err = format.Node(f, fset, astFile)
+	if err != nil {
+		f.Close()
+		return fmt.Errorf("formatting %q: %w", filePath, err)
+	}
+	err = f.Close()
+	if err != nil {
+		return fmt.Errorf("closing %q: %w", filePath, err)
+	}
+
+	return nil
+}
+
+func (m *Migrator) run(cmdName string, args ...string) (int, string, string, error) {
+	cmd := exec.Command(cmdName, args...)
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	cmd.Dir = m.Dir
+	err := cmd.Start()
+	if err != nil {
+		return 0, "", "", fmt.Errorf("running %s %v: %w", cmdName, args, err)
+	}
+	state, err := cmd.Process.Wait()
+	if err != nil {
+		return 0, "", "", fmt.Errorf("waiting for %s %v: %w", cmdName, args, err)
+	}
+	return state.ExitCode(), strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), nil
+}
+
+// FindMigratedDependencies returns a list of dependent module versions like 'module v0.1.0' that have been migrated to go-libipfs.
+func (m *Migrator) FindMigratedDependencies() ([]string, error) {
+	var modVersions []string
+	for _, mod := range m.Config.Modules {
+		exitCode, stdout, stderr, err := m.run("go", "list", "-m", mod)
+		if err != nil {
+			return nil, err
+		}
+		if exitCode == 0 {
+			scanner := bufio.NewScanner(strings.NewReader(stdout))
+			for scanner.Scan() {
+				modVersions = append(modVersions, scanner.Text())
+			}
+		} else {
+			if !strings.Contains(stderr, "not a known dependency") {
+				return nil, fmt.Errorf("non-zero exit code %d finding if current module depends on %q, stderr:\n%s", exitCode, mod, stderr)
+			}
+		}
+	}
+	return modVersions, nil
+}
+
+func (m *Migrator) findSourceFiles() ([]string, error) {
+	exitCode, stdout, stderr, err := m.run("go", "list", "-json", "./...")
+	if err != nil {
+		return nil, fmt.Errorf("finding source files: %w", err)
+	}
+	if exitCode != 0 {
+		return nil, fmt.Errorf("non-zero exit code %d finding source files, stderr:\n%s", exitCode, stderr)
+	}
+
+	var files []string
+	dec := json.NewDecoder(strings.NewReader(stdout))
+	for {
+		var pkg pkgJSON
+		err = dec.Decode(&pkg)
+		if err == io.EOF {
+			return files, nil
+		}
+		if err != nil {
+			return nil, fmt.Errorf("decoding 'go list' JSON: %w", err)
+		}
+		files = append(files, pkg.allSourceFiles()...)
+	}
+}
+
+// UpdateImports rewrites the imports of the current module for any import paths that have been migrated to go-libipfs.
+func (m *Migrator) UpdateImports() error {
+	sourceFiles, err := m.findSourceFiles()
+	if err != nil {
+		return err
+	}
+	for _, sourceFile := range sourceFiles {
+		err := m.updateFileImports(sourceFile)
+		if err != nil {
+			return fmt.Errorf("updating imports in %q: %w", sourceFile, err)
+		}
+	}
+	return nil
+}