feat: 12 to 13

This needs tests.

.github/workflows/test.yml

@@ -1,7 +1,7 @@
 name: Tests
 
 env:
-  GO: 1.16
+  GO: 1.18
 
 on:
   push:

fs-repo-12-to-13/Makefile

@@ -0,0 +1,7 @@
+.PHONY: build clean
+
+build:
+	go build -mod=vendor
+
+clean:
+	go clean

fs-repo-12-to-13/atomicfile/atomicfile.go

@@ -0,0 +1,59 @@
+// Package atomicfile provides the ability to write a file with an eventual
+// rename on Close (using os.Rename). This allows for a file to always be in a
+// consistent state and never represent an in-progress write.
+//
+// NOTE: `os.Rename` may not be atomic on your operating system.
+package atomicfile
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// File behaves like os.File, but does an atomic rename operation at Close.
+type File struct {
+	*os.File
+	path string
+}
+
+// New creates a new temporary file that will replace the file at the given
+// path when Closed.
+func New(path string, mode os.FileMode) (*File, error) {
+	f, err := ioutil.TempFile(filepath.Dir(path), filepath.Base(path))
+	if err != nil {
+		return nil, err
+	}
+	if err := os.Chmod(f.Name(), mode); err != nil {
+		f.Close()
+		os.Remove(f.Name())
+		return nil, err
+	}
+	return &File{File: f, path: path}, nil
+}
+
+// Close the file replacing the configured file.
+func (f *File) Close() error {
+	if err := f.File.Close(); err != nil {
+		os.Remove(f.File.Name())
+		return err
+	}
+	if err := os.Rename(f.Name(), f.path); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Abort closes the file and removes it instead of replacing the configured
+// file. This is useful if after starting to write to the file you decide you
+// don't want it anymore.
+func (f *File) Abort() error {
+	if err := f.File.Close(); err != nil {
+		os.Remove(f.Name())
+		return err
+	}
+	if err := os.Remove(f.Name()); err != nil {
+		return err
+	}
+	return nil
+}

fs-repo-12-to-13/go.mod

@@ -0,0 +1,5 @@
+module github.com/ipfs/fs-repo-migrations/fs-repo-12-to-13
+
+go 1.18
+
+require github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea

fs-repo-12-to-13/go.sum

@@ -0,0 +1,2 @@
+github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea h1:lgfk2PMrJI3bh8FflcBTXyNi3rPLqa75J7KcoUfRJmc=
+github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea/go.mod h1:fADeaHKxwS+SKhc52rsL0P1MUcnyK31a9AcaG0KcfY8=

fs-repo-12-to-13/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	mg12 "github.com/ipfs/fs-repo-migrations/fs-repo-12-to-13/migration"
+	migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate"
+)
+
+func main() {
+	m := mg12.Migration{}
+	migrate.Main(m)
+}

fs-repo-12-to-13/migration/generate_repotest.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# This script can be used to generate the repotest folder that is used
+# to test the migration, using go-ipfs v0.8.0.
+
+export IPFS_PATH=repotest
+
+# use server profile to have some no announces and filters in place
+ipfs init -p server -e
+
+# add a forced announces as they must also be updated
+ipfs config --json Addresses.Announce "[\"/ip4/1.2.3.4/tcp/1337\",\"/ip4/1.2.3.4/udp/1337/quic\"]"
+ipfs config --json Addresses.NoAnnounce "[\"/ip4/1.2.3.4/tcp/1337\",\"/ip4/1.2.3.4/udp/1337/quic\"]"
+ipfs config --json Addresses.AppendAnnounce "[\"/ip4/1.2.3.4/tcp/1337\",\"/ip4/1.2.3.4/udp/1337/quic\"]"

fs-repo-12-to-13/migration/migration.go

@@ -0,0 +1,216 @@
+// package mg12 contains the code to perform 12-13 repository migration in Kubo.
+// This just change some config fields to add webtransport listens on ones that quic uses.
+package mg12
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate"
+	mfsr "github.com/ipfs/fs-repo-migrations/tools/mfsr"
+	lock "github.com/ipfs/fs-repo-migrations/tools/repolock"
+	log "github.com/ipfs/fs-repo-migrations/tools/stump"
+
+	"github.com/ipfs/fs-repo-migrations/fs-repo-12-to-13/atomicfile"
+)
+
+const backupSuffix = ".12-to-13.bak"
+
+// Migration implements the migration described above.
+type Migration struct{}
+
+// Versions returns the current version string for this migration.
+func (m *Migration) Versions() string {
+	return "12-to-13"
+}
+
+// Reversible returns false, as you can just use the backup config file.
+func (m *Migration) Reversible() bool {
+	return true
+}
+
+// Apply update the config.
+func (m Migration) Apply(opts migrate.Options) error {
+	log.Verbose = opts.Verbose
+	log.Log("applying %s repo migration", m.Versions())
+
+	log.VLog("locking repo at %q", opts.Path)
+	lk, err := lock.Lock2(opts.Path)
+	if err != nil {
+		return err
+	}
+	defer lk.Close()
+
+	repo := mfsr.RepoPath(opts.Path)
+
+	log.VLog("  - verifying version is '12'")
+	if err := repo.CheckVersion("12"); err != nil {
+		return err
+	}
+
+	log.Log("> Upgrading config to new format")
+
+	path := filepath.Join(opts.Path, "config")
+	if err := convertFile(path); err != nil {
+		return err
+	}
+
+	if err := repo.WriteVersion("13"); err != nil {
+		log.Error("failed to update version file to 13")
+		return err
+	}
+
+	log.Log("updated version file")
+
+	log.Log("Migration 12 to 13 succeeded")
+	return nil
+}
+
+func (m Migration) Revert(opts migrate.Options) error {
+	log.Verbose = opts.Verbose
+	log.Log("reverting migration")
+	lk, err := lock.Lock2(opts.Path)
+	if err != nil {
+		return err
+	}
+	defer lk.Close()
+
+	repo := mfsr.RepoPath(opts.Path)
+	if err := repo.CheckVersion("13"); err != nil {
+		return err
+	}
+
+	cfg := filepath.Join(opts.Path, "config")
+	if err := os.Rename(cfg+backupSuffix, cfg); err != nil {
+		return err
+	}
+
+	if err := repo.WriteVersion("12"); err != nil {
+		return err
+	}
+	if opts.Verbose {
+		log.Log("lowered version number to 12")
+	}
+
+	return nil
+}
+
+// convertFile converts a config file from 12 version to 13
+func convertFile(path string) error {
+	in, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+
+	// make backup
+	backup, err := atomicfile.New(path+backupSuffix, 0600)
+	if err != nil {
+		return err
+	}
+	if _, err := backup.ReadFrom(in); err != nil {
+		backup.Abort()
+		return err
+	}
+	if _, err := in.Seek(0, io.SeekStart); err != nil {
+		backup.Abort()
+		return err
+	}
+
+	// Create a temp file to write the output to on success
+	out, err := atomicfile.New(path, 0600)
+	if err != nil {
+		in.Close()
+		backup.Abort()
+		return err
+	}
+
+	err = convert(in, out)
+
+	in.Close()
+
+	if err != nil {
+		// There was an error so abort writing the output and clean up temp file
+		out.Abort()
+		backup.Abort()
+	} else {
+		// Write the output and clean up temp file
+		out.Close()
+		backup.Close()
+	}
+
+	return err
+}
+
+// convert converts the config from one version to another
+func convert(in io.Reader, out io.Writer) error {
+	confMap := make(map[string]any)
+	if err := json.NewDecoder(in).Decode(&confMap); err != nil {
+		return err
+	}
+
+	applyChangeOnLevelPlusOnes(confMap, webtransportify, "Addresses", "Announce", "AppendAnnounce", "NoAnnounce", "Swarm")
+	applyChangeOnLevelPlusOnes(confMap, webtransportify, "Swarm", "AddrFilters")
+
+	fixed, err := json.MarshalIndent(confMap, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	if _, err := out.Write(fixed); err != nil {
+		return err
+	}
+	_, err = out.Write([]byte("\n"))
+	return err
+}
+
+// this walk one step in m, then walk all of vs, then try to cast to T, if all of this succeeded for thoses elements, pass it through transform
+func applyChangeOnLevelPlusOnes[T any, O any, K comparable, V comparable](m map[K]any, transform func(T) O, l0 K, vs ...V) {
+	if addresses, ok := m[l0].(map[V]any); ok {
+		for _, v := range vs {
+			if addrs, ok := addresses[v].(T); ok {
+				addresses[v] = transform(addrs)
+			}
+		}
+	}
+}
+
+func webtransportify(in []any) (out []any) {
+	for _, w := range in {
+		out = append(out, w)
+
+		// only run if w is a string
+		v, ok := w.(string)
+		if !ok {
+			continue
+		}
+
+		// implement /quic(?=(/|$))(?!.*/p2p-circuit(/|$)) manually because regexp doesn't have lookaheads
+		// iterate over the string backward, if we found quic first replace it, else if we found p2p-circuit stop
+		// assume ASCII
+		var r string
+		last := len(v)
+	ScanLoop:
+		for i := last - len("/quic"); i != 0; i-- {
+			switch {
+			case hasPrefixAndEndsOrSlash(v[i:], "/quic"):
+				r = "/quic/webtransport" + v[i+len("/quic"):last] + r
+				last = i
+			case hasPrefixAndEndsOrSlash(v[i:], "/p2p-circuit"):
+				break ScanLoop
+			}
+		}
+		r = v[:last] + r
+
+		if r != v {
+			out = append(out, r)
+		}
+	}
+	return
+}
+
+func hasPrefixAndEndsOrSlash(s, prefix string) bool {
+	return strings.HasPrefix(s, prefix) && (len(prefix) == len(s) || s[len(prefix)] == '/')
+}

fs-repo-12-to-13/migration/repotest/blocks/SHARDING

@@ -0,0 +1 @@
+/repo/flatfs/shard/v1/next-to-last/2

fs-repo-12-to-13/migration/repotest/blocks/X3/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data

@@ -0,0 +1,2 @@
+
+

fs-repo-12-to-13/migration/repotest/blocks/_README

@@ -0,0 +1,30 @@
+This is a repository of IPLD objects. Each IPLD object is in a single file,
+named <base32 encoding of cid>.data. Where <base32 encoding of cid> is the
+"base32" encoding of the CID (as specified in
+https://github.com/multiformats/multibase) without the 'B' prefix.
+All the object files are placed in a tree of directories, based on a
+function of the CID. This is a form of sharding similar to
+the objects directory in git repositories. Previously, we used
+prefixes, we now use the next-to-last two characters.
+
+    func NextToLast(base32cid string) {
+      nextToLastLen := 2
+      offset := len(base32cid) - nextToLastLen - 1
+      return str[offset : offset+nextToLastLen]
+    }
+
+For example, an object with a base58 CIDv1 of
+
+    zb2rhYSxw4ZjuzgCnWSt19Q94ERaeFhu9uSqRgjSdx9bsgM6f
+
+has a base32 CIDv1 of
+
+    BAFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA
+
+and will be placed at
+
+    SC/AFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA.data
+
+with 'SC' being the last-to-next two characters and the 'B' at the
+beginning of the CIDv1 string is the multibase prefix that is not
+stored in the filename.

fs-repo-12-to-13/migration/repotest/blocks/diskUsage.cache

@@ -0,0 +1 @@
+{"diskUsage":13452,"accuracy":"initial-exact"}