feat: add 13 to 14 migration

Makefile

@@ -4,7 +4,7 @@ MIG_DIRS = $(shell ls -d fs-repo-*-to-*)
 IGNORED_DIRS := $(shell cat ignored-migrations)
 ACTIVE_DIRS := $(filter-out $(IGNORED_DIRS),$(MIG_DIRS))
 
-.PHONY: all build clean cmd sharness test test_go test_12_to_13
+.PHONY: all build clean cmd sharness test test_go test_12_to_13 test_13_to_14
 
 all: build
 
@@ -26,7 +26,7 @@ fs-repo-migrations/fs-repo-migrations:
 sharness:
 	make -C sharness
 
-test: test_go sharness test_12_to_13
+test: test_go sharness test_12_to_13 test_13_to_14
 
 clean: $(subst fs-repo,clean.fs-repo,$(ACTIVE_DIRS))
 	@make -C sharness clean
@@ -46,3 +46,6 @@ test_go.%:
 
 test_12_to_13:
 	@cd fs-repo-12-to-13/not-sharness && ./test.sh
+
+test_13_to_14:
+	@cd fs-repo-13-to-14/not-sharness && ./test.sh

README.md

@@ -48,7 +48,8 @@ Here is the table showing which repo version corresponds to which Kubo version:
 |                10 | 0.6.0 - 0.7.0    |
 |                11 | 0.8.0 - 0.11.0   |
 |                12 | 0.12.0 - 0.17.0  |
-|                13 | 0.18.0 - current |
+|                13 | 0.18.0 - 0.20.0  |
+|                14 | 0.21.0 - current |
 
 ### How to Run Migrations
 

fs-repo-13-to-14/Makefile

@@ -0,0 +1,7 @@
+.PHONY: build clean
+
+build:
+	go build -mod=vendor
+
+clean:
+	go clean

fs-repo-13-to-14/atomicfile/atomicfile.go

@@ -0,0 +1,59 @@
+// Package atomicfile provides the ability to write a file with an eventual
+// rename on Close (using os.Rename). This allows for a file to always be in a
+// consistent state and never represent an in-progress write.
+//
+// NOTE: `os.Rename` may not be atomic on your operating system.
+package atomicfile
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// File behaves like os.File, but does an atomic rename operation at Close.
+type File struct {
+	*os.File
+	path string
+}
+
+// New creates a new temporary file that will replace the file at the given
+// path when Closed.
+func New(path string, mode os.FileMode) (*File, error) {
+	f, err := ioutil.TempFile(filepath.Dir(path), filepath.Base(path))
+	if err != nil {
+		return nil, err
+	}
+	if err := os.Chmod(f.Name(), mode); err != nil {
+		f.Close()
+		os.Remove(f.Name())
+		return nil, err
+	}
+	return &File{File: f, path: path}, nil
+}
+
+// Close the file replacing the configured file.
+func (f *File) Close() error {
+	if err := f.File.Close(); err != nil {
+		os.Remove(f.File.Name())
+		return err
+	}
+	if err := os.Rename(f.Name(), f.path); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Abort closes the file and removes it instead of replacing the configured
+// file. This is useful if after starting to write to the file you decide you
+// don't want it anymore.
+func (f *File) Abort() error {
+	if err := f.File.Close(); err != nil {
+		os.Remove(f.Name())
+		return err
+	}
+	if err := os.Remove(f.Name()); err != nil {
+		return err
+	}
+	return nil
+}

fs-repo-13-to-14/go.mod

@@ -0,0 +1,5 @@
+module github.com/ipfs/fs-repo-migrations/fs-repo-13-to-14
+
+go 1.18
+
+require github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea

fs-repo-13-to-14/go.sum

@@ -0,0 +1,2 @@
+github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea h1:lgfk2PMrJI3bh8FflcBTXyNi3rPLqa75J7KcoUfRJmc=
+github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea/go.mod h1:fADeaHKxwS+SKhc52rsL0P1MUcnyK31a9AcaG0KcfY8=

fs-repo-13-to-14/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	mg13 "github.com/ipfs/fs-repo-migrations/fs-repo-13-to-14/migration"
+	migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate"
+)
+
+func main() {
+	m := mg13.Migration{}
+	migrate.Main(m)
+}

fs-repo-13-to-14/migration/migration.go

@@ -0,0 +1,204 @@
+// package mg13 contains the code to perform 13-14 repository migration in Kubo.
+// This just move the AcceleratedDHTClient from the Experimental section to the Routing one.
+package mg13
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate"
+	mfsr "github.com/ipfs/fs-repo-migrations/tools/mfsr"
+	lock "github.com/ipfs/fs-repo-migrations/tools/repolock"
+	log "github.com/ipfs/fs-repo-migrations/tools/stump"
+
+	"github.com/ipfs/fs-repo-migrations/fs-repo-13-to-14/atomicfile"
+)
+
+const backupSuffix = ".13-to-14.bak"
+
+// Migration implements the migration described above.
+type Migration struct{}
+
+// Versions returns the current version string for this migration.
+func (m Migration) Versions() string {
+	return "13-to-14"
+}
+
+// Reversible returns true, as we keep old config around
+func (m Migration) Reversible() bool {
+	return true
+}
+
+// Apply update the config.
+func (m Migration) Apply(opts migrate.Options) error {
+	log.Verbose = opts.Verbose
+	log.Log("applying %s repo migration", m.Versions())
+
+	log.VLog("locking repo at %q", opts.Path)
+	lk, err := lock.Lock2(opts.Path)
+	if err != nil {
+		return err
+	}
+	defer lk.Close()
+
+	repo := mfsr.RepoPath(opts.Path)
+
+	log.VLog("  - verifying version is '13'")
+	if err := repo.CheckVersion("13"); err != nil {
+		return err
+	}
+
+	log.Log("> Upgrading config to new format")
+
+	path := filepath.Join(opts.Path, "config")
+	in, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+
+	// make backup
+	backup, err := atomicfile.New(path+backupSuffix, 0600)
+	if err != nil {
+		return err
+	}
+	if _, err := backup.ReadFrom(in); err != nil {
+		panicOnError(backup.Abort())
+		return err
+	}
+	if _, err := in.Seek(0, io.SeekStart); err != nil {
+		panicOnError(backup.Abort())
+		return err
+	}
+
+	// Create a temp file to write the output to on success
+	out, err := atomicfile.New(path, 0600)
+	if err != nil {
+		panicOnError(backup.Abort())
+		panicOnError(in.Close())
+		return err
+	}
+
+	if err := convert(in, out); err != nil {
+		panicOnError(out.Abort())
+		panicOnError(backup.Abort())
+		panicOnError(in.Close())
+		return err
+	}
+
+	if err := in.Close(); err != nil {
+		panicOnError(out.Abort())
+		panicOnError(backup.Abort())
+	}
+
+	if err := repo.WriteVersion("14"); err != nil {
+		log.Error("failed to update version file to 14")
+		// There was an error so abort writing the output and clean up temp file
+		panicOnError(out.Abort())
+		panicOnError(backup.Abort())
+		return err
+	} else {
+		// Write the output and clean up temp file
+		panicOnError(out.Close())
+		panicOnError(backup.Close())
+	}
+
+	log.Log("updated version file")
+
+	log.Log("Migration 13 to 14 succeeded")
+	return nil
+}
+
+// panicOnError is reserved for checks we can't solve transactionally if an error occurs
+func panicOnError(e error) {
+	if e != nil {
+		panic(fmt.Errorf("error can't be dealt with transactionally: %w", e))
+	}
+}
+
+func (m Migration) Revert(opts migrate.Options) error {
+	log.Verbose = opts.Verbose
+	log.Log("reverting migration")
+	lk, err := lock.Lock2(opts.Path)
+	if err != nil {
+		return err
+	}
+	defer lk.Close()
+
+	repo := mfsr.RepoPath(opts.Path)
+	if err := repo.CheckVersion("14"); err != nil {
+		return err
+	}
+
+	cfg := filepath.Join(opts.Path, "config")
+	if err := os.Rename(cfg+backupSuffix, cfg); err != nil {
+		return err
+	}
+
+	if err := repo.WriteVersion("13"); err != nil {
+		return err
+	}
+	if opts.Verbose {
+		log.Log("lowered version number to 13")
+	}
+
+	return nil
+}
+
+// convert converts the config from one version to another
+func convert(in io.Reader, out io.Writer) error {
+	confMap := make(map[string]any)
+	if err := json.NewDecoder(in).Decode(&confMap); err != nil {
+		return err
+	}
+
+	// Move AcceleratedDHTClient key.
+	var acceleratedDHTClient bool
+	if e, ok := confMap["Experimental"]; ok {
+		exp, ok := e.(map[string]any)
+		if !ok {
+			return fmt.Errorf("invalid type for .Experimental got %T expected json map", e)
+		}
+		if a, ok := exp["AcceleratedDHTClient"]; ok {
+			acc, ok := a.(bool)
+			if !ok {
+				return fmt.Errorf("invalid type for .Experimental.AcceleratedDHTClient got %T expected bool", e)
+			}
+			acceleratedDHTClient = acc
+			delete(exp, "AcceleratedDHTClient")
+
+			if len(exp) == 0 {
+				delete(confMap, "Experimental")
+			}
+		}
+	}
+
+	// If the key missing insert new into routing
+	var rr map[string]any
+	if r, ok := confMap["Routing"]; ok {
+		rr, ok = r.(map[string]any)
+		if !ok {
+			return fmt.Errorf("invalid type for .Routing, got %T expected json map", r)
+		}
+	} else {
+		rr = make(map[string]any)
+		confMap["Routing"] = rr
+	}
+	if _, ok := rr["AcceleratedDHTClient"]; !ok {
+		// Only add the key if it's not already present in the destination
+		rr["AcceleratedDHTClient"] = acceleratedDHTClient
+	}
+
+	fixed, err := json.MarshalIndent(confMap, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	if _, err := out.Write(fixed); err != nil {
+		return err
+	}
+	_, err = out.Write([]byte("\n"))
+	return err
+}

fs-repo-13-to-14/not-sharness/.gitignore

@@ -0,0 +1 @@
+repotest

fs-repo-13-to-14/not-sharness/repotest-golden/config

@@ -0,0 +1,5 @@
+{
+  "Routing": {
+    "AcceleratedDHTClient": true
+  }
+}

fs-repo-13-to-14/not-sharness/repotest-golden/config.13-to-14.bak

@@ -0,0 +1,5 @@
+{
+  "Experimental": {
+    "AcceleratedDHTClient": true
+  }
+}

fs-repo-13-to-14/not-sharness/repotest-golden/version

@@ -0,0 +1 @@
+14

fs-repo-13-to-14/not-sharness/repotest-init/config

@@ -0,0 +1,5 @@
+{
+  "Experimental": {
+    "AcceleratedDHTClient": true
+  }
+}

fs-repo-13-to-14/not-sharness/repotest-init/version

@@ -0,0 +1 @@
+13

fs-repo-13-to-14/not-sharness/test.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -x
+
+echo "Migration 13 to 14" &&
+cp -r repotest-init repotest && # init repo
+go run .. -verbose -path=repotest && # run forward migration
+diff -r repotest-golden repotest && # check forward migration against golden
+go run .. -verbose -revert -path=repotest && # run backward migration
+diff -r repotest-init repotest # check that after backward migration everything is back to how it used to be
+
+FINISH="$?" # save exit code
+
+rm -r repotest # cleanup
+
+exit "$FINISH" # forward exit code