Merge pull request #174 from ipfs/14-to-15

feat: migration from 14 to 15

Makefile

@@ -4,7 +4,7 @@ MIG_DIRS = $(shell ls -d fs-repo-*-to-*)
 IGNORED_DIRS := $(shell cat ignored-migrations)
 ACTIVE_DIRS := $(filter-out $(IGNORED_DIRS),$(MIG_DIRS))
 
-.PHONY: all build clean cmd sharness test test_go test_12_to_13 test_13_to_14
+.PHONY: all build clean cmd sharness test test_go test_13_to_14 test_14_to_15
 
 all: build
 
@@ -26,7 +26,7 @@ fs-repo-migrations/fs-repo-migrations:
 sharness:
 	make -C sharness
 
-test: test_go sharness test_12_to_13 test_13_to_14
+test: test_go sharness test_13_to_14 test_14_to_15
 
 clean: $(subst fs-repo,clean.fs-repo,$(ACTIVE_DIRS))
 	@make -C sharness clean
@@ -49,3 +49,6 @@ test_12_to_13:
 
 test_13_to_14:
 	@cd fs-repo-13-to-14/not-sharness && ./test.sh
+
+test_14_to_15:
+	@cd fs-repo-14-to-15/not-sharness && ./test.sh

README.md

@@ -49,7 +49,8 @@ Here is the table showing which repo version corresponds to which Kubo version:
 |                11 | 0.8.0 - 0.11.0   |
 |                12 | 0.12.0 - 0.17.0  |
 |                13 | 0.18.0 - 0.20.0  |
-|                14 | 0.21.0 - current |
+|                14 | 0.21.0 - 0.22.0  |
+|                15 | 0.23.0 - current |
 
 ### How to Run Migrations
 

fs-repo-14-to-15/Makefile

@@ -0,0 +1,7 @@
+.PHONY: build clean
+
+build:
+	go build -mod=vendor
+
+clean:
+	go clean

fs-repo-14-to-15/atomicfile/atomicfile.go

@@ -0,0 +1,59 @@
+// Package atomicfile provides the ability to write a file with an eventual
+// rename on Close (using os.Rename). This allows for a file to always be in a
+// consistent state and never represent an in-progress write.
+//
+// NOTE: `os.Rename` may not be atomic on your operating system.
+package atomicfile
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// File behaves like os.File, but does an atomic rename operation at Close.
+type File struct {
+	*os.File
+	path string
+}
+
+// New creates a new temporary file that will replace the file at the given
+// path when Closed.
+func New(path string, mode os.FileMode) (*File, error) {
+	f, err := ioutil.TempFile(filepath.Dir(path), filepath.Base(path))
+	if err != nil {
+		return nil, err
+	}
+	if err := os.Chmod(f.Name(), mode); err != nil {
+		f.Close()
+		os.Remove(f.Name())
+		return nil, err
+	}
+	return &File{File: f, path: path}, nil
+}
+
+// Close the file replacing the configured file.
+func (f *File) Close() error {
+	if err := f.File.Close(); err != nil {
+		os.Remove(f.File.Name())
+		return err
+	}
+	if err := os.Rename(f.Name(), f.path); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Abort closes the file and removes it instead of replacing the configured
+// file. This is useful if after starting to write to the file you decide you
+// don't want it anymore.
+func (f *File) Abort() error {
+	if err := f.File.Close(); err != nil {
+		os.Remove(f.Name())
+		return err
+	}
+	if err := os.Remove(f.Name()); err != nil {
+		return err
+	}
+	return nil
+}

fs-repo-14-to-15/go.mod

@@ -0,0 +1,5 @@
+module github.com/ipfs/fs-repo-migrations/fs-repo-14-to-15
+
+go 1.20
+
+require github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea

fs-repo-14-to-15/go.sum

@@ -0,0 +1,2 @@
+github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea h1:lgfk2PMrJI3bh8FflcBTXyNi3rPLqa75J7KcoUfRJmc=
+github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea/go.mod h1:fADeaHKxwS+SKhc52rsL0P1MUcnyK31a9AcaG0KcfY8=

fs-repo-14-to-15/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	mg14 "github.com/ipfs/fs-repo-migrations/fs-repo-14-to-15/migration"
+	migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate"
+)
+
+func main() {
+	m := mg14.Migration{}
+	migrate.Main(m)
+}

fs-repo-14-to-15/migration/migration.go

@@ -0,0 +1,253 @@
+// package mg14 contains the code to perform 14-15 repository migration in Kubo.
+// This handles the following:
+// - Replaces bootstrap QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ from /quic to /quic-v1
+// - Removes /quic only addresses from .Addresses.Swarm
+package mg14
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate"
+	mfsr "github.com/ipfs/fs-repo-migrations/tools/mfsr"
+	lock "github.com/ipfs/fs-repo-migrations/tools/repolock"
+	log "github.com/ipfs/fs-repo-migrations/tools/stump"
+
+	"github.com/ipfs/fs-repo-migrations/fs-repo-14-to-15/atomicfile"
+)
+
+const backupSuffix = ".14-to-15.bak"
+
+// Migration implements the migration described above.
+type Migration struct{}
+
+// Versions returns the current version string for this migration.
+func (m Migration) Versions() string {
+	return "14-to-15"
+}
+
+// Reversible returns true, as we keep old config around
+func (m Migration) Reversible() bool {
+	return true
+}
+
+// Apply update the config.
+func (m Migration) Apply(opts migrate.Options) error {
+	log.Verbose = opts.Verbose
+	log.Log("applying %s repo migration", m.Versions())
+
+	log.VLog("locking repo at %q", opts.Path)
+	lk, err := lock.Lock2(opts.Path)
+	if err != nil {
+		return err
+	}
+	defer lk.Close()
+
+	repo := mfsr.RepoPath(opts.Path)
+
+	log.VLog("  - verifying version is '14'")
+	if err := repo.CheckVersion("14"); err != nil {
+		return err
+	}
+
+	log.Log("> Upgrading config to new format")
+
+	path := filepath.Join(opts.Path, "config")
+	in, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+
+	// make backup
+	backup, err := atomicfile.New(path+backupSuffix, 0600)
+	if err != nil {
+		return err
+	}
+	if _, err := backup.ReadFrom(in); err != nil {
+		panicOnError(backup.Abort())
+		return err
+	}
+	if _, err := in.Seek(0, io.SeekStart); err != nil {
+		panicOnError(backup.Abort())
+		return err
+	}
+
+	// Create a temp file to write the output to on success
+	out, err := atomicfile.New(path, 0600)
+	if err != nil {
+		panicOnError(backup.Abort())
+		panicOnError(in.Close())
+		return err
+	}
+
+	if err := convert(in, out); err != nil {
+		panicOnError(out.Abort())
+		panicOnError(backup.Abort())
+		panicOnError(in.Close())
+		return err
+	}
+
+	if err := in.Close(); err != nil {
+		panicOnError(out.Abort())
+		panicOnError(backup.Abort())
+	}
+
+	if err := repo.WriteVersion("15"); err != nil {
+		log.Error("failed to update version file to 15")
+		// There was an error so abort writing the output and clean up temp file
+		panicOnError(out.Abort())
+		panicOnError(backup.Abort())
+		return err
+	} else {
+		// Write the output and clean up temp file
+		panicOnError(out.Close())
+		panicOnError(backup.Close())
+	}
+
+	log.Log("updated version file")
+
+	log.Log("Migration 14 to 15 succeeded")
+	return nil
+}
+
+// panicOnError is reserved for checks we can't solve transactionally if an error occurs
+func panicOnError(e error) {
+	if e != nil {
+		panic(fmt.Errorf("error can't be dealt with transactionally: %w", e))
+	}
+}
+
+func (m Migration) Revert(opts migrate.Options) error {
+	log.Verbose = opts.Verbose
+	log.Log("reverting migration")
+	lk, err := lock.Lock2(opts.Path)
+	if err != nil {
+		return err
+	}
+	defer lk.Close()
+
+	repo := mfsr.RepoPath(opts.Path)
+	if err := repo.CheckVersion("15"); err != nil {
+		return err
+	}
+
+	cfg := filepath.Join(opts.Path, "config")
+	if err := os.Rename(cfg+backupSuffix, cfg); err != nil {
+		return err
+	}
+
+	if err := repo.WriteVersion("14"); err != nil {
+		return err
+	}
+	if opts.Verbose {
+		log.Log("lowered version number to 14")
+	}
+
+	return nil
+}
+
+// convert converts the config from one version to another
+func convert(in io.Reader, out io.Writer) error {
+	confMap := make(map[string]any)
+	if err := json.NewDecoder(in).Decode(&confMap); err != nil {
+		return err
+	}
+
+	// Upgrade bootstrapper QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ from /quic to /quic-v1
+	if b, ok := confMap["Bootstrap"]; ok {
+		bootstrap, ok := b.([]interface{})
+		if !ok {
+			return fmt.Errorf("invalid type for .Bootstrap got %T expected json array", b)
+		}
+
+		for i, v := range bootstrap {
+			if v == "/ip4/104.131.131.82/udp/4001/quic/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" {
+				bootstrap[i] = "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"
+			}
+		}
+	}
+
+	// Remove /quic only addresses from .Addresses.Swarm
+	if a, ok := confMap["Addresses"]; ok {
+		addresses, ok := a.(map[string]any)
+		if !ok {
+			return fmt.Errorf("invalid type for .Addresses got %T expected json map", a)
+		}
+
+		for _, addressToRemove := range [...]string{"Swarm", "Announce", "AppendAnnounce", "NoAnnounce"} {
+			s, ok := addresses[addressToRemove]
+			if !ok {
+				continue
+			}
+
+			swarm, ok := s.([]interface{})
+			if !ok {
+				return fmt.Errorf("invalid type for .Addresses.%s got %T expected json array", addressToRemove, s)
+			}
+
+			var newSwarm []interface{}
+			for _, v := range swarm {
+				if addr, ok := v.(string); ok {
+					if !strings.Contains(addr, "/quic") || strings.Contains(addr, "/quic-v1") {
+						newSwarm = append(newSwarm, addr)
+					}
+				} else {
+					newSwarm = append(newSwarm, v)
+				}
+			}
+			addresses[addressToRemove] = newSwarm
+		}
+	}
+
+	// Remove legacy Gateway.HTTPHeaders values that were hardcoded since years ago, but no longer necessary
+	// (but leave as-is if user made any changes)
+	// https://github.com/ipfs/kubo/issues/10005
+	if a, ok := confMap["Gateway"]; ok {
+		addresses, ok := a.(map[string]any)
+		if !ok {
+			return fmt.Errorf("invalid type for .Gateway got %T expected json map", a)
+		}
+
+		if s, ok := addresses["HTTPHeaders"]; ok {
+			headers, ok := s.(map[string]any)
+			if !ok {
+				return fmt.Errorf("invalid type for .Gateway.HTTPHeaders got %T expected json map", s)
+			}
+
+			if acaos, ok := headers["Access-Control-Allow-Origin"].([]interface{}); ok && len(acaos) == 1 && acaos[0] == "*" {
+				delete(headers, "Access-Control-Allow-Origin")
+			} else {
+				return fmt.Errorf("invalid type for .Gateway.HTTPHeaders[Access-Control-Allow-Origin] got %T expected json array", headers["Access-Control-Allow-Origin"])
+			}
+
+			if acams, ok := headers["Access-Control-Allow-Methods"].([]interface{}); ok && len(acams) == 1 && acams[0] == "GET" {
+				delete(headers, "Access-Control-Allow-Methods")
+			} else {
+				return fmt.Errorf("invalid type for .Gateway.HTTPHeaders[Access-Control-Allow-Methods] got %T expected json array", headers["Access-Control-Allow-Methods"])
+			}
+			if acahs, ok := headers["Access-Control-Allow-Headers"].([]interface{}); ok && len(acahs) == 3 {
+				if acahs[0] == "X-Requested-With" && acahs[1] == "Range" && acahs[2] == "User-Agent" {
+					delete(headers, "Access-Control-Allow-Headers")
+				}
+			} else {
+				return fmt.Errorf("invalid type for .Gateway.HTTPHeaders[Access-Control-Allow-Headers] got %T expected json array", headers["Access-Control-Allow-Headers"])
+			}
+		}
+	}
+
+	// Save new config
+	fixed, err := json.MarshalIndent(confMap, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	if _, err := out.Write(fixed); err != nil {
+		return err
+	}
+	_, err = out.Write([]byte("\n"))
+	return err
+}

fs-repo-14-to-15/not-sharness/.gitignore

@@ -0,0 +1 @@
+repotest