diff --git a/pkg/storage/utils/decomposedfs/migrator/0001_create_spaces_directory_structure.go b/pkg/storage/utils/decomposedfs/migrator/0001_create_spaces_directory_structure.go index 4a45de1525..f915023e8f 100644 --- a/pkg/storage/utils/decomposedfs/migrator/0001_create_spaces_directory_structure.go +++ b/pkg/storage/utils/decomposedfs/migrator/0001_create_spaces_directory_structure.go @@ -29,12 +29,18 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" ) -// Migration0001 creates the spaces directory structure -func (m *Migrator) Up0001() (Result, error) { - m.log.Info().Msg("Migrating spaces directory structure...") +func init() { + registerMigration("0001", Migration0001{}) +} + +type Migration0001 struct{} + +// Migrate creates the spaces directory structure +func (m Migration0001) Migrate(migrator *Migrator) (Result, error) { + migrator.log.Info().Msg("Migrating spaces directory structure...") // create spaces folder and iterate over existing nodes to populate it - nodesPath := filepath.Join(m.lu.InternalRoot(), "nodes") + nodesPath := filepath.Join(migrator.lu.InternalRoot(), "nodes") fi, err := os.Stat(nodesPath) if err == nil && fi.IsDir() { f, err := os.Open(nodesPath) @@ -49,15 +55,15 @@ func (m *Migrator) Up0001() (Result, error) { for _, n := range nodes { nodePath := filepath.Join(nodesPath, n.Name()) - attr, err := m.lu.MetadataBackend().Get(context.Background(), nodePath, prefixes.ParentidAttr) + attr, err := migrator.lu.MetadataBackend().Get(context.Background(), nodePath, prefixes.ParentidAttr) if err == nil && string(attr) == node.RootID { - if err := m.moveNode(n.Name(), n.Name()); err != nil { - m.log.Error().Err(err). + if err := m.moveNode(migrator, n.Name(), n.Name()); err != nil { + migrator.log.Error().Err(err). Str("space", n.Name()). Msg("could not move space") continue } - m.linkSpaceNode("personal", n.Name()) + m.linkSpaceNode(migrator, "personal", n.Name()) } } // TODO delete nodesPath if empty @@ -65,8 +71,13 @@ func (m *Migrator) Up0001() (Result, error) { return stateSucceeded, nil } -func (m *Migrator) moveNode(spaceID, nodeID string) error { - dirPath := filepath.Join(m.lu.InternalRoot(), "nodes", nodeID) +// Rollback is not implemented +func (Migration0001) Rollback(_ *Migrator) (Result, error) { + return stateFailed, errors.New("rollback not implemented") +} + +func (m Migration0001) moveNode(migrator *Migrator, spaceID, nodeID string) error { + dirPath := filepath.Join(migrator.lu.InternalRoot(), "nodes", nodeID) f, err := os.Open(dirPath) if err != nil { return err @@ -76,10 +87,10 @@ func (m *Migrator) moveNode(spaceID, nodeID string) error { return err } for _, child := range children { - old := filepath.Join(m.lu.InternalRoot(), "nodes", child.Name()) - new := filepath.Join(m.lu.InternalRoot(), "spaces", lookup.Pathify(spaceID, 1, 2), "nodes", lookup.Pathify(child.Name(), 4, 2)) + old := filepath.Join(migrator.lu.InternalRoot(), "nodes", child.Name()) + new := filepath.Join(migrator.lu.InternalRoot(), "spaces", lookup.Pathify(spaceID, 1, 2), "nodes", lookup.Pathify(child.Name(), 4, 2)) if err := os.Rename(old, new); err != nil { - m.log.Error().Err(err). + migrator.log.Error().Err(err). Str("space", spaceID). Str("nodes", child.Name()). Str("oldpath", old). @@ -87,7 +98,7 @@ func (m *Migrator) moveNode(spaceID, nodeID string) error { Msg("could not rename node") } if child.IsDir() { - if err := m.moveNode(spaceID, child.Name()); err != nil { + if err := m.moveNode(migrator, spaceID, child.Name()); err != nil { return err } } @@ -96,27 +107,27 @@ func (m *Migrator) moveNode(spaceID, nodeID string) error { } // linkSpace creates a new symbolic link for a space with the given type st, and node id -func (m *Migrator) linkSpaceNode(spaceType, spaceID string) { - spaceTypesPath := filepath.Join(m.lu.InternalRoot(), "spacetypes", spaceType, spaceID) +func (m Migration0001) linkSpaceNode(migrator *Migrator, spaceType, spaceID string) { + spaceTypesPath := filepath.Join(migrator.lu.InternalRoot(), "spacetypes", spaceType, spaceID) expectedTarget := "../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2) linkTarget, err := os.Readlink(spaceTypesPath) if errors.Is(err, os.ErrNotExist) { err = os.Symlink(expectedTarget, spaceTypesPath) if err != nil { - m.log.Error().Err(err). + migrator.log.Error().Err(err). Str("space_type", spaceType). Str("space", spaceID). Msg("could not create symlink") } } else { if err != nil { - m.log.Error().Err(err). + migrator.log.Error().Err(err). Str("space_type", spaceType). Str("space", spaceID). Msg("could not read symlink") } if linkTarget != expectedTarget { - m.log.Warn(). + migrator.log.Warn(). Str("space_type", spaceType). Str("space", spaceID). Str("expected", expectedTarget). diff --git a/pkg/storage/utils/decomposedfs/migrator/0002_move_spacetypes_to_indexes.go b/pkg/storage/utils/decomposedfs/migrator/0002_move_spacetypes_to_indexes.go index de0804f42a..1250c69cfe 100644 --- a/pkg/storage/utils/decomposedfs/migrator/0002_move_spacetypes_to_indexes.go +++ b/pkg/storage/utils/decomposedfs/migrator/0002_move_spacetypes_to_indexes.go @@ -19,6 +19,7 @@ package migrator import ( + "errors" "io" "os" "path/filepath" @@ -26,11 +27,17 @@ import ( "github.com/cs3org/reva/v2/pkg/logger" ) -// Migration0002 migrates spacetypes to indexes -func (m *Migrator) Up0002() (Result, error) { - m.log.Info().Msg("Migrating space types indexes...") +func init() { + registerMigration("0002", Migration0002{}) +} + +type Migration0002 struct{} + +// Migrate migrates spacetypes to indexes +func (m Migration0002) Migrate(migrator *Migrator) (Result, error) { + migrator.log.Info().Msg("Migrating space types indexes...") - spaceTypesPath := filepath.Join(m.lu.InternalRoot(), "spacetypes") + spaceTypesPath := filepath.Join(migrator.lu.InternalRoot(), "spacetypes") fi, err := os.Stat(spaceTypesPath) if err == nil && fi.IsDir() { @@ -44,7 +51,7 @@ func (m *Migrator) Up0002() (Result, error) { } for _, st := range spaceTypes { - err := m.moveSpaceType(st.Name()) + err := m.moveSpaceType(migrator, st.Name()) if err != nil { logger.New().Error().Err(err). Str("space", st.Name()). @@ -80,8 +87,13 @@ func (m *Migrator) Up0002() (Result, error) { return stateSucceeded, nil } -func (m *Migrator) moveSpaceType(spaceType string) error { - dirPath := filepath.Join(m.lu.InternalRoot(), "spacetypes", spaceType) +// Rollback is not implemented +func (Migration0002) Rollback(_ *Migrator) (Result, error) { + return stateFailed, errors.New("rollback not implemented") +} + +func (m Migration0002) moveSpaceType(migrator *Migrator, spaceType string) error { + dirPath := filepath.Join(migrator.lu.InternalRoot(), "spacetypes", spaceType) f, err := os.Open(dirPath) if err != nil { return err @@ -91,7 +103,7 @@ func (m *Migrator) moveSpaceType(spaceType string) error { return err } for _, child := range children { - old := filepath.Join(m.lu.InternalRoot(), "spacetypes", spaceType, child.Name()) + old := filepath.Join(migrator.lu.InternalRoot(), "spacetypes", spaceType, child.Name()) target, err := os.Readlink(old) if err != nil { logger.New().Error().Err(err). @@ -101,7 +113,7 @@ func (m *Migrator) moveSpaceType(spaceType string) error { Msg("could not read old symlink") continue } - newDir := filepath.Join(m.lu.InternalRoot(), "indexes", "by-type", spaceType) + newDir := filepath.Join(migrator.lu.InternalRoot(), "indexes", "by-type", spaceType) if err := os.MkdirAll(newDir, 0700); err != nil { logger.New().Error().Err(err). Str("space", spaceType). diff --git a/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go b/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go index ef1c299d70..73c919d0fa 100644 --- a/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go +++ b/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go @@ -31,23 +31,29 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" ) -// Migration0003 migrates the file metadata to the current backend. +func init() { + registerMigration("0003", Migration0003{}) +} + +type Migration0003 struct{} + +// Migrate migrates the file metadata to the current backend. // Only the xattrs -> messagepack path is supported. -func (m *Migrator) Up0003() (Result, error) { - bod := lookup.DetectBackendOnDisk(m.lu.InternalRoot()) +func (m Migration0003) Migrate(migrator *Migrator) (Result, error) { + bod := lookup.DetectBackendOnDisk(migrator.lu.InternalRoot()) if bod == "" { return stateFailed, errors.New("could not detect metadata backend on disk") } - if bod != "xattrs" || m.lu.MetadataBackend().Name() != "messagepack" { + if bod != "xattrs" || migrator.lu.MetadataBackend().Name() != "messagepack" { return stateSucceededRunAgain, nil } - m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating to messagepack metadata backend...") + migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating to messagepack metadata backend...") xattrs := metadata.XattrsBackend{} - mpk := metadata.NewMessagePackBackend(m.lu.InternalRoot(), cache.Config{}) + mpk := metadata.NewMessagePackBackend(migrator.lu.InternalRoot(), cache.Config{}) - spaces, _ := filepath.Glob(filepath.Join(m.lu.InternalRoot(), "spaces", "*", "*")) + spaces, _ := filepath.Glob(filepath.Join(migrator.lu.InternalRoot(), "spaces", "*", "*")) for _, space := range spaces { err := filepath.WalkDir(filepath.Join(space, "nodes"), func(path string, _ fs.DirEntry, err error) error { // Do not continue on error @@ -77,7 +83,7 @@ func (m *Migrator) Up0003() (Result, error) { attribs, err := xattrs.All(context.Background(), path) if err != nil { - m.log.Error().Err(err).Str("path", path).Msg("error converting file") + migrator.log.Error().Err(err).Str("path", path).Msg("error converting file") return err } if len(attribs) == 0 { @@ -86,24 +92,29 @@ func (m *Migrator) Up0003() (Result, error) { err = mpk.SetMultiple(context.Background(), path, attribs, false) if err != nil { - m.log.Error().Err(err).Str("path", path).Msg("error setting attributes") + migrator.log.Error().Err(err).Str("path", path).Msg("error setting attributes") return err } for k := range attribs { err = xattrs.Remove(context.Background(), path, k) if err != nil { - m.log.Debug().Err(err).Str("path", path).Msg("error removing xattr") + migrator.log.Debug().Err(err).Str("path", path).Msg("error removing xattr") } } return nil }) if err != nil { - m.log.Error().Err(err).Msg("error migrating nodes to messagepack metadata backend") + migrator.log.Error().Err(err).Msg("error migrating nodes to messagepack metadata backend") } } - m.log.Info().Msg("done.") + migrator.log.Info().Msg("done.") return stateSucceeded, nil } + +// Rollback is not implemented +func (Migration0003) Rollback(_ *Migrator) (Result, error) { + return stateFailed, errors.New("rollback not implemented") +} diff --git a/pkg/storage/utils/decomposedfs/migrator/0004_switch_to_messagepack_space_index.go b/pkg/storage/utils/decomposedfs/migrator/0004_switch_to_messagepack_space_index.go index 98389943be..87dd226c0c 100644 --- a/pkg/storage/utils/decomposedfs/migrator/0004_switch_to_messagepack_space_index.go +++ b/pkg/storage/utils/decomposedfs/migrator/0004_switch_to_messagepack_space_index.go @@ -26,14 +26,20 @@ import ( "github.com/shamaton/msgpack/v2" ) -// Migration0004 migrates the directory tree based space indexes to messagepack -func (m *Migrator) Up0004() (Result, error) { - root := m.lu.InternalRoot() +func init() { + registerMigration("0004", Migration0004{}) +} + +type Migration0004 struct{} + +// Migrate migrates the directory tree based space indexes to messagepack +func (Migration0004) Migrate(migrator *Migrator) (Result, error) { + root := migrator.lu.InternalRoot() // migrate user indexes users, err := os.ReadDir(filepath.Join(root, "indexes", "by-user-id")) if err != nil { - m.log.Warn().Err(err).Msg("error listing user indexes") + migrator.log.Warn().Err(err).Msg("error listing user indexes") } for _, user := range users { if !user.IsDir() { @@ -43,17 +49,17 @@ func (m *Migrator) Up0004() (Result, error) { indexPath := filepath.Join(root, "indexes", "by-user-id", id+".mpk") dirIndexPath := filepath.Join(root, "indexes", "by-user-id", id) - m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...") + migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...") err := migrateSpaceIndex(indexPath, dirIndexPath) if err != nil { - m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index") + migrator.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index") } } // migrate group indexes groups, err := os.ReadDir(filepath.Join(root, "indexes", "by-group-id")) if err != nil { - m.log.Warn().Err(err).Msg("error listing group indexes") + migrator.log.Warn().Err(err).Msg("error listing group indexes") } for _, group := range groups { if !group.IsDir() { @@ -63,10 +69,10 @@ func (m *Migrator) Up0004() (Result, error) { indexPath := filepath.Join(root, "indexes", "by-group-id", id+".mpk") dirIndexPath := filepath.Join(root, "indexes", "by-group-id", id) - m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...") + migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...") err := migrateSpaceIndex(indexPath, dirIndexPath) if err != nil { - m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index") + migrator.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index") } } @@ -80,14 +86,14 @@ func (m *Migrator) Up0004() (Result, error) { continue } - m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...") + migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...") err = migrateSpaceIndex(indexPath, dirIndexPath) if err != nil { - m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index") + migrator.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index") } } - m.log.Info().Msg("done.") + migrator.log.Info().Msg("done.") return stateSucceeded, nil } @@ -117,8 +123,8 @@ func migrateSpaceIndex(indexPath, dirIndexPath string) error { return os.RemoveAll(dirIndexPath) } -// Down0004 migrates the directory messagepack indexes to symlinks -func (m *Migrator) Down0004() (Result, error) { +// Rollback migrates the directory messagepack indexes to symlinks +func (Migration0004) Rollback(m *Migrator) (Result, error) { root := m.lu.InternalRoot() // migrate user indexes diff --git a/pkg/storage/utils/decomposedfs/migrator/0005_fix_messagepack_space_index_format.go b/pkg/storage/utils/decomposedfs/migrator/0005_fix_messagepack_space_index_format.go index 014c2114ea..d3a7bded47 100644 --- a/pkg/storage/utils/decomposedfs/migrator/0005_fix_messagepack_space_index_format.go +++ b/pkg/storage/utils/decomposedfs/migrator/0005_fix_messagepack_space_index_format.go @@ -25,16 +25,22 @@ import ( "github.com/shamaton/msgpack/v2" ) -// Migration0005 fixes the messagepack space index data structure -func (m *Migrator) Up0005() (Result, error) { - root := m.lu.InternalRoot() +func init() { + registerMigration("0005", Migration0005{}) +} + +type Migration0005 struct{} + +// Migrate fixes the messagepack space index data structure +func (Migration0005) Migrate(migrator *Migrator) (Result, error) { + root := migrator.lu.InternalRoot() indexes, err := filepath.Glob(filepath.Join(root, "indexes", "**", "*.mpk")) if err != nil { return stateFailed, err } for _, i := range indexes { - m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Fixing index format of " + i) + migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Fixing index format of " + i) // Read old-format index oldData, err := os.ReadFile(i) @@ -45,7 +51,7 @@ func (m *Migrator) Up0005() (Result, error) { err = msgpack.Unmarshal(oldData, &oldIndex) if err != nil { // likely already migrated -> skip - m.log.Warn().Str("root", m.lu.InternalRoot()).Msg("Invalid index format found in " + i) + migrator.log.Warn().Str("root", migrator.lu.InternalRoot()).Msg("Invalid index format found in " + i) continue } @@ -63,19 +69,20 @@ func (m *Migrator) Up0005() (Result, error) { return stateFailed, err } } - m.log.Info().Msg("done.") + migrator.log.Info().Msg("done.") return stateSucceeded, nil } -func (m *Migrator) Down0005() (Result, error) { - root := m.lu.InternalRoot() +// Rollback rolls back the migration +func (Migration0005) Rollback(migrator *Migrator) (Result, error) { + root := migrator.lu.InternalRoot() indexes, err := filepath.Glob(filepath.Join(root, "indexes", "**", "*.mpk")) if err != nil { return stateFailed, err } for _, i := range indexes { - m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Fixing index format of " + i) + migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Fixing index format of " + i) oldData, err := os.ReadFile(i) if err != nil { @@ -84,7 +91,7 @@ func (m *Migrator) Down0005() (Result, error) { oldIndex := map[string]string{} err = msgpack.Unmarshal(oldData, &oldIndex) if err != nil { - m.log.Warn().Str("root", m.lu.InternalRoot()).Msg("Invalid index format found in " + i) + migrator.log.Warn().Str("root", migrator.lu.InternalRoot()).Msg("Invalid index format found in " + i) continue } @@ -102,6 +109,6 @@ func (m *Migrator) Down0005() (Result, error) { return stateFailed, err } } - m.log.Info().Msg("done.") + migrator.log.Info().Msg("done.") return stateDown, nil } diff --git a/pkg/storage/utils/decomposedfs/migrator/migrator.go b/pkg/storage/utils/decomposedfs/migrator/migrator.go index 98f1367ea8..39bbe9a984 100644 --- a/pkg/storage/utils/decomposedfs/migrator/migrator.go +++ b/pkg/storage/utils/decomposedfs/migrator/migrator.go @@ -23,15 +23,13 @@ import ( "fmt" "os" "path/filepath" - "reflect" + "sort" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/rogpeppe/go-internal/lockedfile" "github.com/rs/zerolog" ) -var allMigrations = []string{"0001", "0002", "0003", "0004", "0005"} - const ( statePending = "pending" stateFailed = "failed" @@ -40,12 +38,35 @@ const ( stateSucceededRunAgain = "runagain" ) +type migration interface { + Migrate(*Migrator) (Result, error) + Rollback(*Migrator) (Result, error) +} + +var migrations = map[string]migration{} + +type migrationStates map[string]MigrationState + +func registerMigration(name string, migration migration) { + migrations[name] = migration +} + +func allMigrations() []string { + ms := []string{} + + for k, _ := range migrations { + ms = append(ms, k) + } + + sort.Strings(ms) + return ms +} + // MigrationState holds the state of a migration type MigrationState struct { State string Message string } -type migrationStates map[string]MigrationState // Result represents the result of a migration run type Result string @@ -73,7 +94,7 @@ func (m *Migrator) Migrations() (map[string]MigrationState, error) { } states := map[string]MigrationState{} - for _, migration := range allMigrations { + for _, migration := range allMigrations() { if s, ok := m.states[migration]; ok { states[migration] = s } else { @@ -88,14 +109,7 @@ func (m *Migrator) Migrations() (map[string]MigrationState, error) { // RunMigration runs or rolls back a migration func (m *Migrator) RunMigration(id string, rollback bool) error { - validMigration := false - for _, m := range allMigrations { - if m == id { - validMigration = true - break - } - } - if !validMigration { + if _, ok := migrations[id]; !ok { return fmt.Errorf("invalid migration '%s'", id) } @@ -110,26 +124,20 @@ func (m *Migrator) RunMigration(id string, rollback bool) error { return err } - method := "Up" + id - if rollback { - method = "Down" + id + var res Result + if !rollback { + m.log.Info().Msg("Running migration " + id + "...") + res, err = migrations[id].Migrate(m) + } else { + m.log.Info().Msg("Rolling back migration " + id + "...") + res, err = migrations[id].Rollback(m) } - msg := "Running migration " + id + "..." - if rollback { - msg += " (down)" - } - m.log.Info().Msg(msg) - - migrateMethod := reflect.ValueOf(m).MethodByName(method) - v := migrateMethod.Call(nil) - // write back state s := m.states[id] - s.State = string(v[0].Interface().(Result)) + s.State = string(res) - if v[1].Interface() != nil { - err := v[1].Interface().(error) + if err != nil { m.log.Error().Err(err).Msg("migration " + id + " failed") s.Message = err.Error() } @@ -157,17 +165,15 @@ func (m *Migrator) RunMigrations() error { return err } - for _, migration := range allMigrations { + for _, migration := range allMigrations() { s := m.states[migration] if s.State == stateSucceeded || s.State == stateDown { continue } - migrateMethod := reflect.ValueOf(m).MethodByName("Up" + migration) - v := migrateMethod.Call(nil) - s.State = string(v[0].Interface().(Result)) - if v[1].Interface() != nil { - err := v[1].Interface().(error) + res, err := migrations[migration].Migrate(m) + s.State = string(res) + if err != nil { m.log.Error().Err(err).Msg("migration " + migration + " failed") s.Message = err.Error() }