Fixed migrations to allow upgrading from previous version

server/services/store/sqlstore/migrate.go

@@ -6,6 +6,8 @@ import (
 	"database/sql"
 	"embed"
 	"fmt"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+	"github.com/mattermost/morph/models"
 	"path/filepath"
 	"text/template"
 
@@ -19,6 +21,8 @@ import (
 	mysqldriver "github.com/go-sql-driver/mysql"
 	_ "github.com/lib/pq" // postgres driver
 
+	sq "github.com/Masterminds/squirrel"
+
 	"github.com/mattermost/focalboard/server/model"
 	"github.com/mattermost/mattermost-plugin-api/cluster"
 )
@@ -28,6 +32,8 @@ var assets embed.FS
 
 const (
 	uniqueIDsMigrationRequiredVersion = 14
+
+	tempSchemaMigrationTableName = "temp_schema_migration"
 )
 
 func appendMultipleStatementsFlag(connectionString string) (string, error) {
@@ -126,7 +132,7 @@ func (s *SQLStore) Migrate() error {
 		"plugin":   s.isPlugin,
 	}
 
-	src, err := mbindata.WithInstance(&mbindata.AssetSource{
+	migrationAssets := &mbindata.AssetSource{
 		Names: assetNamesForDriver,
 		AssetFunc: func(name string) ([]byte, error) {
 			asset, mErr := assets.ReadFile(filepath.Join("migrations", name))
@@ -147,7 +153,9 @@ func (s *SQLStore) Migrate() error {
 
 			return buffer.Bytes(), nil
 		},
-	})
+	}
+
+	src, err := mbindata.WithInstance(migrationAssets)
 	if err != nil {
 		return err
 	}
@@ -181,6 +189,10 @@ func (s *SQLStore) Migrate() error {
 		mutex.Lock()
 	}
 
+	if err := s.migrateSchemaVersionTable(src.Migrations()); err != nil {
+		return err
+	}
+
 	if err := ensureMigrationsAppliedUpToVersion(engine, driver, uniqueIDsMigrationRequiredVersion); err != nil {
 		return err
 	}
@@ -205,6 +217,10 @@ func (s *SQLStore) Migrate() error {
 		return fmt.Errorf("error running categoryID migration: %w", err)
 	}
 
+	if err := s.deleteOldSchemaMigrationTable(); err != nil {
+		return err
+	}
+
 	if s.isPlugin {
 		s.logger.Debug("Releasing cluster lock for Unique IDs migration")
 		mutex.Unlock()
@@ -213,6 +229,219 @@ func (s *SQLStore) Migrate() error {
 	return engine.ApplyAll()
 }
 
+// migrateSchemaVersionTable converts the schema version table from
+// the old format used by go-migrate to the new format used by
+// gomorph.
+// When running the Focalboard with go-migrate's schema version table
+// existing in the database, gomorph is unable to amke sense of it as it's
+// not in the format required by gomorph.
+func (s *SQLStore) migrateSchemaVersionTable(migrations []*models.Migration) error {
+	migrationNeeded, err := s.isSchemaMigrationNeeded()
+	if err != nil {
+		return err
+	}
+
+	if !migrationNeeded {
+		return nil
+	}
+
+	s.logger.Info("Migrating schema migration to new format")
+
+	legacySchemaVersion, err := s.getLegacySchemaVersion()
+	if err != nil {
+		return err
+	}
+
+	if err := s.createTempSchemaTable(); err != nil {
+		return err
+	}
+
+	if err := s.populateTempSchemaTable(migrations, legacySchemaVersion); err != nil {
+		return err
+	}
+
+	if err := s.useNewSchemaTable(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *SQLStore) isSchemaMigrationNeeded() (bool, error) {
+	// Check if `dirty` column exists on schema version table.
+	// This column exists only for the old schema version table.
+
+	// SQLite needs a bit of a special handling
+	if s.dbType == model.SqliteDBType {
+		return s.isSchemaMigrationNeededSQLite()
+	}
+
+	query := s.getQueryBuilder(s.db).
+		Select("count(*)").
+		From("information_schema.COLUMNS").
+		Where(sq.Eq{
+			"TABLE_NAME":  s.tablePrefix + "schema_migrations",
+			"COLUMN_NAME": "dirty",
+		})
+
+	row := query.QueryRow()
+
+	var count int
+	if err := row.Scan(&count); err != nil {
+		s.logger.Error("failed to check for columns of schema_migrations table", mlog.Err(err))
+		return false, err
+	}
+
+	return count == 1, nil
+}
+
+func (s *SQLStore) isSchemaMigrationNeededSQLite() (bool, error) {
+	// the way to check presence of a column is different
+	// for SQLite. Hence, the separate function
+
+	query := fmt.Sprintf("PRAGMA table_info(\"%sschema_migrations\");", s.tablePrefix)
+	rows, err := s.db.Query(query)
+	if err != nil {
+		s.logger.Error("SQLite - failed to check for columns in schema_migrations table", mlog.Err(err))
+		return false, err
+	}
+
+	defer s.CloseRows(rows)
+
+	data := [][]*string{}
+	for rows.Next() {
+		// PRAGMA returns 6 columns
+		row := make([]*string, 6)
+
+		err := rows.Scan(
+			&row[0],
+			&row[1],
+			&row[2],
+			&row[3],
+			&row[4],
+			&row[5],
+		)
+		if err != nil {
+			s.logger.Error("error scanning rows from SQLite schema_migrations table definition", mlog.Err(err))
+			return false, err
+		}
+
+		data = append(data, row)
+	}
+
+	nameColumnFound := false
+	for _, row := range data {
+		if len(row) >= 2 && *row[1] == "dirty" {
+			nameColumnFound = true
+			break
+		}
+	}
+
+	return nameColumnFound, nil
+}
+
+func (s *SQLStore) getLegacySchemaVersion() (uint32, error) {
+	query := s.getQueryBuilder(s.db).
+		Select("version").
+		From(s.tablePrefix + "schema_migrations")
+
+	row := query.QueryRow()
+
+	var version uint32
+	if err := row.Scan(&version); err != nil {
+		s.logger.Error("error fetching legacy schema version", mlog.Err(err))
+		s.logger.Error("getLegacySchemaVersion err " + err.Error())
+		return version, err
+	}
+
+	return version, nil
+}
+
+func (s *SQLStore) createTempSchemaTable() error {
+	// squirrel doesn't support DDL query in query builder
+	// so, we need to use a plain old string
+	query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (Version bigint NOT NULL, Name varchar(64) NOT NULL, PRIMARY KEY (Version))", s.tablePrefix+tempSchemaMigrationTableName)
+	if _, err := s.db.Exec(query); err != nil {
+		s.logger.Error("failed to create temporary schema migration table", mlog.Err(err))
+		s.logger.Error("createTempSchemaTable error  " + err.Error())
+		return err
+	}
+
+	return nil
+}
+func (s *SQLStore) populateTempSchemaTable(migrations []*models.Migration, legacySchemaVersion uint32) error {
+	query := s.getQueryBuilder(s.db).
+		Insert(s.tablePrefix+tempSchemaMigrationTableName).
+		Columns("Version", "Name")
+
+	for _, migration := range migrations {
+		// migrations param contains both up and down variant for
+		// each migration. Skipping for either one (down in this case)
+		// to process a migration only a single time.
+		if migration.Direction == models.Down {
+			continue
+		}
+
+		if migration.Version > legacySchemaVersion {
+			break
+		}
+
+		query = query.Values(migration.Version, migration.Name)
+	}
+
+	if _, err := query.Exec(); err != nil {
+		s.logger.Error("failed to insert migration records into temporary schema table", mlog.Err(err))
+		return err
+	}
+
+	return nil
+}
+
+func (s *SQLStore) useNewSchemaTable() error {
+	// first delete the old table, then
+	// rename the new table to old table's name
+
+	// renaming old schema migration table. Will delete later once the migration is
+	// complete, just in case.
+	var query string
+	if s.dbType == model.MysqlDBType {
+		query = fmt.Sprintf("RENAME TABLE `%sschema_migrations` TO `%sschema_migrations_old_temp`", s.tablePrefix, s.tablePrefix)
+	} else {
+		query = fmt.Sprintf("ALTER TABLE %sschema_migrations RENAME TO %sschema_migrations_old_temp", s.tablePrefix, s.tablePrefix)
+	}
+
+	s.logger.Error(query)
+
+	if _, err := s.db.Exec(query); err != nil {
+		s.logger.Error("failed to rename old schema migration table", mlog.Err(err))
+		return err
+	}
+
+	// renaming new temp table to old table's name
+	if s.dbType == model.MysqlDBType {
+		query = fmt.Sprintf("RENAME TABLE `%s%s` TO `%sschema_migrations`", s.tablePrefix, tempSchemaMigrationTableName, s.tablePrefix)
+	} else {
+		query = fmt.Sprintf("ALTER TABLE %s%s RENAME TO %sschema_migrations", s.tablePrefix, tempSchemaMigrationTableName, s.tablePrefix)
+	}
+
+	if _, err := s.db.Exec(query); err != nil {
+		s.logger.Error("failed to rename temp schema table", mlog.Err(err))
+		return err
+	}
+
+	return nil
+}
+
+func (s *SQLStore) deleteOldSchemaMigrationTable() error {
+	query := "DROP TABLE " + s.tablePrefix + "schema_migrations_old_temp"
+	if _, err := s.db.Exec(query); err != nil {
+		s.logger.Error("failed to delete old temp schema migrations table", mlog.Err(err))
+		return err
+	}
+
+	return nil
+}
+
 func ensureMigrationsAppliedUpToVersion(engine *morph.Morph, driver drivers.Driver, version int) error {
 	applied, err := driver.AppliedMigrations()
 	if err != nil {

server/services/store/sqlstore/migrations/000017_add_teams_and_boards.up.sql

@@ -41,7 +41,7 @@ CREATE TABLE {{.prefix}}boards (
     icon VARCHAR(256),
     show_description BOOLEAN,
     is_template BOOLEAN,
-    template_version INT NOT NULL DEFAULT 0, 
+    template_version INT DEFAULT 0,
     {{if .mysql}}
     properties JSON,
     card_properties JSON,
@@ -79,7 +79,7 @@ CREATE TABLE {{.prefix}}boards_history (
     icon VARCHAR(256),
     show_description BOOLEAN,
     is_template BOOLEAN,
-    template_version INT NOT NULL DEFAULT 0, 
+    template_version INT DEFAULT 0,
     {{if .mysql}}
     properties JSON,
     card_properties JSON,
@@ -109,7 +109,7 @@ CREATE TABLE {{.prefix}}boards_history (
   INSERT INTO {{.prefix}}boards (
       SELECT B.id, B.insert_at, C.TeamId, B.channel_id, B.created_by, B.modified_by, C.type, B.title, (B.fields->>'description')::text,
                  B.fields->>'icon', (B.fields->'showDescription')::text::boolean, (B.fields->'isTemplate')::text::boolean,
-                (B.fields->'templateVer')::text::int,
+                COALESCE((B.fields->'templateVer')::text, '0')::int,
                  '{}', B.fields->'cardProperties', B.fields->'columnCalculations', B.create_at,
                  B.update_at, B.delete_at
           FROM {{.prefix}}blocks AS B
@@ -119,7 +119,7 @@ CREATE TABLE {{.prefix}}boards_history (
   INSERT INTO {{.prefix}}boards_history (
       SELECT B.id, B.insert_at, C.TeamId, B.channel_id, B.created_by, B.modified_by, C.type, B.title, (B.fields->>'description')::text,
                  B.fields->>'icon', (B.fields->'showDescription')::text::boolean, (B.fields->'isTemplate')::text::boolean,
-                (B.fields->'templateVer')::text::int,
+                COALESCE((B.fields->'templateVer')::text, '0')::int,
                  '{}', B.fields->'cardProperties', B.fields->'columnCalculations', B.create_at,
                  B.update_at, B.delete_at
           FROM {{.prefix}}blocks_history AS B
@@ -130,8 +130,10 @@ CREATE TABLE {{.prefix}}boards_history (
   {{if .mysql}}
   INSERT INTO {{.prefix}}boards (
       SELECT B.id, B.insert_at, C.TeamId, B.channel_id, B.created_by, B.modified_by, C.Type, B.title, JSON_UNQUOTE(JSON_EXTRACT(B.fields,'$.description')),
-                 JSON_UNQUOTE(JSON_EXTRACT(B.fields,'$.icon')), B.fields->'$.showDescription', B.fields->'$.isTemplate',
-                 B.fields->'$.templateVer',
+                 JSON_UNQUOTE(JSON_EXTRACT(B.fields,'$.icon')),
+                 COALESCE(B.fields->'$.showDescription', 'false') = 'true',
+                 COALESCE(JSON_EXTRACT(B.fields, '$.isTemplate'), 'false') = 'true',
+                 COALESCE(B.fields->'$.templateVer', 0),
                  '{}', B.fields->'$.cardProperties', B.fields->'$.columnCalculations', B.create_at,
                  B.update_at, B.delete_at
           FROM {{.prefix}}blocks AS B
@@ -140,8 +142,10 @@ CREATE TABLE {{.prefix}}boards_history (
   );
   INSERT INTO {{.prefix}}boards_history (
       SELECT B.id, B.insert_at, C.TeamId, B.channel_id, B.created_by, B.modified_by, C.Type, B.title, JSON_UNQUOTE(JSON_EXTRACT(B.fields,'$.description')),
-                 JSON_UNQUOTE(JSON_EXTRACT(B.fields,'$.icon')), B.fields->'$.showDescription', B.fields->'$.isTemplate',
-                 B.fields->'$.templateVer',
+                 JSON_UNQUOTE(JSON_EXTRACT(B.fields,'$.icon')),
+                 COALESCE(B.fields->'$.showDescription', 'false') = 'true',
+                 COALESCE(JSON_EXTRACT(B.fields, '$.isTemplate'), 'false') = 'true',
+                 COALESCE(B.fields->'$.templateVer', 0),
                  '{}', B.fields->'$.cardProperties', B.fields->'$.columnCalculations', B.create_at,
                  B.update_at, B.delete_at
           FROM {{.prefix}}blocks_history AS B