From 544769307bf79e98b5f8a1ed68bc836b72e89f3d Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Tue, 18 Aug 2020 16:21:03 +0300 Subject: [PATCH 01/17] feat: implement db migrations add db-migration command to cli --- migrations/index.ts | 64 ++++++++++++++++++ package.json | 2 + src/cli/db-migration.ts | 141 ++++++++++++++++++++++++++++++++++++++++ src/sequelize.ts | 7 +- 4 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 migrations/index.ts create mode 100644 src/cli/db-migration.ts diff --git a/migrations/index.ts b/migrations/index.ts new file mode 100644 index 00000000..ee974932 --- /dev/null +++ b/migrations/index.ts @@ -0,0 +1,64 @@ +import path from 'path' +import Umzug from 'umzug' + +import { loggingFactory } from '../src/logger' +import { Sequelize } from 'sequelize-typescript' + +const logger = loggingFactory('db:migrations') + +export class Migration { + private umzugIns: Umzug.Umzug + + constructor (sequelize: Sequelize) { + this.umzugIns = new Umzug({ + storage: 'sequelize', + logging: logger.info, + storageOptions: { sequelize }, + migrations: { + path: path.resolve(__dirname, './scripts'), + params: [sequelize.getQueryInterface(), sequelize], + pattern: /^\d+[\w-]+\.ts$/ + } + }) + } + + // eslint-disable-next-line require-await + async up (options?: string | string[] | Umzug.UpToOptions | Umzug.UpDownMigrationsOptions): Promise { + return this.umzugIns.up(options as any) + } + + // eslint-disable-next-line require-await + async down (options?: string | string[] | Umzug.DownToOptions | Umzug.UpDownMigrationsOptions): Promise { + return this.umzugIns.down(options as any) + } + + get on (): Function { + return this.umzugIns.on + } + + // eslint-disable-next-line require-await + async pending (): Promise { + return this.umzugIns.pending().catch(e => { + if (e.code === 'ENOENT') return [] + throw e + }) + } + + // eslint-disable-next-line require-await + async executed (): Promise { + return this.umzugIns.executed() + } +} + +export default class DbMigration { + private static ins: Migration | undefined + + static getInstance (sequelize?: Sequelize): Migration { + if (!DbMigration.ins) { + if (!sequelize) throw new Error('You need to provide Sequelize instance') + DbMigration.ins = new Migration(sequelize) + } + + return DbMigration.ins + } +} diff --git a/package.json b/package.json index ff70861f..644b2fc0 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "@rsksmart/rns-auction-registrar": "1.0.2", "@rsksmart/rns-reverse": "^1.0.2", "@rsksmart/rns-rskregistrar": "^1.2.1", + "@types/umzug": "^2.2.3", "abi-decoder": "^2.3.0", "async-sema": "^3.1.0", "bignumber.js": "^9.0.0", @@ -86,6 +87,7 @@ "sequelize-typescript": "^1.1.0", "sql-formatter": "^2.3.3", "sqlite3": "^5.0.0", + "umzug": "^2.3.0", "web3-core": "^1.3.0", "web3-core-subscriptions": "^1.2.11", "web3-eth": "^1.2.11", diff --git a/src/cli/db-migration.ts b/src/cli/db-migration.ts new file mode 100644 index 00000000..e24780ef --- /dev/null +++ b/src/cli/db-migration.ts @@ -0,0 +1,141 @@ +import fs from 'fs' +import path from 'path' +import { flags } from '@oclif/command' +import { OutputFlags } from '@oclif/parser' + +import DbMigration from '../../migrations' +import { BaseCLICommand } from '../utils' +import { sequelizeFactory } from '../sequelize' +import config from 'config' + +const MigrationTemplate = `import { QueryInterface } from 'sequelize' +import { Sequelize } from 'sequelize-typescript' + +export default { + // eslint-disable-next-line require-await + async up (queryInterface: QueryInterface, sequelize: Sequelize): Promise { + return Promise.reject(Error('Not implemented')) + }, + // eslint-disable-next-line require-await + async down (queryInterface: QueryInterface, sequelize: Sequelize): Promise { + return Promise.reject(Error('Not implemented')) + } +} +` + +export default class DbMigrationCommand extends BaseCLICommand { + static hidden: boolean; + static flags = { + ...BaseCLICommand.flags, + db: flags.string({ description: 'database connection URI', env: 'RIFM_DB' }), + up: flags.boolean({ + char: 'u', + description: 'Migrate DB', + exclusive: ['down', 'generate'] + }), + down: flags.boolean({ + char: 'd', + description: 'Undo db migrations', + exclusive: ['up', 'generate'] + }), + generate: flags.string({ + char: 'd', + description: 'Generate migrations', + exclusive: ['up', 'down'] + }), + to: flags.string({ + char: 't', + description: 'Migrate to' + }), + migration: flags.string({ + char: 'm', + description: 'Migration file', + multiple: true + }) + } + + static description = 'DB migrations' + + static examples = [ + '$ rif-marketplace-cache db --up', + '$ rif-marketplace-cache db --down', + '$ rif-marketplace-cache db --up --to 0-test', + '$ rif-marketplace-cache --up --migrations 01-test --migrations 02-test', + '$ rif-marketplace-cache --up --db ./test.sqlite --to 09-test', + '$ rif-marketplace-cache --down --db ./test.sqlite --to 09-test' + ] + + protected resolveDbPath (db: string): string { + if (!db) return path.resolve(this.config.dataDir, config.get('db')) + + const parsed = path.parse(db) + + // File name + if (!parsed.dir) { + return path.resolve( + this.config.dataDir, + parsed.ext + ? db + : `${parsed.base}.sqlite` + ) + } else { + if (db[db.length - 1] === '/') throw new Error('Path should include the file name') + return path.resolve(`${db}${parsed.ext ? '' : '.sqlite'}`) + } + } + + async migrate (migrations?: string[], options?: { to: string }): Promise { + this.log('DB migrations') + await DbMigration.getInstance().up(options) + this.log('Done') + } + + async undo (migrations?: string[], options?: { to: string }): Promise { + this.log('Undo DB migrations') + await DbMigration.getInstance().down(options) + this.log('Done') + } + + generateMigration (name: string): void { + const migrationsFolder = path.resolve(process.cwd(), './migrations') + const scriptsFolder = path.resolve(process.cwd(), './migrations/scripts') + const fileName = `./${Date.now()}-${name}.ts` + const filePath = path.resolve(scriptsFolder, fileName) + + if (!fs.existsSync(migrationsFolder)) { + throw new Error('Migrations folder not found. Please run command from project root and make sure that you have \'migrations\' folder setup') + } + + this.log(`Creating migration ${fileName}`) + + if (!fs.existsSync(scriptsFolder)) { + fs.mkdirSync(scriptsFolder) + } + + fs.writeFileSync(filePath, MigrationTemplate) + this.log('Done') + } + + async run (): Promise { + const { flags: originalFlags } = this.parse(DbMigrationCommand) + const flags = originalFlags as OutputFlags + + if (flags.db) { + config.util.extendDeep(config, { db: `sqlite://${this.resolveDbPath(flags.db)}` }) + } + + if (flags.generate) this.generateMigration(flags.generate) + + // Init database connection + const sequelize = sequelizeFactory() + DbMigration.getInstance(sequelize) + + if (!flags.up && !flags.down && !flags.generate) throw new Error('One of \'--generate, --up, --down\' required') + + if (flags.up) await this.migrate(flags.migration, flags) + + if (flags.down) await this.undo(flags.migration, flags) + + this.exit() + } +} diff --git a/src/sequelize.ts b/src/sequelize.ts index 4f7dcb5c..6a73fe37 100644 --- a/src/sequelize.ts +++ b/src/sequelize.ts @@ -7,6 +7,7 @@ import sqlFormatter from 'sql-formatter' import { Application } from './definitions' import { loggingFactory } from './logger' +import DbMigration from '../migrations' const logger = loggingFactory('db') @@ -58,12 +59,16 @@ export function BigNumberStringType (propName: string): Partial { const sequelize = sequelizeFactory() const oldSetup = app.setup app.set('sequelize', sequelize) + // Run migration + logger.info('Run DB Migrations') + await DbMigration.getInstance(app.get('sequelizeSync')).up() + app.setup = function (...args): ReturnType { const result = oldSetup.apply(this, args) From d3b0c17df2a4131a5038dd2bdb4e8952ae24e940 Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Tue, 18 Aug 2020 16:27:15 +0300 Subject: [PATCH 02/17] feat: fix lint --- migrations/index.ts | 17 ++++++++++++----- src/cli/db-migration.ts | 30 +++++++++++++++++++++--------- src/sequelize.ts | 2 +- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/migrations/index.ts b/migrations/index.ts index ee974932..8929e494 100644 --- a/migrations/index.ts +++ b/migrations/index.ts @@ -6,8 +6,11 @@ import { Sequelize } from 'sequelize-typescript' const logger = loggingFactory('db:migrations') +type UpOptions = string | string[] | Umzug.UpToOptions | Umzug.UpDownMigrationsOptions +type DownOptions = string | string[] | Umzug.DownToOptions | Umzug.UpDownMigrationsOptions + export class Migration { - private umzugIns: Umzug.Umzug + private readonly umzugIns: Umzug.Umzug constructor (sequelize: Sequelize) { this.umzugIns = new Umzug({ @@ -23,12 +26,12 @@ export class Migration { } // eslint-disable-next-line require-await - async up (options?: string | string[] | Umzug.UpToOptions | Umzug.UpDownMigrationsOptions): Promise { + async up (options?: UpOptions): Promise { return this.umzugIns.up(options as any) } // eslint-disable-next-line require-await - async down (options?: string | string[] | Umzug.DownToOptions | Umzug.UpDownMigrationsOptions): Promise { + async down (options?: DownOptions): Promise { return this.umzugIns.down(options as any) } @@ -39,7 +42,9 @@ export class Migration { // eslint-disable-next-line require-await async pending (): Promise { return this.umzugIns.pending().catch(e => { - if (e.code === 'ENOENT') return [] + if (e.code === 'ENOENT') { + return [] + } throw e }) } @@ -55,7 +60,9 @@ export default class DbMigration { static getInstance (sequelize?: Sequelize): Migration { if (!DbMigration.ins) { - if (!sequelize) throw new Error('You need to provide Sequelize instance') + if (!sequelize) { + throw new Error('You need to provide Sequelize instance') + } DbMigration.ins = new Migration(sequelize) } diff --git a/src/cli/db-migration.ts b/src/cli/db-migration.ts index e24780ef..5b62d3ef 100644 --- a/src/cli/db-migration.ts +++ b/src/cli/db-migration.ts @@ -66,7 +66,9 @@ export default class DbMigrationCommand extends BaseCLICommand { ] protected resolveDbPath (db: string): string { - if (!db) return path.resolve(this.config.dataDir, config.get('db')) + if (!db) { + return path.resolve(this.config.dataDir, config.get('db')) + } const parsed = path.parse(db) @@ -79,7 +81,9 @@ export default class DbMigrationCommand extends BaseCLICommand { : `${parsed.base}.sqlite` ) } else { - if (db[db.length - 1] === '/') throw new Error('Path should include the file name') + if (db[db.length - 1] === '/') { + throw new Error('Path should include the file name') + } return path.resolve(`${db}${parsed.ext ? '' : '.sqlite'}`) } } @@ -118,23 +122,31 @@ export default class DbMigrationCommand extends BaseCLICommand { async run (): Promise { const { flags: originalFlags } = this.parse(DbMigrationCommand) - const flags = originalFlags as OutputFlags + const parsedFlags = originalFlags as OutputFlags - if (flags.db) { - config.util.extendDeep(config, { db: `sqlite://${this.resolveDbPath(flags.db)}` }) + if (parsedFlags.db) { + config.util.extendDeep(config, { db: `sqlite://${this.resolveDbPath(parsedFlags.db)}` }) } - if (flags.generate) this.generateMigration(flags.generate) + if (parsedFlags.generate) { + this.generateMigration(parsedFlags.generate) + } // Init database connection const sequelize = sequelizeFactory() DbMigration.getInstance(sequelize) - if (!flags.up && !flags.down && !flags.generate) throw new Error('One of \'--generate, --up, --down\' required') + if (!parsedFlags.up && !parsedFlags.down && !parsedFlags.generate) { + throw new Error('One of \'--generate, --up, --down\' required') + } - if (flags.up) await this.migrate(flags.migration, flags) + if (parsedFlags.up) { + await this.migrate(parsedFlags.migration, parsedFlags) + } - if (flags.down) await this.undo(flags.migration, flags) + if (parsedFlags.down) { + await this.undo(parsedFlags.migration, parsedFlags) + } this.exit() } diff --git a/src/sequelize.ts b/src/sequelize.ts index 6a73fe37..73e8d869 100644 --- a/src/sequelize.ts +++ b/src/sequelize.ts @@ -67,7 +67,7 @@ export default async function (app: Application): Promise { // Run migration logger.info('Run DB Migrations') - await DbMigration.getInstance(app.get('sequelizeSync')).up() + await DbMigration.getInstance(sequelize).up() app.setup = function (...args): ReturnType { const result = oldSetup.apply(this, args) From e65755d2a983ad7e1887b25e269ca8c1274af9e7 Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Tue, 18 Aug 2020 16:34:42 +0300 Subject: [PATCH 03/17] feat: fix db path resolution --- src/cli/db-migration.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cli/db-migration.ts b/src/cli/db-migration.ts index 5b62d3ef..372af7db 100644 --- a/src/cli/db-migration.ts +++ b/src/cli/db-migration.ts @@ -67,7 +67,8 @@ export default class DbMigrationCommand extends BaseCLICommand { protected resolveDbPath (db: string): string { if (!db) { - return path.resolve(this.config.dataDir, config.get('db')) + const dbFromConfig = path.resolve(this.config.dataDir, config.get('db')) + return dbFromConfig.split(':')[1] } const parsed = path.parse(db) @@ -125,7 +126,7 @@ export default class DbMigrationCommand extends BaseCLICommand { const parsedFlags = originalFlags as OutputFlags if (parsedFlags.db) { - config.util.extendDeep(config, { db: `sqlite://${this.resolveDbPath(parsedFlags.db)}` }) + config.util.extendDeep(config, { db: `sqlite:${this.resolveDbPath(parsedFlags.db)}` }) } if (parsedFlags.generate) { From c3a135b821d2fb5827153c99c2ed6de2de2fe91b Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Thu, 20 Aug 2020 10:08:57 +0300 Subject: [PATCH 04/17] feat: resolve pr comments --- migrations/index.ts | 2 +- package.json | 2 +- src/cli/db-migration.ts | 2 +- src/sequelize.ts | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/migrations/index.ts b/migrations/index.ts index 8929e494..1c6b537b 100644 --- a/migrations/index.ts +++ b/migrations/index.ts @@ -45,7 +45,7 @@ export class Migration { if (e.code === 'ENOENT') { return [] } - throw e + return Promise.reject(e) }) } diff --git a/package.json b/package.json index 644b2fc0..b4355095 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,6 @@ "@rsksmart/rns-auction-registrar": "1.0.2", "@rsksmart/rns-reverse": "^1.0.2", "@rsksmart/rns-rskregistrar": "^1.2.1", - "@types/umzug": "^2.2.3", "abi-decoder": "^2.3.0", "async-sema": "^3.1.0", "bignumber.js": "^9.0.0", @@ -120,6 +119,7 @@ "@types/sql-formatter": "^2.3.0", "@types/validator": "^13.1.0", "bignumber.js": "^9.0.0", + "@types/umzug": "^2.2.3", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "cross-env": "^7.0.2", diff --git a/src/cli/db-migration.ts b/src/cli/db-migration.ts index 372af7db..5367a617 100644 --- a/src/cli/db-migration.ts +++ b/src/cli/db-migration.ts @@ -40,7 +40,7 @@ export default class DbMigrationCommand extends BaseCLICommand { }), generate: flags.string({ char: 'd', - description: 'Generate migrations', + description: 'Generate migrations using template [--generate=migration_name]', exclusive: ['up', 'down'] }), to: flags.string({ diff --git a/src/sequelize.ts b/src/sequelize.ts index 73e8d869..73b14850 100644 --- a/src/sequelize.ts +++ b/src/sequelize.ts @@ -65,9 +65,9 @@ export default async function (app: Application): Promise { app.set('sequelize', sequelize) - // Run migration - logger.info('Run DB Migrations') - await DbMigration.getInstance(sequelize).up() + if ((await DbMigration.getInstance(sequelize).pending()).length) { + throw new Error('DB Migration required. Please use \'db-migration\' command to proceed') + } app.setup = function (...args): ReturnType { const result = oldSetup.apply(this, args) From 8c4fbba759aa03853d5900a3727131e7d426bebd Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Thu, 20 Aug 2020 10:10:44 +0300 Subject: [PATCH 05/17] feat: add init migration with db sync --- migrations/scripts/1597906360418-init.ts | 11 +++++++++++ package.json | 3 ++- src/cli/db-migration.ts | 10 ++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 migrations/scripts/1597906360418-init.ts diff --git a/migrations/scripts/1597906360418-init.ts b/migrations/scripts/1597906360418-init.ts new file mode 100644 index 00000000..fd473160 --- /dev/null +++ b/migrations/scripts/1597906360418-init.ts @@ -0,0 +1,11 @@ +import { QueryInterface } from 'sequelize' +import { Sequelize } from 'sequelize-typescript' + +export default { + // eslint-disable-next-line require-await + async up (queryInterface: QueryInterface, sequelize: Sequelize): Promise { + await sequelize.sync({ force: true }) + }, + // eslint-disable-next-line @typescript-eslint/no-empty-function + async down (): Promise {} +} diff --git a/package.json b/package.json index b4355095..9577f663 100644 --- a/package.json +++ b/package.json @@ -118,8 +118,9 @@ "@types/sinon-chai": "^3.2.4", "@types/sql-formatter": "^2.3.0", "@types/validator": "^13.1.0", - "bignumber.js": "^9.0.0", "@types/umzug": "^2.2.3", + "bignumber.js": "^9.0.0", + "@types/validator": "^13.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "cross-env": "^7.0.2", diff --git a/src/cli/db-migration.ts b/src/cli/db-migration.ts index 5367a617..4e04d963 100644 --- a/src/cli/db-migration.ts +++ b/src/cli/db-migration.ts @@ -90,12 +90,22 @@ export default class DbMigrationCommand extends BaseCLICommand { } async migrate (migrations?: string[], options?: { to: string }): Promise { + if (!(await DbMigration.getInstance().pending()).length) { + this.log('No pending migrations found') + this.exit() + } + this.log('DB migrations') await DbMigration.getInstance().up(options) this.log('Done') } async undo (migrations?: string[], options?: { to: string }): Promise { + if (!(await DbMigration.getInstance().executed()).length) { + this.log('No executed migrations found') + this.exit() + } + this.log('Undo DB migrations') await DbMigration.getInstance().down(options) this.log('Done') From 6e3bdb71b47e1c66e99ff54aca6e155a171c1627 Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Thu, 20 Aug 2020 10:21:46 +0300 Subject: [PATCH 06/17] feat: remove db-sync command --- src/cli/db-migration.ts | 3 ++- src/cli/db-sync.ts | 29 ----------------------------- 2 files changed, 2 insertions(+), 30 deletions(-) delete mode 100644 src/cli/db-sync.ts diff --git a/src/cli/db-migration.ts b/src/cli/db-migration.ts index 4e04d963..89cc500e 100644 --- a/src/cli/db-migration.ts +++ b/src/cli/db-migration.ts @@ -62,7 +62,8 @@ export default class DbMigrationCommand extends BaseCLICommand { '$ rif-marketplace-cache db --up --to 0-test', '$ rif-marketplace-cache --up --migrations 01-test --migrations 02-test', '$ rif-marketplace-cache --up --db ./test.sqlite --to 09-test', - '$ rif-marketplace-cache --down --db ./test.sqlite --to 09-test' + '$ rif-marketplace-cache --down --db ./test.sqlite --to 09-test', + '$ rif-pinning db --generate my_first_migration' ] protected resolveDbPath (db: string): string { diff --git a/src/cli/db-sync.ts b/src/cli/db-sync.ts deleted file mode 100644 index 408a5cfe..00000000 --- a/src/cli/db-sync.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { sequelizeFactory } from '../sequelize' -import { BaseCLICommand } from '../utils' -import { flags } from '@oclif/command' -import config from 'config' - -export default class DBSync extends BaseCLICommand { - static description = 'synchronize database schema' - - static flags = { - db: flags.string({ description: 'database connection URI', env: 'RIFM_DB' }), - force: flags.boolean({ description: 'removes all tables and recreates them' }), - ...BaseCLICommand.flags - } - - async run (): Promise { - const { flags } = this.parse(DBSync) - - if (flags.db) { - config.util.extendDeep(config, { db: flags.db }) - } - - // Init database connection - const sequelize = sequelizeFactory() - this.log('Syncing database') - await sequelize.sync({ force: flags.force }) - this.log('Done') - this.exit(0) - } -} From ff3d5b81cc9955bbf5a28d6547521797d3960e4e Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Thu, 20 Aug 2020 10:35:03 +0300 Subject: [PATCH 07/17] chore: update readme --- README.md | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c4947541..c5ad740c 100644 --- a/README.md +++ b/README.md @@ -380,8 +380,8 @@ $ npm install -g @rsksmart/rif-marketplace-cache // Connection to your database $ export RIFM_DB=postgres://user:pass@localhost/db -// Sync the schema of database -$ rif-marketplace-cache db-sync +// Database migrations +$ rif-marketplace-cache db-migration --up // Connection to your blockchain provider $ export RIFM_PROVIDER=ws://localhost:8545 @@ -400,26 +400,39 @@ For some more details on how to deploy this server please see [Deployment guide] ### Commands -* [`rif-marketplace-cache db-sync`](#rif-marketplace-cache-db-sync) +* [`rif-marketplace-cache db-migration`](#rif-marketplace-cache-db-migration) * [`rif-marketplace-cache precache [SERVICE]`](#rif-marketplace-cache-precache-service) * [`rif-marketplace-cache purge [SERVICE]`](#rif-marketplace-cache-purge-service) * [`rif-marketplace-cache start`](#rif-marketplace-cache-start) -#### `rif-marketplace-cache db-sync` +#### `rif-marketplace-cache db-migration` synchronize database schema ``` USAGE - $ rif-marketplace-cache db-sync + $ rif-marketplace-cache db-migration OPTIONS - --config=config path to JSON config file to load - --db=db database connection URI - --force removes all tables and recreates them - --log=error|warn|info|debug [default: error] what level of information to log - --log-filter=log-filter what components should be logged (+-, chars allowed) - --log-path=log-path log to file, default is STDOUT + -d, --down Undo db migrations + -d, --generate=generate Generate migrations using template [--generate=migration_name] + -m, --migration=migration Migration file + -t, --to=to Migrate to + -u, --up Migrate DB + --config=config path to JSON config file to load + --db=db database connection URI + --log=error|warn|info|verbose|debug [default: warn] what level of information to log + --log-filter=log-filter what components should be logged (+-, chars allowed) + --log-path=log-path log to file, default is STDOUT + +EXAMPLES + $ rif-marketplace-cache db --up + $ rif-marketplace-cache db --down + $ rif-marketplace-cache db --up --to 0-test + $ rif-marketplace-cache --up --migrations 01-test --migrations 02-test + $ rif-marketplace-cache --up --db ./test.sqlite --to 09-test + $ rif-marketplace-cache --down --db ./test.sqlite --to 09-test + $ rif-pinning db --generate my_first_migration ``` #### `rif-marketplace-cache precache [SERVICE]` From 5e3ef37ad4bf8750780df1dad882ddc78058f6b0 Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Thu, 20 Aug 2020 10:35:59 +0300 Subject: [PATCH 08/17] chore: update deployment --- DEPLOYMENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index db08d857..f1f644a3 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -46,7 +46,7 @@ npm install -g @rsksmart/rif-marketplace-cache First synchronize database scheme: ```bash -$ rif-marketplace-cache db-sync --config ./path/to/custom_config +$ rif-marketplace-cache db-migration --up --config ./path/to/custom_config ``` Pre-fetch all previous events: From a8f9b9d47b1951f976b261bdd0dafafeacc3e41b Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Thu, 20 Aug 2020 14:28:33 +0300 Subject: [PATCH 09/17] chore: remove sync from sequelize init add initial db migration --- migrations/scripts/1597906360418-init.ts | 11 - migrations/scripts/1597921613888-initial.ts | 613 ++++++++++++++++++++ src/sequelize.ts | 2 - src/utils.ts | 1 - 4 files changed, 613 insertions(+), 14 deletions(-) delete mode 100644 migrations/scripts/1597906360418-init.ts create mode 100644 migrations/scripts/1597921613888-initial.ts diff --git a/migrations/scripts/1597906360418-init.ts b/migrations/scripts/1597906360418-init.ts deleted file mode 100644 index fd473160..00000000 --- a/migrations/scripts/1597906360418-init.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { QueryInterface } from 'sequelize' -import { Sequelize } from 'sequelize-typescript' - -export default { - // eslint-disable-next-line require-await - async up (queryInterface: QueryInterface, sequelize: Sequelize): Promise { - await sequelize.sync({ force: true }) - }, - // eslint-disable-next-line @typescript-eslint/no-empty-function - async down (): Promise {} -} diff --git a/migrations/scripts/1597921613888-initial.ts b/migrations/scripts/1597921613888-initial.ts new file mode 100644 index 00000000..e7be30f3 --- /dev/null +++ b/migrations/scripts/1597921613888-initial.ts @@ -0,0 +1,613 @@ +import Sequelize, { QueryInterface } from 'sequelize' +import { Sequelize as SequelizeTs } from 'sequelize-typescript' + +/** + * Actions summary: + * + * createTable "event", deps: [] + * createTable "storage_offer", deps: [] + * createTable "rates", deps: [] + * createTable "rns_transfer", deps: [] + * createTable "rns_domain", deps: [] + * createTable "rns_domain_expiration", deps: [rns_domain] + * createTable "rns_sold-domain", deps: [rns_domain] + * createTable "rns_domain-offer", deps: [rns_domain] + * createTable "storage_agreement", deps: [storage_offer] + * createTable "storage_billing-plan", deps: [storage_offer] + * createTable "rns_owner", deps: [rns_domain] + * addIndex "event_transaction_hash_log_index" to table "event" + * + **/ +type Commands = { fn: keyof QueryInterface, [key: string]: any }[] +const migrationCommands = function (transaction: any): Commands { + return [ + { + fn: 'createTable', + params: [ + 'event', + { + id: { + type: Sequelize.INTEGER, + field: 'id', + autoIncrement: true, + primaryKey: true, + allowNull: false + }, + blockNumber: { + type: Sequelize.INTEGER, + field: 'blockNumber' + }, + transactionHash: { + type: Sequelize.STRING(66), + field: 'transactionHash' + }, + logIndex: { + type: Sequelize.INTEGER, + field: 'logIndex' + }, + targetConfirmation: { + type: Sequelize.INTEGER, + field: 'targetConfirmation' + }, + contractAddress: { + type: Sequelize.STRING(66), + field: 'contractAddress' + }, + event: { + type: Sequelize.TEXT, + field: 'event' + }, + content: { + type: Sequelize.TEXT, + field: 'content' + }, + emitted: { + type: Sequelize.BOOLEAN, + field: 'emitted', + defaultValue: false + }, + createdAt: { + type: Sequelize.DATE, + field: 'createdAt', + allowNull: false + }, + updatedAt: { + type: Sequelize.DATE, + field: 'updatedAt', + allowNull: false + } + }, + { + transaction: transaction + } + ] + }, + { + fn: 'createTable', + params: [ + 'storage_offer', + { + provider: { + type: Sequelize.STRING(64), + field: 'provider', + primaryKey: true + }, + totalCapacity: { + type: Sequelize.STRING, + field: 'totalCapacity' + }, + peerId: { + type: Sequelize.STRING, + field: 'peerId' + }, + createdAt: { + type: Sequelize.DATE, + field: 'createdAt', + allowNull: false + }, + updatedAt: { + type: Sequelize.DATE, + field: 'updatedAt', + allowNull: false + } + }, + { + transaction: transaction + } + ] + }, + { + fn: 'createTable', + params: [ + 'rates', + { + token: { + type: Sequelize.STRING(15), + field: 'token', + primaryKey: true + }, + usd: { + type: Sequelize.FLOAT, + field: 'usd' + }, + eur: { + type: Sequelize.FLOAT, + field: 'eur' + }, + btc: { + type: Sequelize.FLOAT, + field: 'btc' + }, + ars: { + type: Sequelize.FLOAT, + field: 'ars' + }, + cny: { + type: Sequelize.FLOAT, + field: 'cny' + }, + krw: { + type: Sequelize.FLOAT, + field: 'krw' + }, + jpy: { + type: Sequelize.FLOAT, + field: 'jpy' + }, + createdAt: { + type: Sequelize.DATE, + field: 'createdAt', + allowNull: false + }, + updatedAt: { + type: Sequelize.DATE, + field: 'updatedAt', + allowNull: false + } + }, + { + transaction: transaction + } + ] + }, + { + fn: 'createTable', + params: [ + 'rns_transfer', + { + id: { + type: Sequelize.STRING, + field: 'id', + primaryKey: true + }, + tokenId: { + type: Sequelize.STRING, + field: 'tokenId', + primaryKey: true + }, + sellerAddress: { + type: Sequelize.STRING, + field: 'sellerAddress' + }, + buyerAddress: { + type: Sequelize.STRING, + field: 'buyerAddress' + } + }, + { + transaction: transaction + } + ] + }, + { + fn: 'createTable', + params: [ + 'rns_domain', + { + tokenId: { + type: Sequelize.STRING, + field: 'tokenId', + primaryKey: true + }, + name: { + type: Sequelize.STRING, + field: 'name' + } + }, + { + transaction: transaction + } + ] + }, + { + fn: 'createTable', + params: [ + 'rns_domain_expiration', + { + tokenId: { + type: Sequelize.STRING, + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + references: { + model: 'rns_domain', + key: 'tokenId' + }, + allowNull: true, + name: 'tokenId', + field: 'tokenId', + primaryKey: true + }, + date: { + type: Sequelize.DATE, + field: 'date' + } + }, + { + transaction: transaction + } + ] + }, + { + fn: 'createTable', + params: [ + 'rns_sold-domain', + { + id: { + type: Sequelize.STRING, + allowNull: true, + name: 'id', + field: 'id', + primaryKey: true + }, + tokenId: { + type: Sequelize.STRING, + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + references: { + model: 'rns_domain', + key: 'tokenId' + }, + allowNull: true, + name: 'tokenId', + field: 'tokenId' + }, + paymentToken: { + type: Sequelize.STRING, + field: 'paymentToken' + }, + price: { + type: Sequelize.DECIMAL, + field: 'price' + }, + priceString: { + type: Sequelize.STRING, + field: 'priceString' + }, + soldDate: { + type: Sequelize.DATE, + field: 'soldDate' + } + }, + { + transaction: transaction + } + ] + }, + { + fn: 'createTable', + params: [ + 'rns_domain-offer', + { + offerId: { + type: Sequelize.STRING, + field: 'offerId', + primaryKey: true + }, + tokenId: { + type: Sequelize.STRING, + onUpdate: 'CASCADE', + onDelete: 'NO ACTION', + references: { + model: 'rns_domain', + key: 'tokenId' + }, + allowNull: true, + name: 'tokenId', + field: 'tokenId' + }, + ownerAddress: { + type: Sequelize.STRING, + field: 'ownerAddress' + }, + ownerDomain: { + type: Sequelize.STRING, + field: 'ownerDomain' + }, + paymentToken: { + type: Sequelize.STRING, + field: 'paymentToken' + }, + price: { + type: Sequelize.DECIMAL, + field: 'price' + }, + priceString: { + type: Sequelize.STRING, + field: 'priceString' + }, + creationDate: { + type: Sequelize.DATE, + field: 'creationDate' + } + }, + { + transaction: transaction + } + ] + }, + { + fn: 'createTable', + params: [ + 'storage_agreement', + { + agreementReference: { + type: Sequelize.STRING(67), + field: 'agreementReference', + primaryKey: true + }, + dataReference: { + type: Sequelize.STRING, + field: 'dataReference' + }, + consumer: { + type: Sequelize.STRING(64), + field: 'consumer' + }, + size: { + type: Sequelize.STRING, + field: 'size' + }, + isActive: { + type: Sequelize.BOOLEAN, + field: 'isActive', + defaultValue: true + }, + billingPeriod: { + type: Sequelize.STRING, + field: 'billingPeriod' + }, + billingPrice: { + type: Sequelize.STRING, + field: 'billingPrice' + }, + availableFunds: { + type: Sequelize.STRING, + field: 'availableFunds' + }, + lastPayout: { + type: Sequelize.DATE, + field: 'lastPayout' + }, + offerId: { + type: Sequelize.STRING, + onUpdate: 'CASCADE', + onDelete: 'NO ACTION', + references: { + model: 'storage_offer', + key: 'provider' + }, + allowNull: true, + name: 'offerId', + field: 'offerId' + } + }, + { + transaction: transaction + } + ] + }, + { + fn: 'createTable', + params: [ + 'storage_billing-plan', + { + id: { + type: Sequelize.INTEGER, + field: 'id', + autoIncrement: true, + primaryKey: true, + allowNull: false + }, + period: { + type: Sequelize.STRING, + field: 'period' + }, + price: { + type: Sequelize.STRING, + field: 'price' + }, + offerId: { + type: Sequelize.STRING, + onUpdate: 'CASCADE', + onDelete: 'NO ACTION', + references: { + model: 'storage_offer', + key: 'provider' + }, + allowNull: true, + name: 'offerId', + field: 'offerId' + }, + createdAt: { + type: Sequelize.DATE, + field: 'createdAt', + allowNull: false + }, + updatedAt: { + type: Sequelize.DATE, + field: 'updatedAt', + allowNull: false + } + }, + { + transaction: transaction + } + ] + }, + { + fn: 'createTable', + params: [ + 'rns_owner', + { + tokenId: { + type: Sequelize.STRING, + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + references: { + model: 'rns_domain', + key: 'tokenId' + }, + allowNull: true, + name: 'tokenId', + field: 'tokenId', + primaryKey: true + }, + address: { + type: Sequelize.STRING, + field: 'address' + } + }, + { + transaction: transaction + } + ] + }, + { + fn: 'addIndex', + params: [ + 'event', + ['transactionHash', 'logIndex'], + { + indexName: 'event_transaction_hash_log_index', + name: 'event_transaction_hash_log_index', + indicesType: 'UNIQUE', + type: 'UNIQUE', + transaction: transaction + } + ] + } + ] +} +const rollbackCommands = function (transaction: any): Commands { + return [ + { + fn: 'dropTable', + params: [ + 'rns_owner', { + transaction: transaction + } + ] + }, + { + fn: 'dropTable', + params: [ + 'event', { + transaction: transaction + } + ] + }, + { + fn: 'dropTable', + params: [ + 'rns_domain-offer', { + transaction: transaction + } + ] + }, + { + fn: 'dropTable', + params: [ + 'rns_domain', { + transaction: transaction + } + ] + }, + { + fn: 'dropTable', + params: [ + 'rns_domain_expiration', { + transaction: transaction + } + ] + }, + { + fn: 'dropTable', + params: [ + 'rates', { + transaction: transaction + } + ] + }, + { + fn: 'dropTable', + params: [ + 'rns_sold-domain', { + transaction: transaction + } + ] + }, + { + fn: 'dropTable', + params: [ + 'rns_transfer', { + transaction: transaction + } + ] + }, + { + fn: 'dropTable', + params: [ + 'storage_agreement', { + transaction: transaction + } + ] + }, + { + fn: 'dropTable', + params: [ + 'storage_billing-plan', { + transaction: transaction + } + ] + }, + { + fn: 'dropTable', + params: [ + 'storage_offer', { + transaction: transaction + } + ] + } + ] +} + +function run (queryInterface: QueryInterface, _commands: (transaction: any) => Commands) { + return async function (transaction: any): Promise { + for (const command of _commands(transaction)) { + // @ts-ignore + // eslint-disable-next-line prefer-spread + await queryInterface[command.fn].apply(queryInterface, command.params) + } + } +} + +export default { + // eslint-disable-next-line require-await + async up (queryInterface: QueryInterface, sequelize: SequelizeTs): Promise { + await queryInterface.sequelize.transaction(run(queryInterface, migrationCommands)) + }, + // eslint-disable-next-line require-await + async down (queryInterface: QueryInterface, sequelize: SequelizeTs): Promise { + await queryInterface.sequelize.transaction(run(queryInterface, rollbackCommands)) + } +} diff --git a/src/sequelize.ts b/src/sequelize.ts index 73b14850..8149271f 100644 --- a/src/sequelize.ts +++ b/src/sequelize.ts @@ -79,8 +79,6 @@ export default async function (app: Application): Promise { (models[name] as any).associate(models) } }) - // Sync to the database - app.set('sequelizeSync', sequelize.sync()) return result } diff --git a/src/utils.ts b/src/utils.ts index 34cb7d35..ce7a84fb 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -133,7 +133,6 @@ export function errorHandler (fn: (...args: any[]) => Promise, logger: Logg */ export async function waitForReadyApp (app: Application): Promise { await app.get('storeInit') - await app.get('sequelizeSync') } /** From 034d6583e5ffb7537c387e64afd05e1d0fb56207 Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Thu, 20 Aug 2020 15:01:33 +0300 Subject: [PATCH 10/17] chore: add db migration check to pre-cache command exit process if migration require in sequelize init --- src/cli/precache.ts | 5 +++++ src/sequelize.ts | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/cli/precache.ts b/src/cli/precache.ts index 73caba40..cae8ac69 100644 --- a/src/cli/precache.ts +++ b/src/cli/precache.ts @@ -5,6 +5,7 @@ import { sequelizeFactory } from '../sequelize' import { BaseCLICommand, capitalizeFirstLetter, validateServices } from '../utils' import { initStore } from '../store' import { SupportedServices } from '../definitions' +import DbMigration from '../../migrations' export default class PreCache extends BaseCLICommand { static get description () { @@ -41,6 +42,10 @@ ${formattedServices}` // Init database connection const sequelize = sequelizeFactory() + if ((await DbMigration.getInstance(sequelize).pending()).length) { + throw new Error('DB Migration required. Please use \'db-migration\' command to proceed') + } + // Init Store await initStore(sequelize) diff --git a/src/sequelize.ts b/src/sequelize.ts index 8149271f..ad2b69d2 100644 --- a/src/sequelize.ts +++ b/src/sequelize.ts @@ -66,7 +66,8 @@ export default async function (app: Application): Promise { app.set('sequelize', sequelize) if ((await DbMigration.getInstance(sequelize).pending()).length) { - throw new Error('DB Migration required. Please use \'db-migration\' command to proceed') + logger.error('DB Migration required. Please use \'db-migration\' command to proceed') + process.exit() } app.setup = function (...args): ReturnType { From 85e724c40160adb8fddebd81677a1ed02d98a37a Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Thu, 20 Aug 2020 17:32:48 +0300 Subject: [PATCH 11/17] chore: move migration folder to src --- src/cli/db-migration.ts | 2 +- src/cli/precache.ts | 2 +- {migrations => src/migrations}/index.ts | 2 +- {migrations => src/migrations}/scripts/1597921613888-initial.ts | 2 -- src/sequelize.ts | 2 +- 5 files changed, 4 insertions(+), 6 deletions(-) rename {migrations => src/migrations}/index.ts (97%) rename {migrations => src/migrations}/scripts/1597921613888-initial.ts (99%) diff --git a/src/cli/db-migration.ts b/src/cli/db-migration.ts index 89cc500e..8deee8ba 100644 --- a/src/cli/db-migration.ts +++ b/src/cli/db-migration.ts @@ -3,7 +3,7 @@ import path from 'path' import { flags } from '@oclif/command' import { OutputFlags } from '@oclif/parser' -import DbMigration from '../../migrations' +import DbMigration from '../migrations' import { BaseCLICommand } from '../utils' import { sequelizeFactory } from '../sequelize' import config from 'config' diff --git a/src/cli/precache.ts b/src/cli/precache.ts index cae8ac69..568d03a6 100644 --- a/src/cli/precache.ts +++ b/src/cli/precache.ts @@ -5,7 +5,7 @@ import { sequelizeFactory } from '../sequelize' import { BaseCLICommand, capitalizeFirstLetter, validateServices } from '../utils' import { initStore } from '../store' import { SupportedServices } from '../definitions' -import DbMigration from '../../migrations' +import DbMigration from '../migrations' export default class PreCache extends BaseCLICommand { static get description () { diff --git a/migrations/index.ts b/src/migrations/index.ts similarity index 97% rename from migrations/index.ts rename to src/migrations/index.ts index 1c6b537b..613d2ad3 100644 --- a/migrations/index.ts +++ b/src/migrations/index.ts @@ -1,7 +1,7 @@ import path from 'path' import Umzug from 'umzug' -import { loggingFactory } from '../src/logger' +import { loggingFactory } from '../logger' import { Sequelize } from 'sequelize-typescript' const logger = loggingFactory('db:migrations') diff --git a/migrations/scripts/1597921613888-initial.ts b/src/migrations/scripts/1597921613888-initial.ts similarity index 99% rename from migrations/scripts/1597921613888-initial.ts rename to src/migrations/scripts/1597921613888-initial.ts index e7be30f3..c99bcb8d 100644 --- a/migrations/scripts/1597921613888-initial.ts +++ b/src/migrations/scripts/1597921613888-initial.ts @@ -602,11 +602,9 @@ function run (queryInterface: QueryInterface, _commands: (transaction: any) => C } export default { - // eslint-disable-next-line require-await async up (queryInterface: QueryInterface, sequelize: SequelizeTs): Promise { await queryInterface.sequelize.transaction(run(queryInterface, migrationCommands)) }, - // eslint-disable-next-line require-await async down (queryInterface: QueryInterface, sequelize: SequelizeTs): Promise { await queryInterface.sequelize.transaction(run(queryInterface, rollbackCommands)) } diff --git a/src/sequelize.ts b/src/sequelize.ts index ad2b69d2..e605aa7e 100644 --- a/src/sequelize.ts +++ b/src/sequelize.ts @@ -7,7 +7,7 @@ import sqlFormatter from 'sql-formatter' import { Application } from './definitions' import { loggingFactory } from './logger' -import DbMigration from '../migrations' +import DbMigration from './migrations' const logger = loggingFactory('db') From 7806dd2df69363db3f76af7fb96875c66c24afd2 Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Fri, 21 Aug 2020 12:21:52 +0300 Subject: [PATCH 12/17] chore: make migration not singleton --- src/cli/db-migration.ts | 12 +++++++----- src/cli/precache.ts | 3 ++- src/migrations/index.ts | 17 +---------------- src/sequelize.ts | 3 ++- 4 files changed, 12 insertions(+), 23 deletions(-) diff --git a/src/cli/db-migration.ts b/src/cli/db-migration.ts index 8deee8ba..58e5d325 100644 --- a/src/cli/db-migration.ts +++ b/src/cli/db-migration.ts @@ -66,6 +66,8 @@ export default class DbMigrationCommand extends BaseCLICommand { '$ rif-pinning db --generate my_first_migration' ] + private migration: DbMigration | undefined + protected resolveDbPath (db: string): string { if (!db) { const dbFromConfig = path.resolve(this.config.dataDir, config.get('db')) @@ -91,24 +93,24 @@ export default class DbMigrationCommand extends BaseCLICommand { } async migrate (migrations?: string[], options?: { to: string }): Promise { - if (!(await DbMigration.getInstance().pending()).length) { + if (!(await this.migration!.pending()).length) { this.log('No pending migrations found') this.exit() } this.log('DB migrations') - await DbMigration.getInstance().up(options) + await this.migration!.up(options) this.log('Done') } async undo (migrations?: string[], options?: { to: string }): Promise { - if (!(await DbMigration.getInstance().executed()).length) { + if (!(await this.migration!.executed()).length) { this.log('No executed migrations found') this.exit() } this.log('Undo DB migrations') - await DbMigration.getInstance().down(options) + await this.migration!.down(options) this.log('Done') } @@ -146,7 +148,7 @@ export default class DbMigrationCommand extends BaseCLICommand { // Init database connection const sequelize = sequelizeFactory() - DbMigration.getInstance(sequelize) + this.migration = new DbMigration(sequelize) if (!parsedFlags.up && !parsedFlags.down && !parsedFlags.generate) { throw new Error('One of \'--generate, --up, --down\' required') diff --git a/src/cli/precache.ts b/src/cli/precache.ts index 568d03a6..21d6c2b6 100644 --- a/src/cli/precache.ts +++ b/src/cli/precache.ts @@ -41,8 +41,9 @@ ${formattedServices}` // Init database connection const sequelize = sequelizeFactory() + const migration = new DbMigration(sequelize) - if ((await DbMigration.getInstance(sequelize).pending()).length) { + if ((await migration.pending()).length) { throw new Error('DB Migration required. Please use \'db-migration\' command to proceed') } diff --git a/src/migrations/index.ts b/src/migrations/index.ts index 613d2ad3..abd1316a 100644 --- a/src/migrations/index.ts +++ b/src/migrations/index.ts @@ -9,7 +9,7 @@ const logger = loggingFactory('db:migrations') type UpOptions = string | string[] | Umzug.UpToOptions | Umzug.UpDownMigrationsOptions type DownOptions = string | string[] | Umzug.DownToOptions | Umzug.UpDownMigrationsOptions -export class Migration { +export default class Migration { private readonly umzugIns: Umzug.Umzug constructor (sequelize: Sequelize) { @@ -54,18 +54,3 @@ export class Migration { return this.umzugIns.executed() } } - -export default class DbMigration { - private static ins: Migration | undefined - - static getInstance (sequelize?: Sequelize): Migration { - if (!DbMigration.ins) { - if (!sequelize) { - throw new Error('You need to provide Sequelize instance') - } - DbMigration.ins = new Migration(sequelize) - } - - return DbMigration.ins - } -} diff --git a/src/sequelize.ts b/src/sequelize.ts index e605aa7e..50735913 100644 --- a/src/sequelize.ts +++ b/src/sequelize.ts @@ -61,11 +61,12 @@ export function BigNumberStringType (propName: string): Partial { const sequelize = sequelizeFactory() + const migration = new DbMigration(sequelize) const oldSetup = app.setup app.set('sequelize', sequelize) - if ((await DbMigration.getInstance(sequelize).pending()).length) { + if ((await migration.pending()).length) { logger.error('DB Migration required. Please use \'db-migration\' command to proceed') process.exit() } From b9dc0a00d26588a001da98e828560b735c405e21 Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Tue, 25 Aug 2020 13:33:57 +0300 Subject: [PATCH 13/17] fix: remove db dialect from config and move it to sequelize factory --- src/cli/db-migration.ts | 5 ++--- src/migrations/index.ts | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/cli/db-migration.ts b/src/cli/db-migration.ts index 58e5d325..0c9e197b 100644 --- a/src/cli/db-migration.ts +++ b/src/cli/db-migration.ts @@ -70,8 +70,7 @@ export default class DbMigrationCommand extends BaseCLICommand { protected resolveDbPath (db: string): string { if (!db) { - const dbFromConfig = path.resolve(this.config.dataDir, config.get('db')) - return dbFromConfig.split(':')[1] + return path.resolve(this.config.dataDir, config.get('db')) } const parsed = path.parse(db) @@ -139,7 +138,7 @@ export default class DbMigrationCommand extends BaseCLICommand { const parsedFlags = originalFlags as OutputFlags if (parsedFlags.db) { - config.util.extendDeep(config, { db: `sqlite:${this.resolveDbPath(parsedFlags.db)}` }) + config.util.extendDeep(config, { db: this.resolveDbPath(parsedFlags.db) }) } if (parsedFlags.generate) { diff --git a/src/migrations/index.ts b/src/migrations/index.ts index abd1316a..d0dd63c2 100644 --- a/src/migrations/index.ts +++ b/src/migrations/index.ts @@ -41,7 +41,7 @@ export default class Migration { // eslint-disable-next-line require-await async pending (): Promise { - return this.umzugIns.pending().catch(e => { + return this.umzugIns.pending().catch((e: any) => { if (e.code === 'ENOENT') { return [] } From 9179a77321ed0f45bc2305fdfcbfa715284293e3 Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Tue, 20 Oct 2020 14:19:08 +0300 Subject: [PATCH 14/17] feat: adjust initial migration file --- package-lock.json | 58 +++ package.json | 1 - .../scripts/1597921613888-initial.ts | 405 ++++++++++++------ 3 files changed, 321 insertions(+), 143 deletions(-) diff --git a/package-lock.json b/package-lock.json index 15c3317c..453a4f17 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3879,6 +3879,15 @@ "@types/node": "*" } }, + "@types/bson": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/bson/-/bson-4.0.3.tgz", + "integrity": "sha512-mVRvYnTOZJz3ccpxhr3wgxVmSeiYinW+zlzQz3SXWaJmD1DuL05Jeq7nKw3SnbKmbleW5qrLG5vdyWe/A9sXhw==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/cbor": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@types/cbor/-/cbor-2.0.0.tgz", @@ -3930,6 +3939,15 @@ "@types/node": "*" } }, + "@types/continuation-local-storage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@types/continuation-local-storage/-/continuation-local-storage-3.2.2.tgz", + "integrity": "sha512-aItm+aYPJ4rT1cHmAxO+OdWjSviQ9iB5UKb5f0Uvgln0N4hS2mcDodHtPiqicYBXViUYhqyBjhA5uyOcT+S34Q==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/cors": { "version": "2.8.7", "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.7.tgz", @@ -4069,6 +4087,16 @@ "integrity": "sha512-vyxR57nv8NfcU0GZu8EUXZLTbCMupIUwy95LJ6lllN+JRPG25CwMHoB1q5xKh8YKhQnHYRAn4yW2yuHbf/5xgg==", "dev": true }, + "@types/mongodb": { + "version": "3.5.29", + "resolved": "https://registry.npmjs.org/@types/mongodb/-/mongodb-3.5.29.tgz", + "integrity": "sha512-aFEMjwSMJ6pstVDwDP4k7cys1iCbAGL01mAp5opOdbK9jv6JmwvYgpGje22Mp2HtrKq5Seea+5ti7CQ/Ovyw2Q==", + "dev": true, + "requires": { + "@types/bson": "*", + "@types/node": "*" + } + }, "@types/node": { "version": "14.11.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.3.tgz", @@ -4129,6 +4157,18 @@ "@types/node": "*" } }, + "@types/sequelize": { + "version": "4.28.9", + "resolved": "https://registry.npmjs.org/@types/sequelize/-/sequelize-4.28.9.tgz", + "integrity": "sha512-QqYgkw/2fEc0FyEQejnxM7cHKB8XBV3Y69k7GSFOToQBOXos0PJVqNpgROXZddXIkl2d6zicYssHuy75ws84sw==", + "dev": true, + "requires": { + "@types/bluebird": "*", + "@types/continuation-local-storage": "*", + "@types/lodash": "*", + "@types/validator": "*" + } + }, "@types/serve-favicon": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/@types/serve-favicon/-/serve-favicon-2.5.0.tgz", @@ -4205,6 +4245,16 @@ "integrity": "sha512-7NQmHra/JILCd1QqpSzl8+mJRc8ZHz3uDm8YV1Ks9IhK0epEiTw8aIErbvH9PI+6XbqhyIQy3462nEsn7UVzjQ==", "dev": true }, + "@types/umzug": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@types/umzug/-/umzug-2.3.0.tgz", + "integrity": "sha512-DUwMKFa+QmApO7I5zhqpc12n9+Mqm006+PTPKrqoZ4F38rk2+iWl8K5IjVvWemERajtkfkiBkHtH/YkOPcBQjA==", + "dev": true, + "requires": { + "@types/mongodb": "*", + "@types/sequelize": "*" + } + }, "@types/unist": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.3.tgz", @@ -26343,6 +26393,14 @@ "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz", "integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==" }, + "umzug": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/umzug/-/umzug-2.3.0.tgz", + "integrity": "sha512-Z274K+e8goZK8QJxmbRPhl89HPO1K+ORFtm6rySPhFKfKc5GHhqdzD0SGhSWHkzoXasqJuItdhorSvY7/Cgflw==", + "requires": { + "bluebird": "^3.7.2" + } + }, "unbzip2-stream": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", diff --git a/package.json b/package.json index 9577f663..e35f0a4b 100644 --- a/package.json +++ b/package.json @@ -120,7 +120,6 @@ "@types/validator": "^13.1.0", "@types/umzug": "^2.2.3", "bignumber.js": "^9.0.0", - "@types/validator": "^13.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "cross-env": "^7.0.2", diff --git a/src/migrations/scripts/1597921613888-initial.ts b/src/migrations/scripts/1597921613888-initial.ts index c99bcb8d..f2af6bd2 100644 --- a/src/migrations/scripts/1597921613888-initial.ts +++ b/src/migrations/scripts/1597921613888-initial.ts @@ -21,6 +21,7 @@ import { Sequelize as SequelizeTs } from 'sequelize-typescript' type Commands = { fn: keyof QueryInterface, [key: string]: any }[] const migrationCommands = function (transaction: any): Commands { return [ + // Event { fn: 'createTable', params: [ @@ -82,6 +83,62 @@ const migrationCommands = function (transaction: any): Commands { } ] }, + // Rate + { + fn: 'createTable', + params: [ + 'rates', + { + token: { + type: Sequelize.STRING(15), + field: 'token', + primaryKey: true + }, + usd: { + type: Sequelize.FLOAT, + field: 'usd' + }, + eur: { + type: Sequelize.FLOAT, + field: 'eur' + }, + btc: { + type: Sequelize.FLOAT, + field: 'btc' + }, + ars: { + type: Sequelize.FLOAT, + field: 'ars' + }, + cny: { + type: Sequelize.FLOAT, + field: 'cny' + }, + krw: { + type: Sequelize.FLOAT, + field: 'krw' + }, + jpy: { + type: Sequelize.FLOAT, + field: 'jpy' + }, + createdAt: { + type: Sequelize.DATE, + field: 'createdAt', + allowNull: false + }, + updatedAt: { + type: Sequelize.DATE, + field: 'updatedAt', + allowNull: false + } + }, + { + transaction: transaction + } + ] + }, + // Storage { fn: 'createTable', params: [ @@ -119,40 +176,118 @@ const migrationCommands = function (transaction: any): Commands { { fn: 'createTable', params: [ - 'rates', + 'storage_agreement', { - token: { - type: Sequelize.STRING(15), - field: 'token', + agreementReference: { + type: Sequelize.STRING(67), + field: 'agreementReference', primaryKey: true }, - usd: { - type: Sequelize.FLOAT, - field: 'usd' + dataReference: { + type: Sequelize.STRING, + field: 'dataReference' }, - eur: { - type: Sequelize.FLOAT, - field: 'eur' + consumer: { + type: Sequelize.STRING(64), + field: 'consumer' }, - btc: { - type: Sequelize.FLOAT, - field: 'btc' + size: { + type: Sequelize.STRING, + field: 'size' }, - ars: { - type: Sequelize.FLOAT, - field: 'ars' + isActive: { + type: Sequelize.BOOLEAN, + field: 'isActive', + defaultValue: true }, - cny: { - type: Sequelize.FLOAT, - field: 'cny' + billingPeriod: { + type: Sequelize.STRING, + field: 'billingPeriod' }, - krw: { - type: Sequelize.FLOAT, - field: 'krw' + billingPrice: { + type: Sequelize.STRING, + field: 'billingPrice' }, - jpy: { - type: Sequelize.FLOAT, - field: 'jpy' + tokenAddress: { + type: Sequelize.STRING, + field: 'tokenAddress' + }, + availableFunds: { + type: Sequelize.STRING, + field: 'availableFunds' + }, + lastPayout: { + type: Sequelize.DATE, + field: 'lastPayout' + }, + offerId: { + type: Sequelize.STRING, + onUpdate: 'CASCADE', + onDelete: 'NO ACTION', + references: { + model: 'storage_offer', + key: 'provider' + }, + allowNull: false, + name: 'offerId', + field: 'offerId' + } + }, + { + transaction: transaction + } + ] + }, + { + fn: 'createTable', + params: [ + 'storage_billing-plan', + { + id: { + type: Sequelize.INTEGER, + field: 'id', + autoIncrement: true, + primaryKey: true, + allowNull: false + }, + period: { + type: Sequelize.STRING, + field: 'period', + allowNull: false + }, + price: { + type: Sequelize.STRING, + field: 'price', + allowNull: false + }, + tokenAddress: { + type: Sequelize.STRING, + field: 'tokenAddress', + allowNull: false + }, + offerId: { + type: Sequelize.STRING, + onUpdate: 'CASCADE', + onDelete: 'NO ACTION', + references: { + model: 'storage_offer', + key: 'provider' + }, + name: 'offerId', + field: 'offerId', + allowNull: false + }, + rateId: { + type: Sequelize.STRING, + onUpdate: 'CASCADE', + onDelete: 'NO ACTION', + references: { + model: 'rates', + key: 'token' + }, + name: 'rateId', + field: 'rateId', + allowNull: false }, createdAt: { type: Sequelize.DATE, @@ -173,17 +308,77 @@ const migrationCommands = function (transaction: any): Commands { { fn: 'createTable', params: [ - 'rns_transfer', + 'storage_stakes', { id: { + type: Sequelize.INTEGER, + field: 'id', + autoIncrement: true, + primaryKey: true, + allowNull: false + }, + total: { + type: Sequelize.STRING, + field: 'total', + allowNull: false + }, + token: { type: Sequelize.STRING, + field: 'token', + allowNull: false + }, + account: { + type: Sequelize.STRING, + field: 'account', + allowNull: false + }, + symbol: { + type: Sequelize.STRING, + onUpdate: 'CASCADE', + onDelete: 'NO ACTION', + references: { + model: 'rates', + key: 'token' + }, + name: 'symbol', + field: 'symbol', + allowNull: false + }, + createdAt: { + type: Sequelize.DATE, + field: 'createdAt', + allowNull: false + }, + updatedAt: { + type: Sequelize.DATE, + field: 'updatedAt', + allowNull: false + } + }, + { + transaction: transaction + } + ] + }, + // RNS + { + fn: 'createTable', + params: [ + 'rns_transfer', + { + id: { + type: Sequelize.INTEGER, field: 'id', - primaryKey: true + primaryKey: true, + autoIncrement: true + }, + txHash: { + type: Sequelize.STRING, + field: 'txHash' }, tokenId: { type: Sequelize.STRING, - field: 'tokenId', - primaryKey: true + field: 'tokenId' }, sellerAddress: { type: Sequelize.STRING, @@ -253,8 +448,9 @@ const migrationCommands = function (transaction: any): Commands { 'rns_sold-domain', { id: { - type: Sequelize.STRING, + type: Sequelize.INTEGER, allowNull: true, + autoIncrement: true, name: 'id', field: 'id', primaryKey: true @@ -271,6 +467,22 @@ const migrationCommands = function (transaction: any): Commands { name: 'tokenId', field: 'tokenId' }, + transferId: { + type: Sequelize.INTEGER, + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + references: { + model: 'rns_transfer', + key: 'id' + }, + allowNull: true, + name: 'transferId', + field: 'transferId' + }, + txHash: { + type: Sequelize.STRING, + field: 'txHash' + }, paymentToken: { type: Sequelize.STRING, field: 'paymentToken' @@ -298,10 +510,15 @@ const migrationCommands = function (transaction: any): Commands { params: [ 'rns_domain-offer', { - offerId: { + id: { + type: Sequelize.INTEGER, + field: 'id', + primaryKey: true, + autoIncrement: true + }, + txHash: { type: Sequelize.STRING, - field: 'offerId', - primaryKey: true + name: 'txHash' }, tokenId: { type: Sequelize.STRING, @@ -335,118 +552,13 @@ const migrationCommands = function (transaction: any): Commands { type: Sequelize.STRING, field: 'priceString' }, - creationDate: { - type: Sequelize.DATE, - field: 'creationDate' - } - }, - { - transaction: transaction - } - ] - }, - { - fn: 'createTable', - params: [ - 'storage_agreement', - { - agreementReference: { - type: Sequelize.STRING(67), - field: 'agreementReference', - primaryKey: true - }, - dataReference: { - type: Sequelize.STRING, - field: 'dataReference' - }, - consumer: { - type: Sequelize.STRING(64), - field: 'consumer' - }, - size: { - type: Sequelize.STRING, - field: 'size' - }, - isActive: { + approved: { type: Sequelize.BOOLEAN, - field: 'isActive', - defaultValue: true - }, - billingPeriod: { - type: Sequelize.STRING, - field: 'billingPeriod' - }, - billingPrice: { - type: Sequelize.STRING, - field: 'billingPrice' - }, - availableFunds: { - type: Sequelize.STRING, - field: 'availableFunds' - }, - lastPayout: { - type: Sequelize.DATE, - field: 'lastPayout' - }, - offerId: { - type: Sequelize.STRING, - onUpdate: 'CASCADE', - onDelete: 'NO ACTION', - references: { - model: 'storage_offer', - key: 'provider' - }, - allowNull: true, - name: 'offerId', - field: 'offerId' - } - }, - { - transaction: transaction - } - ] - }, - { - fn: 'createTable', - params: [ - 'storage_billing-plan', - { - id: { - type: Sequelize.INTEGER, - field: 'id', - autoIncrement: true, - primaryKey: true, - allowNull: false - }, - period: { - type: Sequelize.STRING, - field: 'period' + field: 'approved' }, - price: { - type: Sequelize.STRING, - field: 'price' - }, - offerId: { - type: Sequelize.STRING, - onUpdate: 'CASCADE', - onDelete: 'NO ACTION', - references: { - model: 'storage_offer', - key: 'provider' - }, - allowNull: true, - name: 'offerId', - field: 'offerId' - }, - createdAt: { - type: Sequelize.DATE, - field: 'createdAt', - allowNull: false - }, - updatedAt: { + creationDate: { type: Sequelize.DATE, - field: 'updatedAt', - allowNull: false + field: 'creationDate' } }, { @@ -482,6 +594,7 @@ const migrationCommands = function (transaction: any): Commands { } ] }, + // Indexes { fn: 'addIndex', params: [ @@ -587,6 +700,14 @@ const rollbackCommands = function (transaction: any): Commands { transaction: transaction } ] + }, + { + fn: 'dropTable', + params: [ + 'storage_stakes', { + transaction: transaction + } + ] } ] } From 609f055da09deb594297e0c671019f51f7b073b7 Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Tue, 20 Oct 2020 14:48:44 +0300 Subject: [PATCH 15/17] chore: fix linter --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e35f0a4b..58b6a1ea 100644 --- a/package.json +++ b/package.json @@ -117,8 +117,8 @@ "@types/sinon": "^9.0.3", "@types/sinon-chai": "^3.2.4", "@types/sql-formatter": "^2.3.0", - "@types/validator": "^13.1.0", "@types/umzug": "^2.2.3", + "@types/validator": "^13.1.0", "bignumber.js": "^9.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", From cad3837d31b5e2edb7529d931a83585ea6481134 Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Tue, 20 Oct 2020 14:55:13 +0300 Subject: [PATCH 16/17] test: fix integration test run migration instead of sync --- test/integration/utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/utils.ts b/test/integration/utils.ts index c8cf8ee6..ddb47114 100644 --- a/test/integration/utils.ts +++ b/test/integration/utils.ts @@ -24,6 +24,7 @@ import { initStore } from '../../src/store' import { Application, SupportedServices } from '../../src/definitions' import { ethFactory } from '../../src/blockchain' import { sleep } from '../utils' +import DbMigration from '../../src/migrations' export const ZERO_ADDRESS = '0x0000000000000000000000000000000000000000' export const ZERO_BYTES_32 = '0x0000000000000000000000000000000000000000000000000000000000000000' @@ -124,7 +125,8 @@ export class TestingApp { this.logger.info('Database removed') // Init DB const sequelize = await sequelizeFactory() - await sequelize.sync({ force: true }) + const migration = new DbMigration(sequelize) + await migration.up() await initStore(sequelize) this.logger.info('Database initialized') From 369973dfad1f611b0c849916f020db5df0483bfa Mon Sep 17 00:00:00 2001 From: Nazar Duchak Date: Wed, 11 Nov 2020 20:17:09 +0200 Subject: [PATCH 17/17] feat: update init migration script --- src/migrations/index.ts | 2 +- .../scripts/1597921613888-initial.ts | 57 ++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/src/migrations/index.ts b/src/migrations/index.ts index d0dd63c2..e722a5ba 100644 --- a/src/migrations/index.ts +++ b/src/migrations/index.ts @@ -2,7 +2,7 @@ import path from 'path' import Umzug from 'umzug' import { loggingFactory } from '../logger' -import { Sequelize } from 'sequelize-typescript' +import { Sequelize } from 'sequelize' const logger = loggingFactory('db:migrations') diff --git a/src/migrations/scripts/1597921613888-initial.ts b/src/migrations/scripts/1597921613888-initial.ts index f2af6bd2..b7401a27 100644 --- a/src/migrations/scripts/1597921613888-initial.ts +++ b/src/migrations/scripts/1597921613888-initial.ts @@ -5,6 +5,7 @@ import { Sequelize as SequelizeTs } from 'sequelize-typescript' * Actions summary: * * createTable "event", deps: [] + * createTable "notifications", deps: [] * createTable "storage_offer", deps: [] * createTable "rates", deps: [] * createTable "rns_transfer", deps: [] @@ -83,6 +84,50 @@ const migrationCommands = function (transaction: any): Commands { } ] }, + // Notifications + { + fn: 'createTable', + params: [ + 'notifications', + { + id: { + type: Sequelize.INTEGER, + field: 'id', + autoIncrement: true, + primaryKey: true, + allowNull: false + }, + type: { + type: Sequelize.STRING(), + field: 'type', + allowNull: false + }, + accounts: { + type: Sequelize.STRING, + field: 'accounts', + allowNull: false + }, + payload: { + type: Sequelize.JSON, + field: 'payload', + allowNull: false + }, + createdAt: { + type: Sequelize.DATE, + field: 'createdAt', + allowNull: false + }, + updatedAt: { + type: Sequelize.DATE, + field: 'updatedAt', + allowNull: false + } + }, + { + transaction: transaction + } + ] + }, // Rate { fn: 'createTable', @@ -150,7 +195,7 @@ const migrationCommands = function (transaction: any): Commands { primaryKey: true }, totalCapacity: { - type: Sequelize.STRING, + type: Sequelize.BIGINT, field: 'totalCapacity' }, peerId: { @@ -192,7 +237,7 @@ const migrationCommands = function (transaction: any): Commands { field: 'consumer' }, size: { - type: Sequelize.STRING, + type: Sequelize.BIGINT, field: 'size' }, isActive: { @@ -613,6 +658,14 @@ const migrationCommands = function (transaction: any): Commands { } const rollbackCommands = function (transaction: any): Commands { return [ + { + fn: 'dropTable', + params: [ + 'notifications', { + transaction: transaction + } + ] + }, { fn: 'dropTable', params: [