From c8115fd5e66c0ec98e8173e1657b7ee8b12cb347 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Mon, 9 Mar 2020 20:49:55 +0545 Subject: [PATCH 01/80] Restructure fs util tests --- test/util/fs.test.ts | 65 +++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/test/util/fs.test.ts b/test/util/fs.test.ts index 67b93bc7..046d2771 100644 --- a/test/util/fs.test.ts +++ b/test/util/fs.test.ts @@ -7,6 +7,7 @@ import { write, read, remove, exists, mkdtemp } from '../../src/util/fs'; describe('UTIL: fs', () => { let filePath: string; let invalidFilePath: string; + const fileContent = 'this is random text content'; beforeEach(async () => { @@ -18,49 +19,57 @@ describe('UTIL: fs', () => { await write(filePath, fileContent); }); - it('should write content to the filepath', async () => { - await write(filePath, fileContent); - - const res = await read(filePath); + describe('write', () => { + it('should write content to the filepath', async () => { + await write(filePath, fileContent); - expect(res).to.equal(fileContent); - }); + const res = await read(filePath); - it('should read the content from the filepath', async () => { - const res = await read(filePath); + expect(res).to.equal(fileContent); + }); - expect(res).to.equal(fileContent); + it('should throw error if filepath is invalid (fs.write)', () => { + return expect(write(invalidFilePath, fileContent)).to.eventually.rejected; + }); }); - it('should return true if file exits in given file path', async () => { - const res = await exists(filePath); + describe('read', () => { + it('should read the content from the filepath', async () => { + const res = await read(filePath); + + expect(res).to.equal(fileContent); + }); - expect(res).to.equal(true); + it('should throw error if filepath is invalid (fs.read)', () => { + return expect(read(invalidFilePath)).to.eventually.rejected; + }); }); - it('should return false if file does not exits in given file path', async () => { - const res = await exists('foo/bar'); + describe('exists', () => { + it('should return true if file exits in given file path', async () => { + const res = await exists(filePath); - expect(res).to.equal(false); - }); + expect(res).to.equal(true); + }); - it('should remove the file', async () => { - await remove(filePath); + it('should return false if file does not exits in given file path', async () => { + const res = await exists('foo/bar'); - fs.stat(filePath, err => { - expect(err && err.code).to.be.equal('ENOENT'); + expect(res).to.equal(false); }); }); - it('should throw error if filepath is invalid (fs.write)', () => { - return expect(write(invalidFilePath, fileContent)).to.eventually.rejected; - }); + describe('remove', () => { + it('should remove the file', async () => { + await remove(filePath); - it('should throw error if filepath is invalid (fs.read)', () => { - return expect(read(invalidFilePath)).to.eventually.rejected; - }); + fs.stat(filePath, err => { + expect(err && err.code).to.be.equal('ENOENT'); + }); + }); - it('should throw an error if no such file or directory to remove', () => { - return expect(remove(invalidFilePath)).to.eventually.rejected; + it('should throw an error if no such file or directory to remove', () => { + return expect(remove(invalidFilePath)).to.eventually.rejected; + }); }); }); From 9c22e1f384fad46ad7b940f082cdc372cfa77761 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Mon, 9 Mar 2020 21:20:12 +0545 Subject: [PATCH 02/80] Add tests for glob function --- test/util/fs.test.ts | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/test/util/fs.test.ts b/test/util/fs.test.ts index 046d2771..d7fca021 100644 --- a/test/util/fs.test.ts +++ b/test/util/fs.test.ts @@ -1,8 +1,9 @@ import * as fs from 'fs'; +import * as path from 'path'; import { expect } from 'chai'; import { it, describe } from 'mocha'; -import { write, read, remove, exists, mkdtemp } from '../../src/util/fs'; +import { write, read, remove, exists, mkdtemp, glob } from '../../src/util/fs'; describe('UTIL: fs', () => { let filePath: string; @@ -72,4 +73,29 @@ describe('UTIL: fs', () => { return expect(remove(invalidFilePath)).to.eventually.rejected; }); }); + + describe('glob', async () => { + it('should return the list of all the files under the directory.', async () => { + const tmp1 = await mkdtemp(); + + await Promise.all([ + write(path.join(tmp1, 'file1.txt'), 'Hello World!'), + write(path.join(tmp1, 'file2.txt'), 'Hello World!'), + write(path.join(tmp1, 'file3.txt'), 'Hello World!'), + write(path.join(tmp1, 'file4.txt'), 'Hello World!'), + write(path.join(tmp1, 'file5.txt'), 'Hello World!') + ]); + + const result = await glob(tmp1); + + expect(result).to.deep.equal(['file1.txt', 'file2.txt', 'file3.txt', 'file4.txt', 'file5.txt']); + }); + + it('should return empty array if the directory being globbed is empty.', async () => { + const tmp2 = await mkdtemp(); + const result = await glob(tmp2); + + expect(result).to.deep.equal([]); + }); + }); }); From 111d71929cd4f89f2bed1bd2ebac8a5c2e8a8e66 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Mon, 9 Mar 2020 21:57:44 +0545 Subject: [PATCH 03/80] Add function to get migration entries --- src/services/migrator.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/services/migrator.ts diff --git a/src/services/migrator.ts b/src/services/migrator.ts new file mode 100644 index 00000000..dc93a2bf --- /dev/null +++ b/src/services/migrator.ts @@ -0,0 +1,28 @@ +import { glob } from '../util/fs'; + +const FILE_PATTERN = /(.+)\.(up|down)\.sql$/i; + +/** + * Glob the migration directory and retrieve all the migration entries + * that needs to be run. + * + * Note: The ".up.sql" and ".down.sql" part of the migration files are + * omitted and and a pair of those files are considered a single migration entry. + * + * @param {string} migrationPath + * @returns {Promise} + */ +export async function getSqlMigrationEntries(migrationPath: string): Promise { + const files = await glob(migrationPath); + const migrationSet = new Set(); + + files.forEach(filename => { + const match = filename.match(FILE_PATTERN); + + if (match && match.length === 3) { + migrationSet.add(match[1]); + } + }); + + return Array.from(migrationSet); +} From 12077a476eab970e3fc7b36ed3fd200ccd69863a Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Tue, 10 Mar 2020 00:29:57 +0545 Subject: [PATCH 04/80] Add tests for the migrator service --- test/services/migrator.test.ts | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 test/services/migrator.test.ts diff --git a/test/services/migrator.test.ts b/test/services/migrator.test.ts new file mode 100644 index 00000000..3808d065 --- /dev/null +++ b/test/services/migrator.test.ts @@ -0,0 +1,40 @@ +import * as path from 'path'; +import { describe, it } from 'mocha'; +import { expect } from 'chai'; + +import { write, mkdtemp } from '../../src/util/fs'; +import * as migratorService from '../../src/services/migrator'; + +describe('Services: migrator', () => { + describe('getSqlMigrationEntries', async () => { + it('should return the list of migrations under the directory.', async () => { + const migrationPath = await mkdtemp(); + + // Populate migration files to a directory. + await Promise.all([ + write(path.join(migrationPath, '0001_mgr.up.sql'), 'SELECT 1;'), + write(path.join(migrationPath, '0001_mgr.down.sql'), 'SELECT 1;'), + write(path.join(migrationPath, '0002_mgr.up.sql'), 'SELECT 1;'), + write(path.join(migrationPath, '0002_mgr.down.sql'), 'SELECT 1;'), + write(path.join(migrationPath, '0003_mgr.up.sql'), 'SELECT 1;'), + write(path.join(migrationPath, '0003_mgr.down.sql'), 'SELECT 1;'), + write(path.join(migrationPath, '0004_mgr.up.sql'), 'SELECT 1;'), + write(path.join(migrationPath, '0004_mgr.down.sql'), 'SELECT 1;'), + write(path.join(migrationPath, '0005_mgr.up.sql'), 'SELECT 1;'), + write(path.join(migrationPath, '0005_mgr.down.sql'), 'SELECT 1;') + ]); + + const result = await migratorService.getSqlMigrationEntries(migrationPath); + + // Test the migrations entries retrieved from the directory. + expect(result).to.deep.equal(['0001_mgr', '0002_mgr', '0003_mgr', '0004_mgr', '0005_mgr']); + }); + + it('should return empty array if the migration directory is empty.', async () => { + const migrationPath = await mkdtemp(); + const result = await migratorService.getSqlMigrationEntries(migrationPath); + + expect(result).to.deep.equal([]); + }); + }); +}); From a2dd7097e223d49c3af509cdebed859051591b95 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Tue, 10 Mar 2020 00:34:13 +0545 Subject: [PATCH 05/80] Make things consitent everywhere --- src/api.ts | 1 + src/{services => service}/migrator.ts | 0 test/{services => service}/migrator.test.ts | 4 ++-- 3 files changed, 3 insertions(+), 2 deletions(-) rename src/{services => service}/migrator.ts (100%) rename test/{services => service}/migrator.test.ts (94%) diff --git a/src/api.ts b/src/api.ts index 2ae25901..0f6e8cc4 100644 --- a/src/api.ts +++ b/src/api.ts @@ -7,6 +7,7 @@ import SyncParams from './domain/SyncParams'; import SyncConfig from './domain/SyncConfig'; import SyncResult from './domain/SyncResult'; import { DEFAULT_SYNC_PARAMS } from './constants'; + import ConnectionConfig from './domain/ConnectionConfig'; import ConnectionReference from './domain/ConnectionReference'; import { isKnexInstance, getConfig, createInstance } from './util/db'; diff --git a/src/services/migrator.ts b/src/service/migrator.ts similarity index 100% rename from src/services/migrator.ts rename to src/service/migrator.ts diff --git a/test/services/migrator.test.ts b/test/service/migrator.test.ts similarity index 94% rename from test/services/migrator.test.ts rename to test/service/migrator.test.ts index 3808d065..0887c7b1 100644 --- a/test/services/migrator.test.ts +++ b/test/service/migrator.test.ts @@ -3,9 +3,9 @@ import { describe, it } from 'mocha'; import { expect } from 'chai'; import { write, mkdtemp } from '../../src/util/fs'; -import * as migratorService from '../../src/services/migrator'; +import * as migratorService from '../../src/service/migrator'; -describe('Services: migrator', () => { +describe('SERVICE: migrator', () => { describe('getSqlMigrationEntries', async () => { it('should return the list of migrations under the directory.', async () => { const migrationPath = await mkdtemp(); From f2bb1ef4f94a732851fbcfa2c4a8f32d8d82bf88 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Tue, 10 Mar 2020 01:03:42 +0545 Subject: [PATCH 06/80] Add interface SqlMigrationEntry --- src/domain/SqlMigrationEntry.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/domain/SqlMigrationEntry.ts diff --git a/src/domain/SqlMigrationEntry.ts b/src/domain/SqlMigrationEntry.ts new file mode 100644 index 00000000..e08dc084 --- /dev/null +++ b/src/domain/SqlMigrationEntry.ts @@ -0,0 +1,11 @@ +import SqlCode from './SqlCode'; + +interface SqlMigrationEntry { + name: string; + queries: { + up?: SqlCode; + down?: SqlCode; + }; +} + +export default SqlMigrationEntry; From 523b727e3433a42e2aece12b8589ffe48ace25e1 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Tue, 10 Mar 2020 01:34:50 +0545 Subject: [PATCH 07/80] Implement migrations resolution logic --- src/service/migrator.ts | 39 +++++++++++++++++++++++++++++++---- test/service/migrator.test.ts | 4 ++-- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/service/migrator.ts b/src/service/migrator.ts index dc93a2bf..576fbc4e 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -1,9 +1,13 @@ -import { glob } from '../util/fs'; +import * as path from 'path'; -const FILE_PATTERN = /(.+)\.(up|down)\.sql$/i; +import { glob, exists } from '../util/fs'; +import { resolveFile } from './sqlRunner'; +import SqlMigrationEntry from '../domain/SqlMigrationEntry'; + +const FILE_PATTERN = /(.+)\.(up|down)\.sql$/; /** - * Glob the migration directory and retrieve all the migration entries + * Glob the migration directory and retrieve all the migration entries (names) * that needs to be run. * * Note: The ".up.sql" and ".down.sql" part of the migration files are @@ -12,7 +16,7 @@ const FILE_PATTERN = /(.+)\.(up|down)\.sql$/i; * @param {string} migrationPath * @returns {Promise} */ -export async function getSqlMigrationEntries(migrationPath: string): Promise { +export async function getSqlMigrationNames(migrationPath: string): Promise { const files = await glob(migrationPath); const migrationSet = new Set(); @@ -26,3 +30,30 @@ export async function getSqlMigrationEntries(migrationPath: string): Promise} + */ +export async function resolveSqlMigrations(migrationPath: string): Promise { + const migrationNames = await getSqlMigrationNames(migrationPath); + const migrationPromises = migrationNames.map(async name => { + const upFilename = `${name}.up.sql`; + const downFilename = `${name}.down.sql`; + + const upFileExists = await exists(path.join(migrationPath, upFilename)); + const downFileExists = await exists(path.join(migrationPath, downFilename)); + + const up = upFileExists ? await resolveFile(migrationPath, upFilename) : undefined; + const down = downFileExists ? await resolveFile(migrationPath, downFilename) : undefined; + + return { + name, + queries: { up, down } + }; + }); + + return Promise.all(migrationPromises); +} diff --git a/test/service/migrator.test.ts b/test/service/migrator.test.ts index 0887c7b1..8a360f43 100644 --- a/test/service/migrator.test.ts +++ b/test/service/migrator.test.ts @@ -24,7 +24,7 @@ describe('SERVICE: migrator', () => { write(path.join(migrationPath, '0005_mgr.down.sql'), 'SELECT 1;') ]); - const result = await migratorService.getSqlMigrationEntries(migrationPath); + const result = await migratorService.getSqlMigrationNames(migrationPath); // Test the migrations entries retrieved from the directory. expect(result).to.deep.equal(['0001_mgr', '0002_mgr', '0003_mgr', '0004_mgr', '0005_mgr']); @@ -32,7 +32,7 @@ describe('SERVICE: migrator', () => { it('should return empty array if the migration directory is empty.', async () => { const migrationPath = await mkdtemp(); - const result = await migratorService.getSqlMigrationEntries(migrationPath); + const result = await migratorService.getSqlMigrationNames(migrationPath); expect(result).to.deep.equal([]); }); From c507de162592391a8d540a1a90f2c26490f48cd0 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Tue, 10 Mar 2020 01:36:30 +0545 Subject: [PATCH 08/80] Implement KnexMigrationSource class --- src/KnexMigrationSource.ts | 70 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 src/KnexMigrationSource.ts diff --git a/src/KnexMigrationSource.ts b/src/KnexMigrationSource.ts new file mode 100644 index 00000000..6244ee07 --- /dev/null +++ b/src/KnexMigrationSource.ts @@ -0,0 +1,70 @@ +import * as Knex from 'knex'; + +import { log } from '../src/logger'; +import * as migratorService from '../src/service/migrator'; +import SqlMigrationEntry from './domain/SqlMigrationEntry'; + +/** + * MigrationSource class for the Knex Migration API. + */ +class KnexMigrationSource { + /** + * Migration directory path. + * @type {string} + */ + private migrationPath: string; + + /** + * Creates an instance of KnexMigrationSource. + * + * @param {MigrationEntry[]} migrationLists + */ + constructor(migrationPath: string) { + log('Creating new KnexMigrationSource with path:', migrationPath); + + this.migrationPath = migrationPath; + } + + /** + * Gets a list of migration entries. + * + * + * @returns {Promise} + */ + async getMigrations(): Promise { + const migrations = await migratorService.resolveSqlMigrations(this.migrationPath); + + log('Resolved migrations', migrations); + + return migrations; + } + + /** + * Get the name of the migration entry. + * + * @param {SqlMigrationEntry} migration + * @returns {string} + * @memberof KnexMigrationSource + */ + getMigrationName(migration: SqlMigrationEntry): string { + log(`Returning migration name: ${migration.name}`); + + return migration.name; + } + + /** + * Get the migration functions. + * + * @param {SqlMigrationEntry} migration + */ + getMigration(migration: SqlMigrationEntry) { + const { queries } = migration; + + return { + up: (db: Knex) => (queries.up ? db.raw(queries.up.sql) : Promise.resolve()), + down: (db: Knex) => (queries.down ? db.raw(queries.down.sql) : Promise.resolve()) + }; + } +} + +export default KnexMigrationSource; From a49dd2a2b09e78351b373efd81378aebaf9e7e31 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Tue, 10 Mar 2020 01:58:24 +0545 Subject: [PATCH 09/80] Write tests for the migration resolution logic --- test/service/migrator.test.ts | 81 ++++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 10 deletions(-) diff --git a/test/service/migrator.test.ts b/test/service/migrator.test.ts index 8a360f43..5f9f1a59 100644 --- a/test/service/migrator.test.ts +++ b/test/service/migrator.test.ts @@ -12,16 +12,16 @@ describe('SERVICE: migrator', () => { // Populate migration files to a directory. await Promise.all([ - write(path.join(migrationPath, '0001_mgr.up.sql'), 'SELECT 1;'), - write(path.join(migrationPath, '0001_mgr.down.sql'), 'SELECT 1;'), - write(path.join(migrationPath, '0002_mgr.up.sql'), 'SELECT 1;'), - write(path.join(migrationPath, '0002_mgr.down.sql'), 'SELECT 1;'), - write(path.join(migrationPath, '0003_mgr.up.sql'), 'SELECT 1;'), - write(path.join(migrationPath, '0003_mgr.down.sql'), 'SELECT 1;'), - write(path.join(migrationPath, '0004_mgr.up.sql'), 'SELECT 1;'), - write(path.join(migrationPath, '0004_mgr.down.sql'), 'SELECT 1;'), - write(path.join(migrationPath, '0005_mgr.up.sql'), 'SELECT 1;'), - write(path.join(migrationPath, '0005_mgr.down.sql'), 'SELECT 1;') + write(path.join(migrationPath, '0001_mgr.up.sql'), 'SELECT 1'), + write(path.join(migrationPath, '0001_mgr.down.sql'), 'SELECT 1'), + write(path.join(migrationPath, '0002_mgr.up.sql'), 'SELECT 1'), + write(path.join(migrationPath, '0002_mgr.down.sql'), 'SELECT 1'), + write(path.join(migrationPath, '0003_mgr.up.sql'), 'SELECT 1'), + write(path.join(migrationPath, '0003_mgr.down.sql'), 'SELECT 1'), + write(path.join(migrationPath, '0004_mgr.up.sql'), 'SELECT 1'), + write(path.join(migrationPath, '0004_mgr.down.sql'), 'SELECT 1'), + write(path.join(migrationPath, '0005_mgr.up.sql'), 'SELECT 1'), + write(path.join(migrationPath, '0005_mgr.down.sql'), 'SELECT 1') ]); const result = await migratorService.getSqlMigrationNames(migrationPath); @@ -37,4 +37,65 @@ describe('SERVICE: migrator', () => { expect(result).to.deep.equal([]); }); }); + + describe('resolveSqlMigrations', () => { + it('should resolve all the information related to the migration entries.', async () => { + const migrationPath = await mkdtemp(); + + // Populate migration files to a directory. + await Promise.all([ + write(path.join(migrationPath, '0001_mgr.up.sql'), 'CREATE TABLE test_mgr1'), + write(path.join(migrationPath, '0001_mgr.down.sql'), 'DROP TABLE test_mgr1'), + write(path.join(migrationPath, '0002_mgr.up.sql'), 'CREATE TABLE test_mgr2'), + write(path.join(migrationPath, '0002_mgr.down.sql'), 'DROP TABLE test_mgr2'), + write(path.join(migrationPath, '0003_mgr.up.sql'), 'CREATE TABLE test_mgr3'), + write(path.join(migrationPath, '0003_mgr.down.sql'), 'DROP TABLE test_mgr3'), + write(path.join(migrationPath, '0004_mgr.up.sql'), 'CREATE TABLE test_mgr4'), + write(path.join(migrationPath, '0004_mgr.down.sql'), 'DROP TABLE test_mgr4'), + write(path.join(migrationPath, '0005_mgr.up.sql'), 'CREATE TABLE test_mgr5'), + write(path.join(migrationPath, '0005_mgr.down.sql'), 'DROP TABLE test_mgr5') + ]); + + const result = await migratorService.resolveSqlMigrations(migrationPath); + + // Test the migrations entries retrieved from the directory. + expect(result).to.deep.equal([ + { + name: '0001_mgr', + queries: { + up: { name: '0001_mgr.up.sql', sql: 'CREATE TABLE test_mgr1' }, + down: { name: '0001_mgr.down.sql', sql: 'DROP TABLE test_mgr1' } + } + }, + { + name: '0002_mgr', + queries: { + up: { name: '0002_mgr.up.sql', sql: 'CREATE TABLE test_mgr2' }, + down: { name: '0002_mgr.down.sql', sql: 'DROP TABLE test_mgr2' } + } + }, + { + name: '0003_mgr', + queries: { + up: { name: '0003_mgr.up.sql', sql: 'CREATE TABLE test_mgr3' }, + down: { name: '0003_mgr.down.sql', sql: 'DROP TABLE test_mgr3' } + } + }, + { + name: '0004_mgr', + queries: { + up: { name: '0004_mgr.up.sql', sql: 'CREATE TABLE test_mgr4' }, + down: { name: '0004_mgr.down.sql', sql: 'DROP TABLE test_mgr4' } + } + }, + { + name: '0005_mgr', + queries: { + up: { name: '0005_mgr.up.sql', sql: 'CREATE TABLE test_mgr5' }, + down: { name: '0005_mgr.down.sql', sql: 'DROP TABLE test_mgr5' } + } + } + ]); + }); + }); }); From 07f41f541783837b042d421fb46e1790f2749669 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Tue, 10 Mar 2020 02:11:42 +0545 Subject: [PATCH 10/80] Move empty array test one level up --- test/service/migrator.test.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/service/migrator.test.ts b/test/service/migrator.test.ts index 5f9f1a59..58801902 100644 --- a/test/service/migrator.test.ts +++ b/test/service/migrator.test.ts @@ -29,13 +29,6 @@ describe('SERVICE: migrator', () => { // Test the migrations entries retrieved from the directory. expect(result).to.deep.equal(['0001_mgr', '0002_mgr', '0003_mgr', '0004_mgr', '0005_mgr']); }); - - it('should return empty array if the migration directory is empty.', async () => { - const migrationPath = await mkdtemp(); - const result = await migratorService.getSqlMigrationNames(migrationPath); - - expect(result).to.deep.equal([]); - }); }); describe('resolveSqlMigrations', () => { @@ -97,5 +90,12 @@ describe('SERVICE: migrator', () => { } ]); }); + + it('should return empty array if the migration directory is empty.', async () => { + const migrationPath = await mkdtemp(); + const result = await migratorService.resolveSqlMigrations(migrationPath); + + expect(result).to.deep.equal([]); + }); }); }); From 5ea3af7f325be447e6a1251dfe50a08e3aa1d9df Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Tue, 10 Mar 2020 02:26:58 +0545 Subject: [PATCH 11/80] Test KnexMigrationSource.getMigrations() --- test/KnexMigrationSource.test.ts | 83 ++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 test/KnexMigrationSource.test.ts diff --git a/test/KnexMigrationSource.test.ts b/test/KnexMigrationSource.test.ts new file mode 100644 index 00000000..7b2aadac --- /dev/null +++ b/test/KnexMigrationSource.test.ts @@ -0,0 +1,83 @@ +import * as path from 'path'; +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import { write, mkdtemp } from '../src/util/fs'; +import KnexMigrationSource from '../src/KnexMigrationSource'; + +describe('UTIL: KnexMigrationSource', () => { + const getInstance = async () => { + const migrationPath = await mkdtemp(); + const instance = new KnexMigrationSource(migrationPath); + + return { migrationPath, instance }; + }; + + describe('getMigrations', () => { + it('should return empty array if the migration directory is empty.', async () => { + const { instance } = await getInstance(); + const result = await instance.getMigrations(); + + expect(result).to.deep.equal([]); + }); + + it('should resolve all the information related to the migration entries.', async () => { + const { migrationPath, instance } = await getInstance(); + + // Populate migration files to a directory. + await Promise.all([ + write(path.join(migrationPath, '0001_mgr.up.sql'), 'CREATE TABLE test_mgr1'), + write(path.join(migrationPath, '0001_mgr.down.sql'), 'DROP TABLE test_mgr1'), + write(path.join(migrationPath, '0002_mgr.up.sql'), 'CREATE TABLE test_mgr2'), + write(path.join(migrationPath, '0002_mgr.down.sql'), 'DROP TABLE test_mgr2'), + write(path.join(migrationPath, '0003_mgr.up.sql'), 'CREATE TABLE test_mgr3'), + write(path.join(migrationPath, '0003_mgr.down.sql'), 'DROP TABLE test_mgr3'), + write(path.join(migrationPath, '0004_mgr.up.sql'), 'CREATE TABLE test_mgr4'), + write(path.join(migrationPath, '0004_mgr.down.sql'), 'DROP TABLE test_mgr4'), + write(path.join(migrationPath, '0005_mgr.up.sql'), 'CREATE TABLE test_mgr5'), + write(path.join(migrationPath, '0005_mgr.down.sql'), 'DROP TABLE test_mgr5') + ]); + + const result = await instance.getMigrations(); + + // Test the migrations entries retrieved from the directory. + expect(result).to.deep.equal([ + { + name: '0001_mgr', + queries: { + up: { name: '0001_mgr.up.sql', sql: 'CREATE TABLE test_mgr1' }, + down: { name: '0001_mgr.down.sql', sql: 'DROP TABLE test_mgr1' } + } + }, + { + name: '0002_mgr', + queries: { + up: { name: '0002_mgr.up.sql', sql: 'CREATE TABLE test_mgr2' }, + down: { name: '0002_mgr.down.sql', sql: 'DROP TABLE test_mgr2' } + } + }, + { + name: '0003_mgr', + queries: { + up: { name: '0003_mgr.up.sql', sql: 'CREATE TABLE test_mgr3' }, + down: { name: '0003_mgr.down.sql', sql: 'DROP TABLE test_mgr3' } + } + }, + { + name: '0004_mgr', + queries: { + up: { name: '0004_mgr.up.sql', sql: 'CREATE TABLE test_mgr4' }, + down: { name: '0004_mgr.down.sql', sql: 'DROP TABLE test_mgr4' } + } + }, + { + name: '0005_mgr', + queries: { + up: { name: '0005_mgr.up.sql', sql: 'CREATE TABLE test_mgr5' }, + down: { name: '0005_mgr.down.sql', sql: 'DROP TABLE test_mgr5' } + } + } + ]); + }); + }); +}); From 2311aa9ca3eefe9ed074c59b9fe5f8d7d1222792 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Tue, 10 Mar 2020 02:27:33 +0545 Subject: [PATCH 12/80] Test KnexMigrationSource.getMigrationName --- test/KnexMigrationSource.test.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/KnexMigrationSource.test.ts b/test/KnexMigrationSource.test.ts index 7b2aadac..b4ddce65 100644 --- a/test/KnexMigrationSource.test.ts +++ b/test/KnexMigrationSource.test.ts @@ -80,4 +80,20 @@ describe('UTIL: KnexMigrationSource', () => { ]); }); }); + + describe('getMigrationName', async () => { + it('should return the name of the migration.', async () => { + const { instance } = await getInstance(); + + expect( + instance.getMigrationName({ + name: '0005_mgr', + queries: { + up: { name: '0005_mgr.up.sql', sql: 'CREATE TABLE test_mgr5' }, + down: { name: '0005_mgr.down.sql', sql: 'DROP TABLE test_mgr5' } + } + }) + ).to.equal('0005_mgr'); + }); + }); }); From 5ec99d7d5122c95f07ba1f7f4ba685f6e77ccaf5 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Tue, 10 Mar 2020 02:48:49 +0545 Subject: [PATCH 13/80] Add more tests for KnexMigrationSource --- test/KnexMigrationSource.test.ts | 72 ++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/test/KnexMigrationSource.test.ts b/test/KnexMigrationSource.test.ts index b4ddce65..1e506bca 100644 --- a/test/KnexMigrationSource.test.ts +++ b/test/KnexMigrationSource.test.ts @@ -84,16 +84,70 @@ describe('UTIL: KnexMigrationSource', () => { describe('getMigrationName', async () => { it('should return the name of the migration.', async () => { const { instance } = await getInstance(); + const entry = { + name: '0005_mgr', + queries: { + up: { name: '0005_mgr.up.sql', sql: 'CREATE TABLE test_mgr5' }, + down: { name: '0005_mgr.down.sql', sql: 'DROP TABLE test_mgr5' } + } + }; - expect( - instance.getMigrationName({ - name: '0005_mgr', - queries: { - up: { name: '0005_mgr.up.sql', sql: 'CREATE TABLE test_mgr5' }, - down: { name: '0005_mgr.down.sql', sql: 'DROP TABLE test_mgr5' } - } - }) - ).to.equal('0005_mgr'); + expect(instance.getMigrationName(entry)).to.equal('0005_mgr'); + }); + }); + + describe('getMigration', async () => { + const dbStub = { + raw: (sql: string) => Promise.resolve(`Result of "${sql}"`) + }; + + it('should return the migration functions.', async () => { + const { instance } = await getInstance(); + const entry = { + name: '0005_mgr', + queries: { + up: { name: '0005_mgr.up.sql', sql: 'CREATE TABLE test_mgr5' }, + down: { name: '0005_mgr.down.sql', sql: 'DROP TABLE test_mgr5' } + } + }; + + const migration = instance.getMigration(entry); + + const upResult = await migration.up(dbStub as any); + const downResult = await migration.down(dbStub as any); + + expect(upResult).to.equal('Result of "CREATE TABLE test_mgr5"'); + expect(downResult).to.equal('Result of "DROP TABLE test_mgr5"'); + }); + + it('could return empty promises for the migration functions if the files have been unresolved.', async () => { + const { instance } = await getInstance(); + const entry1 = { + name: '0004_mgr', + queries: { + up: { name: '0004_mgr.up.sql', sql: 'UPDATE test_mgr4 SET is_active = 0' } + } + }; + const migration1 = instance.getMigration(entry1); + const upResult1 = await migration1.up(dbStub as any); + const downResult1 = await migration1.down(dbStub as any); + + expect(upResult1).to.equal('Result of "UPDATE test_mgr4 SET is_active = 0"'); + expect(downResult1).to.equal(undefined); + + const entry2 = { + name: '0005_mgr', + queries: { + down: { name: '0005_mgr.down.sql', sql: 'UPDATE test_mgr5 SET is_disabled = 1' } + } + }; + + const migration2 = instance.getMigration(entry2); + const upResult2 = await migration2.up(dbStub as any); + const downResult2 = await migration2.down(dbStub as any); + + expect(upResult2).to.equal(undefined); + expect(downResult2).to.equal('Result of "UPDATE test_mgr5 SET is_disabled = 1"'); }); }); }); From 3e9a8d079bed876446ab48617b0fb2015df08c1b Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Tue, 10 Mar 2020 02:57:05 +0545 Subject: [PATCH 14/80] Update tests --- test/service/migrator.test.ts | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/test/service/migrator.test.ts b/test/service/migrator.test.ts index 58801902..1d068e36 100644 --- a/test/service/migrator.test.ts +++ b/test/service/migrator.test.ts @@ -32,21 +32,15 @@ describe('SERVICE: migrator', () => { }); describe('resolveSqlMigrations', () => { - it('should resolve all the information related to the migration entries.', async () => { + it('should resolve all the information related to the available migration entries.', async () => { const migrationPath = await mkdtemp(); // Populate migration files to a directory. await Promise.all([ write(path.join(migrationPath, '0001_mgr.up.sql'), 'CREATE TABLE test_mgr1'), - write(path.join(migrationPath, '0001_mgr.down.sql'), 'DROP TABLE test_mgr1'), - write(path.join(migrationPath, '0002_mgr.up.sql'), 'CREATE TABLE test_mgr2'), write(path.join(migrationPath, '0002_mgr.down.sql'), 'DROP TABLE test_mgr2'), write(path.join(migrationPath, '0003_mgr.up.sql'), 'CREATE TABLE test_mgr3'), - write(path.join(migrationPath, '0003_mgr.down.sql'), 'DROP TABLE test_mgr3'), - write(path.join(migrationPath, '0004_mgr.up.sql'), 'CREATE TABLE test_mgr4'), - write(path.join(migrationPath, '0004_mgr.down.sql'), 'DROP TABLE test_mgr4'), - write(path.join(migrationPath, '0005_mgr.up.sql'), 'CREATE TABLE test_mgr5'), - write(path.join(migrationPath, '0005_mgr.down.sql'), 'DROP TABLE test_mgr5') + write(path.join(migrationPath, '0003_mgr.down.sql'), 'DROP TABLE test_mgr3') ]); const result = await migratorService.resolveSqlMigrations(migrationPath); @@ -57,13 +51,13 @@ describe('SERVICE: migrator', () => { name: '0001_mgr', queries: { up: { name: '0001_mgr.up.sql', sql: 'CREATE TABLE test_mgr1' }, - down: { name: '0001_mgr.down.sql', sql: 'DROP TABLE test_mgr1' } + down: undefined } }, { name: '0002_mgr', queries: { - up: { name: '0002_mgr.up.sql', sql: 'CREATE TABLE test_mgr2' }, + up: undefined, down: { name: '0002_mgr.down.sql', sql: 'DROP TABLE test_mgr2' } } }, @@ -73,20 +67,6 @@ describe('SERVICE: migrator', () => { up: { name: '0003_mgr.up.sql', sql: 'CREATE TABLE test_mgr3' }, down: { name: '0003_mgr.down.sql', sql: 'DROP TABLE test_mgr3' } } - }, - { - name: '0004_mgr', - queries: { - up: { name: '0004_mgr.up.sql', sql: 'CREATE TABLE test_mgr4' }, - down: { name: '0004_mgr.down.sql', sql: 'DROP TABLE test_mgr4' } - } - }, - { - name: '0005_mgr', - queries: { - up: { name: '0005_mgr.up.sql', sql: 'CREATE TABLE test_mgr5' }, - down: { name: '0005_mgr.down.sql', sql: 'DROP TABLE test_mgr5' } - } } ]); }); From 22c1967d1a1de365089bd0abbd8e38e11784c1c0 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Tue, 10 Mar 2020 03:06:15 +0545 Subject: [PATCH 15/80] Add more tests for the migrator --- test/service/migrator.test.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test/service/migrator.test.ts b/test/service/migrator.test.ts index 1d068e36..d500b721 100644 --- a/test/service/migrator.test.ts +++ b/test/service/migrator.test.ts @@ -7,7 +7,7 @@ import * as migratorService from '../../src/service/migrator'; describe('SERVICE: migrator', () => { describe('getSqlMigrationEntries', async () => { - it('should return the list of migrations under the directory.', async () => { + it('should return the list of valid migrations under the directory.', async () => { const migrationPath = await mkdtemp(); // Populate migration files to a directory. @@ -29,6 +29,24 @@ describe('SERVICE: migrator', () => { // Test the migrations entries retrieved from the directory. expect(result).to.deep.equal(['0001_mgr', '0002_mgr', '0003_mgr', '0004_mgr', '0005_mgr']); }); + + it('should not return other files under the directory that are not migrations.', async () => { + const migrationPath = await mkdtemp(); + + // Populate migration files to a directory. + await Promise.all([ + write(path.join(migrationPath, '0001_mgr.up.sql'), 'SELECT 1'), + write(path.join(migrationPath, '0002_mgr.down.sql'), 'SELECT 1'), + write(path.join(migrationPath, 'test.sql'), 'SELECT 2'), + write(path.join(migrationPath, 'migrate.sql'), 'SELECT 3'), + write(path.join(migrationPath, '.gitignore'), '') + ]); + + const result = await migratorService.getSqlMigrationNames(migrationPath); + + // Test the migrations entries retrieved from the directory. + expect(result).to.deep.equal(['0001_mgr', '0002_mgr']); + }); }); describe('resolveSqlMigrations', () => { From a51b06f4e899591114b53b8d6bbd421ef5ab4a24 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Tue, 17 Mar 2020 23:53:08 +0545 Subject: [PATCH 16/80] Temporary workaroun with cli options --- src/api.ts | 2 ++ src/cli.ts | 22 +++++++++++++++++++++- src/commands/synchronize.ts | 10 +++++++++- src/domain/SyncDbOptions.ts | 3 +++ 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/api.ts b/src/api.ts index 0f6e8cc4..24b2bbcd 100644 --- a/src/api.ts +++ b/src/api.ts @@ -72,3 +72,5 @@ function mapToConnectionReferences(connectionList: (ConnectionConfig | Knex)[]): return { connection: createInstance(connection), id: getConnectionId(connection) }; }); } + +function migrate diff --git a/src/cli.ts b/src/cli.ts index 6277fcbb..26e74762 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -22,6 +22,18 @@ async function generateConnection(): Promise { await printLine(`Generated file: ${CONNECTIONS_FILENAME}\n`); } +async function migrate(): Promise { + printLine('Migrate.'); +} + +async function rollback(): Promise { + printLine('Rollback.'); +} + +async function list(): Promise { + printLine('List'); +} + /** * Handle the provided CLI flags. * @@ -31,7 +43,15 @@ async function generateConnection(): Promise { export async function handleFlags(flags: SyncDbOptions): Promise { if (flags['generate-connections']) { await generateConnection(); - + process.exit(0); + } else if (flags['migrate']) { + await migrate(); + process.exit(0); + } else if (flags['rollback']) { + await rollback(); + process.exit(0); + } else if (flags['list']) { + await list(); process.exit(0); } } diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index 83c20478..04751795 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -22,7 +22,13 @@ class Synchronize extends Command { version: flags.version({ char: 'v', description: 'Print version', name: 'sync-db' }), help: flags.help({ char: 'h', description: 'Print help information' }), force: flags.boolean({ char: 'f', description: 'Force synchronization' }), - 'generate-connections': flags.boolean({ char: 'c', description: 'Generate connections' }) + 'generate-connections': flags.boolean({ char: 'c', description: 'Generate connections' }), + + // Note: These are temporary workaround using args. + // TODO: Move these into separate commands (multi-commands) + migrate: flags.boolean({ description: 'Run Migrations' }), + rollback: flags.boolean({ description: 'Run Rollback' }), + list: flags.boolean({ description: 'List migrations' }) }; /** @@ -53,6 +59,8 @@ class Synchronize extends Command { const { flags: parsedFlags } = this.parse(Synchronize); const params = this.getSyncParams({ ...parsedFlags }); + console.log({ parsedFlags }); // tslint:disable-line + try { await handleFlags(parsedFlags); diff --git a/src/domain/SyncDbOptions.ts b/src/domain/SyncDbOptions.ts index 4104dbf6..de8484a3 100644 --- a/src/domain/SyncDbOptions.ts +++ b/src/domain/SyncDbOptions.ts @@ -6,6 +6,9 @@ interface SyncDbOptions { help: void; force: boolean; 'generate-connections': boolean; + migrate: boolean; + rollback: boolean; + list: boolean; } export default SyncDbOptions; From fa9a0497a63683b7c656f78d71e3554d612be2c3 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Wed, 18 Mar 2020 00:05:34 +0545 Subject: [PATCH 17/80] Simplify isCLI dependency --- bin/run | 5 +---- src/api.ts | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/bin/run b/bin/run index a8716196..b0d7ba62 100755 --- a/bin/run +++ b/bin/run @@ -3,7 +3,4 @@ // Set CLI environment as true. process.env.SYNC_DB_CLI = 'true'; -require('@oclif/command') - .run() - .then(require('@oclif/command/flush')) - .catch(require('@oclif/errors/handle')); +require('@oclif/command').run().then(require('@oclif/command/flush')).catch(require('@oclif/errors/handle')); diff --git a/src/api.ts b/src/api.ts index 24b2bbcd..0f6e8cc4 100644 --- a/src/api.ts +++ b/src/api.ts @@ -72,5 +72,3 @@ function mapToConnectionReferences(connectionList: (ConnectionConfig | Knex)[]): return { connection: createInstance(connection), id: getConnectionId(connection) }; }); } - -function migrate From 7261b7d072ad28b0ef717a41cd4f8b37cd351262 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Wed, 18 Mar 2020 00:16:23 +0545 Subject: [PATCH 18/80] Add migration config to SyncConfig --- src/constants.ts | 4 ++++ src/domain/SyncConfig.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/constants.ts b/src/constants.ts index 3c3ae389..33d36991 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -24,6 +24,10 @@ export const DEFAULT_CONFIG: SyncConfig = { }, injectedConfig: { vars: {} + }, + migration: { + directory: 'migration', + tableName: 'knex_migrations' // Note: This is Knex's default value. Just keeping it same. } }; diff --git a/src/domain/SyncConfig.ts b/src/domain/SyncConfig.ts index 1e41219a..b951adad 100644 --- a/src/domain/SyncConfig.ts +++ b/src/domain/SyncConfig.ts @@ -14,6 +14,10 @@ interface SyncConfig { injectedConfig: { vars: Mapping; }; + migration: { + directory: string; + tableName: string; + }; } export default SyncConfig; From 4d6f87349be0ee1de8090cb2068f7cc3677023a0 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Wed, 18 Mar 2020 00:28:28 +0545 Subject: [PATCH 19/80] Add migration service functions --- src/service/migrator.ts | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/service/migrator.ts b/src/service/migrator.ts index 576fbc4e..4735176d 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -1,7 +1,10 @@ +import Knex from 'knex'; import * as path from 'path'; import { glob, exists } from '../util/fs'; import { resolveFile } from './sqlRunner'; +import SyncConfig from '../domain/SyncConfig'; +import KnexMigrationSource from '../KnexMigrationSource'; import SqlMigrationEntry from '../domain/SqlMigrationEntry'; const FILE_PATTERN = /(.+)\.(up|down)\.sql$/; @@ -57,3 +60,53 @@ export async function resolveSqlMigrations(migrationPath: string): Promise} + */ +export function migrateLatest(trx: Knex | Knex.Transaction, config: SyncConfig): Promise { + return trx.migrate.latest(getMigrationConfig(config)); +} + +/** + * Rollback migrations on a target database connection (transaction). + * + * @param {(Knex | Knex.Transaction)} trx + * @param {SyncConfig} config + * @returns {Promise} + */ +export function rollback(trx: Knex | Knex.Transaction, config: SyncConfig): Promise { + return trx.migrate.rollback(getMigrationConfig(config)); +} + +/** + * List migrations on a target database connection (transaction). + * + * @param {(Knex | Knex.Transaction)} trx + * @param {SyncConfig} config + * @returns {Promise} + */ +export function list(trx: Knex | Knex.Transaction, config: SyncConfig): Promise { + return trx.migrate.list(getMigrationConfig(config)); +} From 07b9638cb4b04419bd3109609884dc00e6d30d84 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Wed, 18 Mar 2020 01:00:36 +0545 Subject: [PATCH 20/80] Dedup connection mapping --- src/api.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/api.ts b/src/api.ts index 0f6e8cc4..734339db 100644 --- a/src/api.ts +++ b/src/api.ts @@ -30,8 +30,7 @@ export async function synchronize( options?: SyncParams ): Promise { log('Starting to synchronize.'); - const connectionList = Array.isArray(conn) ? conn : [conn]; - const connections = mapToConnectionReferences(connectionList); + const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); const processes = connections.map(({ connection, id: connectionId }) => () => synchronizeDatabase(connection, { @@ -51,12 +50,14 @@ export async function synchronize( } /** - * Map connection configuration list to the connection instances. + * Map user provided connection(s) to the connection instances. * - * @param {((ConnectionConfig | Knex)[])} connectionList + * @param {(ConnectionConfig[] | ConnectionConfig | Knex[] | Knex)} conn * @returns {ConnectionReference[]} */ -function mapToConnectionReferences(connectionList: (ConnectionConfig | Knex)[]): ConnectionReference[] { +function mapToConnectionReferences(conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex): ConnectionReference[] { + const connectionList = Array.isArray(conn) ? conn : [conn]; + return connectionList.map(connection => { if (isKnexInstance(connection)) { log(`Received connection instance to database: ${connection.client.config.connection.database}`); From bea9fed3157547bcc17c0fe986c11f2db7f57e26 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 21 Mar 2020 18:00:32 +0545 Subject: [PATCH 21/80] Organize directories --- src/KnexMigrationSource.ts | 2 +- src/api.ts | 2 +- src/commands/synchronize.ts | 2 +- src/config.ts | 2 +- src/service/configInjection.ts | 2 +- src/service/execution.ts | 2 +- src/service/sqlRunner.ts | 2 +- src/service/sync.ts | 2 +- src/{ => util}/logger.ts | 0 9 files changed, 8 insertions(+), 8 deletions(-) rename src/{ => util}/logger.ts (100%) diff --git a/src/KnexMigrationSource.ts b/src/KnexMigrationSource.ts index 6244ee07..fe68eed9 100644 --- a/src/KnexMigrationSource.ts +++ b/src/KnexMigrationSource.ts @@ -1,6 +1,6 @@ import * as Knex from 'knex'; -import { log } from '../src/logger'; +import { log } from './util/logger'; import * as migratorService from '../src/service/migrator'; import SqlMigrationEntry from './domain/SqlMigrationEntry'; diff --git a/src/api.ts b/src/api.ts index 734339db..c8a2be39 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,7 +1,7 @@ import * as Knex from 'knex'; import { mergeDeepRight } from 'ramda'; -import { log } from './logger'; +import { log } from './util/logger'; import { getConnectionId } from './config'; import SyncParams from './domain/SyncParams'; import SyncConfig from './domain/SyncConfig'; diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index 04751795..76ca0848 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -1,6 +1,6 @@ import { Command, flags } from '@oclif/command'; -import { log } from '../logger'; +import { log } from '../util/logger'; import { handleFlags } from '../cli'; import { getElapsedTime } from '../util/ts'; import SyncResult from '../domain/SyncResult'; diff --git a/src/config.ts b/src/config.ts index b6ad66e4..e991ede1 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2,8 +2,8 @@ import * as path from 'path'; import * as yaml from 'yamljs'; import { mergeDeepRight } from 'ramda'; -import { log } from './logger'; import * as fs from './util/fs'; +import { log } from './util/logger'; import { isObject } from './util/types'; import DbConfig from './domain/DbConfig'; import SyncConfig from './domain/SyncConfig'; diff --git a/src/service/configInjection.ts b/src/service/configInjection.ts index 7d42bab6..f45f976c 100644 --- a/src/service/configInjection.ts +++ b/src/service/configInjection.ts @@ -2,8 +2,8 @@ import * as fs from 'fs'; import * as path from 'path'; import * as Knex from 'knex'; -import { dbLogger } from '../logger'; import Mapping from '../domain/Mapping'; +import { dbLogger } from '../util/logger'; import SyncContext from '../domain/SyncContext'; import { expandEnvVarsInMap } from '../util/env'; import KeyValuePair from '../domain/KeyValuePair'; diff --git a/src/service/execution.ts b/src/service/execution.ts index 3489db07..7724d1ea 100644 --- a/src/service/execution.ts +++ b/src/service/execution.ts @@ -1,4 +1,4 @@ -import { log } from '../logger'; +import { log } from '../util/logger'; import SyncConfig from '../domain/SyncConfig'; import { Promiser, runSequentially } from '../util/promise'; diff --git a/src/service/sqlRunner.ts b/src/service/sqlRunner.ts index 64450940..69b77f5d 100644 --- a/src/service/sqlRunner.ts +++ b/src/service/sqlRunner.ts @@ -3,9 +3,9 @@ import * as path from 'path'; import { reverse, keys } from 'ramda'; import * as fs from '../util/fs'; -import { dbLogger } from '../logger'; import Mapping from '../domain/Mapping'; import SqlCode from '../domain/SqlCode'; +import { dbLogger } from '../util/logger'; import * as promise from '../util/promise'; import SqlFileInfo from '../domain/SqlFileInfo'; import DatabaseObjectTypes from '../enum/DatabaseObjectTypes'; diff --git a/src/service/sync.ts b/src/service/sync.ts index 557c8cc9..807298ca 100644 --- a/src/service/sync.ts +++ b/src/service/sync.ts @@ -1,8 +1,8 @@ import * as Knex from 'knex'; import { isCLI } from '../config'; -import { dbLogger } from '../logger'; import * as sqlRunner from './sqlRunner'; +import { dbLogger } from '../util/logger'; import { getElapsedTime } from '../util/ts'; import SyncResult from '../domain/SyncResult'; import SyncContext from '../domain/SyncContext'; diff --git a/src/logger.ts b/src/util/logger.ts similarity index 100% rename from src/logger.ts rename to src/util/logger.ts From 41af53db12cff298eb06fc8cca077ad4786b795d Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 22 Mar 2020 14:19:06 +0545 Subject: [PATCH 22/80] Add interfaces for migration --- src/domain/MigrationContext.ts | 12 ++++++++++++ src/domain/MigrationRunner.ts | 11 +++++++++++ 2 files changed, 23 insertions(+) create mode 100644 src/domain/MigrationContext.ts create mode 100644 src/domain/MigrationRunner.ts diff --git a/src/domain/MigrationContext.ts b/src/domain/MigrationContext.ts new file mode 100644 index 00000000..a7c798a4 --- /dev/null +++ b/src/domain/MigrationContext.ts @@ -0,0 +1,12 @@ +import MigrationRunner from './MigrationRunner'; + +/** + * A contract for migration source context. + */ +interface MigrationContext { + connectionId: string; + keys(): string[]; + get(name: string): MigrationRunner; +} + +export default MigrationContext; diff --git a/src/domain/MigrationRunner.ts b/src/domain/MigrationRunner.ts new file mode 100644 index 00000000..2cecde3b --- /dev/null +++ b/src/domain/MigrationRunner.ts @@ -0,0 +1,11 @@ +import Knex from 'knex'; + +/** + * Contract for a migration runner. + */ +interface MigrationRunner { + up: (db: Knex | Knex.Transaction) => Promise; + down: (db: Knex | Knex.Transaction) => Promise; +} + +export default MigrationRunner; From a08577a4c6f237355f6ca2b24b3789a5d4f27c24 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 22 Mar 2020 14:28:10 +0545 Subject: [PATCH 23/80] Revamp KnexMigrationSource --- src/migration/KnexMigrationSource.ts | 60 ++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/migration/KnexMigrationSource.ts diff --git a/src/migration/KnexMigrationSource.ts b/src/migration/KnexMigrationSource.ts new file mode 100644 index 00000000..8f70ce80 --- /dev/null +++ b/src/migration/KnexMigrationSource.ts @@ -0,0 +1,60 @@ +import { dbLogger } from '../util/logger'; +import MigrationContext from '../domain/MigrationContext'; + +/** + * MigrationSource class for the Knex Migration API. + */ +class KnexMigrationSource { + private migrationContext: MigrationContext; + private log: debug.Debugger; + + /** + * KnexMigrationSource constructor. + * + * @param {MigrationEntry[]} migrationLists + */ + constructor(migrationContext: MigrationContext) { + this.log = dbLogger(migrationContext.connectionId); + this.migrationContext = migrationContext; + } + + /** + * Gets a list of migration names. + * + * @returns {Promise} + */ + getMigrations(): Promise { + const migrations = this.migrationContext.keys(); + + this.log('getMigrations - resolve: %o', migrations); + + return Promise.resolve(migrations); + } + + /** + * Gets the name of the migration. + * + * @param {string} migration + * @returns {string} + */ + getMigrationName(migration: string): string { + this.log('getMigrationName - resolve: ', migration); + + return migration; + } + /** + * Get the migration runner. + * + * @param {SqlMigrationEntry} migration + */ + getMigration(name: string) { + const migration = this.migrationContext.get(name); + + this.log(`getMigration - ${name}`); + this.log(`getMigration - resolve: %o`, migration); + + return migration; + } +} + +export default KnexMigrationSource; From 4eb8f7dfbc0ba817121887696cb41844939a7ef9 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 22 Mar 2020 14:31:32 +0545 Subject: [PATCH 24/80] Implement SqlMigrationContext --- src/migration/SqlMigrationContext.ts | 79 ++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 src/migration/SqlMigrationContext.ts diff --git a/src/migration/SqlMigrationContext.ts b/src/migration/SqlMigrationContext.ts new file mode 100644 index 00000000..38497fa3 --- /dev/null +++ b/src/migration/SqlMigrationContext.ts @@ -0,0 +1,79 @@ +import * as Knex from 'knex'; + +import { dbLogger } from '../util/logger'; +import MigrationRunner from '../domain/MigrationRunner'; +import MigrationContext from '../domain/MigrationContext'; +import SqlMigrationEntry from '../domain/SqlMigrationEntry'; + +/** + * SQL source migration context for KnexMigrationSource. + */ +class SqlMigrationContext implements MigrationContext { + private list: SqlMigrationEntry[]; + private log: debug.Debugger; + public connectionId: string; + + /** + * SqlMigrationContext constructor. + * + * @param {string} connectionId + * @param {SqlMigrationEntry[]} list + */ + constructor(connectionId: string, list: SqlMigrationEntry[]) { + this.list = list; + this.connectionId = connectionId; + this.log = dbLogger(connectionId); + } + + /** + * Get migration keys. + * + * @returns {string[]} + */ + keys(): string[] { + return this.list.map(({ name }) => name); + } + + /** + * Get the migration runner. + * + * @param {string} key + * @returns {MigrationRunner} + */ + get(key: string): MigrationRunner { + // TODO: Optimize - no need to find from the list for every item you get. Map it internally. + const entry = this.list.find(({ name }) => name === key); + + if (!entry) { + this.log(`Migration entry not found ${key}.`); + + throw new Error(`Cannot find the migration entry ${key}`); + } + + const logMigration = this.log.extend('migration'); + + return { + up: async (db: Knex) => { + if (entry.queries.up) { + logMigration(`UP - ${key}`); + + return db.raw(entry.queries.up.sql); + } + + return false; + }, + + down: async (db: Knex) => { + if (entry.queries.down) { + logMigration(`DOWN - ${key}`); + + return db.raw(entry.queries.down.sql); + } + + return false; + } + }; + } +} + +export default SqlMigrationContext; From 99ca065f29737a98e28233371135b1f1bca75ac5 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 22 Mar 2020 14:32:35 +0545 Subject: [PATCH 25/80] Remove old KnexMigrationSource --- src/KnexMigrationSource.ts | 70 -------------------------------------- 1 file changed, 70 deletions(-) delete mode 100644 src/KnexMigrationSource.ts diff --git a/src/KnexMigrationSource.ts b/src/KnexMigrationSource.ts deleted file mode 100644 index fe68eed9..00000000 --- a/src/KnexMigrationSource.ts +++ /dev/null @@ -1,70 +0,0 @@ -import * as Knex from 'knex'; - -import { log } from './util/logger'; -import * as migratorService from '../src/service/migrator'; -import SqlMigrationEntry from './domain/SqlMigrationEntry'; - -/** - * MigrationSource class for the Knex Migration API. - */ -class KnexMigrationSource { - /** - * Migration directory path. - * @type {string} - */ - private migrationPath: string; - - /** - * Creates an instance of KnexMigrationSource. - * - * @param {MigrationEntry[]} migrationLists - */ - constructor(migrationPath: string) { - log('Creating new KnexMigrationSource with path:', migrationPath); - - this.migrationPath = migrationPath; - } - - /** - * Gets a list of migration entries. - * - * - * @returns {Promise} - */ - async getMigrations(): Promise { - const migrations = await migratorService.resolveSqlMigrations(this.migrationPath); - - log('Resolved migrations', migrations); - - return migrations; - } - - /** - * Get the name of the migration entry. - * - * @param {SqlMigrationEntry} migration - * @returns {string} - * @memberof KnexMigrationSource - */ - getMigrationName(migration: SqlMigrationEntry): string { - log(`Returning migration name: ${migration.name}`); - - return migration.name; - } - - /** - * Get the migration functions. - * - * @param {SqlMigrationEntry} migration - */ - getMigration(migration: SqlMigrationEntry) { - const { queries } = migration; - - return { - up: (db: Knex) => (queries.up ? db.raw(queries.up.sql) : Promise.resolve()), - down: (db: Knex) => (queries.down ? db.raw(queries.down.sql) : Promise.resolve()) - }; - } -} - -export default KnexMigrationSource; From 7adf06c70d466786015cbec6ab8ba8a2ab97d77a Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 22 Mar 2020 14:41:02 +0545 Subject: [PATCH 26/80] Enhance debug logs --- src/migration/KnexMigrationSource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/migration/KnexMigrationSource.ts b/src/migration/KnexMigrationSource.ts index 8f70ce80..7074c903 100644 --- a/src/migration/KnexMigrationSource.ts +++ b/src/migration/KnexMigrationSource.ts @@ -26,7 +26,7 @@ class KnexMigrationSource { getMigrations(): Promise { const migrations = this.migrationContext.keys(); - this.log('getMigrations - resolve: %o', migrations); + this.log('getMigrations - resolve:\n%O', migrations); return Promise.resolve(migrations); } From 67580fa85396b877a3b1537fbb97d68f12c7119a Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 22 Mar 2020 16:03:30 +0545 Subject: [PATCH 27/80] Rewrite KnexMigrationSource tests --- test/KnexMigrationSource.test.ts | 153 --------------------- test/migration/KnexMigrationSource.test.ts | 60 ++++++++ 2 files changed, 60 insertions(+), 153 deletions(-) delete mode 100644 test/KnexMigrationSource.test.ts create mode 100644 test/migration/KnexMigrationSource.test.ts diff --git a/test/KnexMigrationSource.test.ts b/test/KnexMigrationSource.test.ts deleted file mode 100644 index 1e506bca..00000000 --- a/test/KnexMigrationSource.test.ts +++ /dev/null @@ -1,153 +0,0 @@ -import * as path from 'path'; -import { expect } from 'chai'; -import { describe, it } from 'mocha'; - -import { write, mkdtemp } from '../src/util/fs'; -import KnexMigrationSource from '../src/KnexMigrationSource'; - -describe('UTIL: KnexMigrationSource', () => { - const getInstance = async () => { - const migrationPath = await mkdtemp(); - const instance = new KnexMigrationSource(migrationPath); - - return { migrationPath, instance }; - }; - - describe('getMigrations', () => { - it('should return empty array if the migration directory is empty.', async () => { - const { instance } = await getInstance(); - const result = await instance.getMigrations(); - - expect(result).to.deep.equal([]); - }); - - it('should resolve all the information related to the migration entries.', async () => { - const { migrationPath, instance } = await getInstance(); - - // Populate migration files to a directory. - await Promise.all([ - write(path.join(migrationPath, '0001_mgr.up.sql'), 'CREATE TABLE test_mgr1'), - write(path.join(migrationPath, '0001_mgr.down.sql'), 'DROP TABLE test_mgr1'), - write(path.join(migrationPath, '0002_mgr.up.sql'), 'CREATE TABLE test_mgr2'), - write(path.join(migrationPath, '0002_mgr.down.sql'), 'DROP TABLE test_mgr2'), - write(path.join(migrationPath, '0003_mgr.up.sql'), 'CREATE TABLE test_mgr3'), - write(path.join(migrationPath, '0003_mgr.down.sql'), 'DROP TABLE test_mgr3'), - write(path.join(migrationPath, '0004_mgr.up.sql'), 'CREATE TABLE test_mgr4'), - write(path.join(migrationPath, '0004_mgr.down.sql'), 'DROP TABLE test_mgr4'), - write(path.join(migrationPath, '0005_mgr.up.sql'), 'CREATE TABLE test_mgr5'), - write(path.join(migrationPath, '0005_mgr.down.sql'), 'DROP TABLE test_mgr5') - ]); - - const result = await instance.getMigrations(); - - // Test the migrations entries retrieved from the directory. - expect(result).to.deep.equal([ - { - name: '0001_mgr', - queries: { - up: { name: '0001_mgr.up.sql', sql: 'CREATE TABLE test_mgr1' }, - down: { name: '0001_mgr.down.sql', sql: 'DROP TABLE test_mgr1' } - } - }, - { - name: '0002_mgr', - queries: { - up: { name: '0002_mgr.up.sql', sql: 'CREATE TABLE test_mgr2' }, - down: { name: '0002_mgr.down.sql', sql: 'DROP TABLE test_mgr2' } - } - }, - { - name: '0003_mgr', - queries: { - up: { name: '0003_mgr.up.sql', sql: 'CREATE TABLE test_mgr3' }, - down: { name: '0003_mgr.down.sql', sql: 'DROP TABLE test_mgr3' } - } - }, - { - name: '0004_mgr', - queries: { - up: { name: '0004_mgr.up.sql', sql: 'CREATE TABLE test_mgr4' }, - down: { name: '0004_mgr.down.sql', sql: 'DROP TABLE test_mgr4' } - } - }, - { - name: '0005_mgr', - queries: { - up: { name: '0005_mgr.up.sql', sql: 'CREATE TABLE test_mgr5' }, - down: { name: '0005_mgr.down.sql', sql: 'DROP TABLE test_mgr5' } - } - } - ]); - }); - }); - - describe('getMigrationName', async () => { - it('should return the name of the migration.', async () => { - const { instance } = await getInstance(); - const entry = { - name: '0005_mgr', - queries: { - up: { name: '0005_mgr.up.sql', sql: 'CREATE TABLE test_mgr5' }, - down: { name: '0005_mgr.down.sql', sql: 'DROP TABLE test_mgr5' } - } - }; - - expect(instance.getMigrationName(entry)).to.equal('0005_mgr'); - }); - }); - - describe('getMigration', async () => { - const dbStub = { - raw: (sql: string) => Promise.resolve(`Result of "${sql}"`) - }; - - it('should return the migration functions.', async () => { - const { instance } = await getInstance(); - const entry = { - name: '0005_mgr', - queries: { - up: { name: '0005_mgr.up.sql', sql: 'CREATE TABLE test_mgr5' }, - down: { name: '0005_mgr.down.sql', sql: 'DROP TABLE test_mgr5' } - } - }; - - const migration = instance.getMigration(entry); - - const upResult = await migration.up(dbStub as any); - const downResult = await migration.down(dbStub as any); - - expect(upResult).to.equal('Result of "CREATE TABLE test_mgr5"'); - expect(downResult).to.equal('Result of "DROP TABLE test_mgr5"'); - }); - - it('could return empty promises for the migration functions if the files have been unresolved.', async () => { - const { instance } = await getInstance(); - const entry1 = { - name: '0004_mgr', - queries: { - up: { name: '0004_mgr.up.sql', sql: 'UPDATE test_mgr4 SET is_active = 0' } - } - }; - const migration1 = instance.getMigration(entry1); - const upResult1 = await migration1.up(dbStub as any); - const downResult1 = await migration1.down(dbStub as any); - - expect(upResult1).to.equal('Result of "UPDATE test_mgr4 SET is_active = 0"'); - expect(downResult1).to.equal(undefined); - - const entry2 = { - name: '0005_mgr', - queries: { - down: { name: '0005_mgr.down.sql', sql: 'UPDATE test_mgr5 SET is_disabled = 1' } - } - }; - - const migration2 = instance.getMigration(entry2); - const upResult2 = await migration2.up(dbStub as any); - const downResult2 = await migration2.down(dbStub as any); - - expect(upResult2).to.equal(undefined); - expect(downResult2).to.equal('Result of "UPDATE test_mgr5 SET is_disabled = 1"'); - }); - }); -}); diff --git a/test/migration/KnexMigrationSource.test.ts b/test/migration/KnexMigrationSource.test.ts new file mode 100644 index 00000000..0df6c260 --- /dev/null +++ b/test/migration/KnexMigrationSource.test.ts @@ -0,0 +1,60 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import MigrationContext from '../../src/domain/MigrationContext'; +import KnexMigrationSource from '../../src/migration/KnexMigrationSource'; + +describe('UTIL: KnexMigrationSource', () => { + const getInstance = () => { + const migrationContext: MigrationContext = new (class { + connectionId = 'testdb1'; + + keys(): string[] { + return ['mgr_001', 'mgr_002', 'mgr_003', 'mgr_004']; + } + + get(key: string) { + return { + up: async () => `UP: ${key}`, + down: async () => `DOWN: ${key}` + }; + } + })(); + + const instance = new KnexMigrationSource(migrationContext); + + return instance; + }; + + describe('getMigrations', () => { + it('should return the migration names provided by the MigrationContext.', async () => { + const instance = getInstance(); + const result = await instance.getMigrations(); + + expect(result).to.deep.equal(['mgr_001', 'mgr_002', 'mgr_003', 'mgr_004']); + }); + }); + + describe('getMigrationName', async () => { + it('should return the name as-is.', async () => { + const instance = await getInstance(); + + expect(instance.getMigrationName('mgr_001')).to.equal('mgr_001'); + }); + }); + + describe('getMigration', async () => { + it('should return the migration runner.', async () => { + const instance = await getInstance(); + const dbStub = {} as any; + + const migration = instance.getMigration('mgr_002'); + + const upResult = await migration.up(dbStub); + const downResult = await migration.down(dbStub); + + expect(upResult).to.equal('UP: mgr_002'); + expect(downResult).to.equal('DOWN: mgr_002'); + }); + }); +}); From 73d01bb88ec5bf29fe2eb81a8a58ea7efc79b9a9 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 22 Mar 2020 16:19:59 +0545 Subject: [PATCH 28/80] Update migrator service test --- test/service/migrator.test.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/service/migrator.test.ts b/test/service/migrator.test.ts index d500b721..ce18a302 100644 --- a/test/service/migrator.test.ts +++ b/test/service/migrator.test.ts @@ -6,7 +6,7 @@ import { write, mkdtemp } from '../../src/util/fs'; import * as migratorService from '../../src/service/migrator'; describe('SERVICE: migrator', () => { - describe('getSqlMigrationEntries', async () => { + describe('getSqlMigrationNames', async () => { it('should return the list of valid migrations under the directory.', async () => { const migrationPath = await mkdtemp(); @@ -39,13 +39,14 @@ describe('SERVICE: migrator', () => { write(path.join(migrationPath, '0002_mgr.down.sql'), 'SELECT 1'), write(path.join(migrationPath, 'test.sql'), 'SELECT 2'), write(path.join(migrationPath, 'migrate.sql'), 'SELECT 3'), - write(path.join(migrationPath, '.gitignore'), '') + write(path.join(migrationPath, '.gitignore'), ''), + write(path.join(migrationPath, '0003_mgr.down.sql'), 'SELECT 1') ]); const result = await migratorService.getSqlMigrationNames(migrationPath); // Test the migrations entries retrieved from the directory. - expect(result).to.deep.equal(['0001_mgr', '0002_mgr']); + expect(result).to.deep.equal(['0001_mgr', '0002_mgr', '0003_mgr']); }); }); @@ -58,7 +59,8 @@ describe('SERVICE: migrator', () => { write(path.join(migrationPath, '0001_mgr.up.sql'), 'CREATE TABLE test_mgr1'), write(path.join(migrationPath, '0002_mgr.down.sql'), 'DROP TABLE test_mgr2'), write(path.join(migrationPath, '0003_mgr.up.sql'), 'CREATE TABLE test_mgr3'), - write(path.join(migrationPath, '0003_mgr.down.sql'), 'DROP TABLE test_mgr3') + write(path.join(migrationPath, '0003_mgr.down.sql'), 'DROP TABLE test_mgr3'), + write(path.join(migrationPath, '.gitignore'), '') ]); const result = await migratorService.resolveSqlMigrations(migrationPath); From 44155714df671b58de5c43e86b81693d9c5ab49a Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 22 Mar 2020 16:50:44 +0545 Subject: [PATCH 29/80] Add tests for SqlMigrationContext --- test/migration/SqlMigrationContext.test.ts | 104 +++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 test/migration/SqlMigrationContext.test.ts diff --git a/test/migration/SqlMigrationContext.test.ts b/test/migration/SqlMigrationContext.test.ts new file mode 100644 index 00000000..215f1416 --- /dev/null +++ b/test/migration/SqlMigrationContext.test.ts @@ -0,0 +1,104 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import SqlMigrationEntry from '../../src/domain/SqlMigrationEntry'; +import SqlMigrationContext from '../../src/migration/SqlMigrationContext'; + +describe('UTIL: SqlMigrationContext', () => { + const getInstance = (list: SqlMigrationEntry[]) => new SqlMigrationContext('testdb1', list); + + describe('keys', () => { + it('should return an empty list if migrations are empty.', () => { + const instance = getInstance([]); + + expect(instance.keys()).to.deep.equal([]); + }); + + it('should return the migration names.', () => { + const instance = getInstance([ + { + name: '0001_mgr', + queries: { + up: { name: '0001_mgr.up.sql', sql: 'CREATE TABLE test_mgr1' }, + down: { name: '0001_mgr.down.sql', sql: 'DROP TABLE test_mgr1' } + } + }, + { + name: '0002_mgr', + queries: { + up: { name: '0002_mgr.up.sql', sql: 'CREATE TABLE test_mgr2' }, + down: { name: '0002_mgr.down.sql', sql: 'DROP TABLE test_mgr2' } + } + }, + { + name: '0003_mgr', + queries: { + up: { name: '0003_mgr.up.sql', sql: 'CREATE TABLE test_mgr3' }, + down: { name: '0003_mgr.down.sql', sql: 'DROP TABLE test_mgr3' } + } + } + ]); + + expect(instance.keys()).to.deep.equal(['0001_mgr', '0002_mgr', '0003_mgr']); + }); + }); + + describe('get', () => { + const dbStub = { + raw: (sql: string) => Promise.resolve(`Result of "${sql}"`) + } as any; + const instance = getInstance([ + { + name: '0001_mgr', + queries: { + up: { name: '0001_mgr.up.sql', sql: 'CREATE TABLE test_mgr' }, + down: { name: '0001_mgr.down.sql', sql: 'DROP TABLE test_mgr' } + } + }, + { + name: '0002_mgr', + queries: { + up: { name: '0002_mgr.up.sql', sql: 'UPDATE test_mgr SET is_active = 0' } + } + }, + { + name: '0003_mgr', + queries: { + down: { name: '0003_mgr.down.sql', sql: 'UPDATE test_mgr SET is_disabled = 1' } + } + } + ]); + + it('should return the migration runner.', async () => { + const runner = instance.get('0001_mgr'); + + const upResult = await runner.up(dbStub); + const downResult = await runner.down(dbStub); + + expect(upResult).to.equal('Result of "CREATE TABLE test_mgr"'); + expect(downResult).to.equal('Result of "DROP TABLE test_mgr"'); + }); + + it('returned migration functions should resolve to false the corresponding source are unresolved.', async () => { + const migration1 = instance.get('0002_mgr'); + const upResult1 = await migration1.up(dbStub); + const downResult1 = await migration1.down(dbStub); + + expect(upResult1).to.equal('Result of "UPDATE test_mgr SET is_active = 0"'); + expect(downResult1).to.equal(false); + + const migration2 = instance.get('0003_mgr'); + const upResult2 = await migration2.up(dbStub); + const downResult2 = await migration2.down(dbStub); + + expect(upResult2).to.equal(false); + expect(downResult2).to.equal('Result of "UPDATE test_mgr SET is_disabled = 1"'); + }); + + it('should throw an error if the migration entry was not found.', () => { + expect(() => instance.get('0010_mgr_something_else')).to.throw( + 'Cannot find the migration entry 0010_mgr_something_else' + ); + }); + }); +}); From 4748d00a8b57bc72f5806260b45fa14208c16f22 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 22 Mar 2020 16:53:26 +0545 Subject: [PATCH 30/80] Just do it for now --- src/api.ts | 124 +++++++++++++++++++++++ src/cli.ts | 102 +++++++++++++++++-- src/commands/synchronize.ts | 2 +- src/service/migrator.ts | 193 +++++++++++++++++++++++++++++------- 4 files changed, 372 insertions(+), 49 deletions(-) diff --git a/src/api.ts b/src/api.ts index c8a2be39..8ee602df 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,4 +1,5 @@ import * as Knex from 'knex'; +import * as path from 'path'; import { mergeDeepRight } from 'ramda'; import { log } from './util/logger'; @@ -15,6 +16,9 @@ import { isKnexInstance, getConfig, createInstance } from './util/db'; // Services import { synchronizeDatabase } from './service/sync'; import { executeProcesses } from './service/execution'; +import * as migratorService from './service/migrator'; +import SqlMigrationContext from './migration/SqlMigrationContext'; +import KnexMigrationSource from './migration/KnexMigrationSource'; /** * Synchronize all the configured database connections. @@ -49,6 +53,126 @@ export async function synchronize( return results; } +export async function migrateLatest( + config: SyncConfig, + conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, + options?: SyncParams +): Promise { + log('Starting to migrate.'); + const connections = mapToConnectionReferences(conn); + const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); + + const { basePath, migration } = config; + const migrationPath = path.join(basePath, migration.directory); + const migrations = await migratorService.resolveSqlMigrations(migrationPath); + + log('Migration Path:', migrationPath); + log('Available migrations:'); + log('%O', migrations); + + // TODO: We'll need to support different types of migrations eg both sql & js + // For instance migrations in JS would have different context like JavaScriptMigrationContext. + const getMigrationContext = (connectionId: string) => new SqlMigrationContext(connectionId, migrations); + + const processes = connections.map(({ connection, id: connectionId }) => () => + migratorService.migrateLatest(connection, { + config, + params, + connectionId, + knexMigrationConfig: { + tableName: config.migration.tableName, + migrationSource: new KnexMigrationSource(getMigrationContext(connectionId)) + } + }) + ); + + const results = await executeProcesses(processes, config); + + log('Migrations completed.'); + + return results; +} + +export async function migrateRollback( + config: SyncConfig, + conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, + options?: SyncParams +): Promise { + log('Starting to migrate.'); + const connections = mapToConnectionReferences(conn); + const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); + + const { basePath, migration } = config; + const migrationPath = path.join(basePath, migration.directory); + const migrations = await migratorService.resolveSqlMigrations(migrationPath); + + log('Migration Path:', migrationPath); + log('Available migrations:'); + log('%O', migrations); + + // TODO: We'll need to support different types of migrations eg both sql & js + // For instance migrations in JS would have different context like JavaScriptMigrationContext. + const getMigrationContext = (connectionId: string) => new SqlMigrationContext(connectionId, migrations); + + const processes = connections.map(({ connection, id: connectionId }) => () => + migratorService.migrateRollback(connection, { + config, + params, + connectionId, + knexMigrationConfig: { + tableName: config.migration.tableName, + migrationSource: new KnexMigrationSource(getMigrationContext(connectionId)) + } + }) + ); + + const results = await executeProcesses(processes, config); + + log('Migrations completed.'); + + return results; +} + +export async function migrateList( + config: SyncConfig, + conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, + options?: SyncParams +): Promise { + log('Starting to migrate.'); + const connections = mapToConnectionReferences(conn); + const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); + + const { basePath, migration } = config; + const migrationPath = path.join(basePath, migration.directory); + const migrations = await migratorService.resolveSqlMigrations(migrationPath); + + log('Migration Path:', migrationPath); + log('Available migrations:'); + log('%O', migrations); + + // TODO: We'll need to support different types of migrations eg both sql & js + // For instance migrations in JS would have different context like JavaScriptMigrationContext. + const getMigrationContext = (connectionId: string) => new SqlMigrationContext(connectionId, migrations); + + const processes = connections.map(({ connection, id: connectionId }) => () => + migratorService.migrateList(connection, { + config, + params, + connectionId, + knexMigrationConfig: { + tableName: config.migration.tableName, + migrationSource: new KnexMigrationSource(getMigrationContext(connectionId)) + } + }) + ); + + const results = await executeProcesses(processes, config); + + log('Migrations completed.'); + + return results; +} + /** * Map user provided connection(s) to the connection instances. * diff --git a/src/cli.ts b/src/cli.ts index 26e74762..e7dcb51b 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -4,7 +4,10 @@ import * as fs from './util/fs'; import { printLine } from './util/io'; import SyncDbOptions from './domain/SyncDbOptions'; import { CONNECTIONS_FILENAME } from './constants'; -import { resolveConnectionsFromEnv } from './config'; +import { resolveConnectionsFromEnv, loadConfig, resolveConnections } from './config'; +import { getElapsedTime } from './util/misc'; +import SyncParams from './domain/SyncParams'; +import { log } from './util/logger'; /** * Generates connections.sync-db.json file. @@ -22,16 +25,93 @@ async function generateConnection(): Promise { await printLine(`Generated file: ${CONNECTIONS_FILENAME}\n`); } -async function migrate(): Promise { - printLine('Migrate.'); +async function migrate(params: SyncParams): Promise { + const config = await loadConfig(); + const connections = await resolveConnections(); + const { migrateLatest } = await import('./api'); + const timeStart = process.hrtime(); + + await printLine('Synchronizing...\n'); + + const results = await migrateLatest(config, connections, params); + + log('Results:', results); + console.log('Results', results); // tslint:disable-line + + const successfulCount = results.filter(item => item.success).length; + + if (successfulCount > 0) { + // Display output. + await printLine( + `Migration complete for ${successfulCount} / ${results.length} connection(s). ` + + `(${getElapsedTime(timeStart)}s)` + ); + } + + // If all completed successfully, exit gracefully. + if (results.length === successfulCount) { + return process.exit(0); + } + + throw new Error(`Synchronization failed for some connections.`); } -async function rollback(): Promise { - printLine('Rollback.'); +async function rollback(params: SyncParams): Promise { + const config = await loadConfig(); + const connections = await resolveConnections(); + const { migrateRollback } = await import('./api'); + const timeStart = process.hrtime(); + + await printLine('Rolling back...\n'); + + const results = await migrateRollback(config, connections, params); + + log('Results:', results); + console.log('Results', results); // tslint:disable-line + + const successfulCount = results.filter(item => item.success).length; + + if (successfulCount > 0) { + // Display output. + await printLine( + `Rollback complete for ${successfulCount} / ${results.length} connection(s). ` + `(${getElapsedTime(timeStart)}s)` + ); + } + + // If all completed successfully, exit gracefully. + if (results.length === successfulCount) { + return process.exit(0); + } + + throw new Error(`Synchronization failed for some connections.`); } -async function list(): Promise { - printLine('List'); +async function list(params: SyncParams): Promise { + const config = await loadConfig(); + const connections = await resolveConnections(); + const { migrateList } = await import('./api'); + const timeStart = process.hrtime(); + + const results = await migrateList(config, connections, params); + + log('Results:', results); + console.log('Results', results); // tslint:disable-line + + const successfulCount = results.filter(item => item.success).length; + + if (successfulCount > 0) { + // Display output. + await printLine( + `List complete for ${successfulCount} / ${results.length} connection(s). ` + `(${getElapsedTime(timeStart)}s)` + ); + } + + // If all completed successfully, exit gracefully. + if (results.length === successfulCount) { + return process.exit(0); + } + + throw new Error(`Synchronization failed for some connections.`); } /** @@ -40,18 +120,18 @@ async function list(): Promise { * @param {SyncDbOptions} flags * @returns {Promise} */ -export async function handleFlags(flags: SyncDbOptions): Promise { +export async function handleFlags(flags: SyncDbOptions, params: SyncParams): Promise { if (flags['generate-connections']) { await generateConnection(); process.exit(0); } else if (flags['migrate']) { - await migrate(); + await migrate(params); process.exit(0); } else if (flags['rollback']) { - await rollback(); + await rollback(params); process.exit(0); } else if (flags['list']) { - await list(); + await list(params); process.exit(0); } } diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index 76ca0848..d45c4752 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -62,7 +62,7 @@ class Synchronize extends Command { console.log({ parsedFlags }); // tslint:disable-line try { - await handleFlags(parsedFlags); + await handleFlags(parsedFlags, params); const config = await loadConfig(); const connections = await resolveConnections(); diff --git a/src/service/migrator.ts b/src/service/migrator.ts index 4735176d..3fd9f804 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -3,9 +3,17 @@ import * as path from 'path'; import { glob, exists } from '../util/fs'; import { resolveFile } from './sqlRunner'; -import SyncConfig from '../domain/SyncConfig'; -import KnexMigrationSource from '../KnexMigrationSource'; +// import SqlMigrationContext, { MigrationContext } from '../migration/SqlMigrationContext'; import SqlMigrationEntry from '../domain/SqlMigrationEntry'; +import SyncResult from '../domain/SyncResult'; +// import SyncContext from '../domain/SyncContext'; +import { dbLogger } from '../util/logger'; +import { getElapsedTime } from '../util/misc'; +import { isCLI } from '../config'; +import ExecutionContext from '../domain/ExecutionContext'; +// import KnexMigrationSource from '../migration/KnexMigrationSource'; +import SyncConfig from '../domain/SyncConfig'; +import SyncParams from '../domain/SyncParams'; const FILE_PATTERN = /(.+)\.(up|down)\.sql$/; @@ -61,52 +69,163 @@ export async function resolveSqlMigrations(migrationPath: string): Promise { +// const log = dbLogger(context.connectionId); +// log('Getting migration config'); + +// const { basePath, migration } = context.config; +// const migrationPath = path.join(basePath, migration.directory); + +// const migrations = await resolveSqlMigrations(migrationPath); + +// // TODO: Multiple migration context based on type of migration (sql, js, etc) +// const migrationContext = new SqlMigrationContext(context.connectionId, migrations); + +// log('Available Migrations:', migrations); +// log('Table Name:', migration.tableName); + +// return { +// tableName: migration.tableName, +// migrationSource: new KnexMigrationSource(migrationContext) +// }; +// } + /** - * Get a new instance of KnexMigrationSource. - * - * @param {SyncConfig} config - * @returns {KnexMigrationSource} + * Synchronize context parameters for the current database connection. */ -function getMigrationConfig(config: SyncConfig): Knex.MigratorConfig { - const { basePath, migration } = config; - const migrationPath = path.join(basePath, migration.directory); - - return { - directory: migrationPath, - tableName: migration.tableName, - migrationSource: new KnexMigrationSource(migrationPath) - }; +interface MigrationCommandContext { + config: SyncConfig; + connectionId: string; + params: SyncParams; + knexMigrationConfig: Knex.MigratorConfig; } /** - * Run migrations on a target database connection (transaction). + * Run migrations on a target database connection / transaction. * * @param {(Knex | Knex.Transaction)} trx - * @param {SyncConfig} config + * @param {SyncConfig} context * @returns {Promise} */ -export function migrateLatest(trx: Knex | Knex.Transaction, config: SyncConfig): Promise { - return trx.migrate.latest(getMigrationConfig(config)); +export async function migrateLatest(trx: Knex | Knex.Transaction, context: MigrationCommandContext): Promise { + const { connectionId, knexMigrationConfig } = context; + const log = dbLogger(context.connectionId); + const result: SyncResult = { connectionId, success: false }; + + const timeStart = process.hrtime(); + + try { + log('BEGIN: migrate.latest'); + const migrationResult = await trx.migrate.latest(knexMigrationConfig); + + log('END: migrate.latest'); + log('Migration Result:\n%O', migrationResult); + + result.success = true; + } catch (e) { + log(`Error caught for connection ${connectionId}:`, e); + result.error = e; + } + + const timeElapsed = getElapsedTime(timeStart); + + log(`Execution completed in ${timeElapsed} s`); + + // If it's a CLI environment, invoke the handler. + if (isCLI()) { + const handler = result.success ? context.params.onSuccess : context.params.onFailed; + const execContext: ExecutionContext = { + connectionId, + timeElapsed, + success: result.success + }; + + await handler(execContext); + } + + return result; } -/** - * Rollback migrations on a target database connection (transaction). - * - * @param {(Knex | Knex.Transaction)} trx - * @param {SyncConfig} config - * @returns {Promise} - */ -export function rollback(trx: Knex | Knex.Transaction, config: SyncConfig): Promise { - return trx.migrate.rollback(getMigrationConfig(config)); +export async function migrateRollback(trx: Knex | Knex.Transaction, context: MigrationCommandContext): Promise { + const { connectionId, knexMigrationConfig } = context; + const log = dbLogger(context.connectionId); + const result: SyncResult = { connectionId, success: false }; + + const timeStart = process.hrtime(); + + try { + log('BEGIN: migrate.rollback'); + const migrationResult = await trx.migrate.rollback(knexMigrationConfig); + + log('END: migrate.rollback'); + log('Migration Result:\n%O', migrationResult); + + result.success = true; + } catch (e) { + log(`Error caught for connection ${connectionId}:`, e); + result.error = e; + } + + const timeElapsed = getElapsedTime(timeStart); + + log(`Execution completed in ${timeElapsed} s`); + + // If it's a CLI environment, invoke the handler. + if (isCLI()) { + const handler = result.success ? context.params.onSuccess : context.params.onFailed; + const execContext: ExecutionContext = { + connectionId, + timeElapsed, + success: result.success + }; + + await handler(execContext); + } + + return result; } -/** - * List migrations on a target database connection (transaction). - * - * @param {(Knex | Knex.Transaction)} trx - * @param {SyncConfig} config - * @returns {Promise} - */ -export function list(trx: Knex | Knex.Transaction, config: SyncConfig): Promise { - return trx.migrate.list(getMigrationConfig(config)); +export async function migrateList(trx: Knex | Knex.Transaction, context: MigrationCommandContext): Promise { + const { connectionId, knexMigrationConfig } = context; + const log = dbLogger(context.connectionId); + const result: SyncResult = { connectionId, success: false }; + + const timeStart = process.hrtime(); + + try { + log('BEGIN: migrate.list'); + const migrationResult = await trx.migrate.list(knexMigrationConfig); + + log('END: migrate.list'); + log('Migration Result:\n%O', migrationResult); + + result.success = true; + } catch (e) { + log(`Error caught for connection ${connectionId}:`, e); + result.error = e; + } + + const timeElapsed = getElapsedTime(timeStart); + + log(`Execution completed in ${timeElapsed} s`); + + // If it's a CLI environment, invoke the handler. + if (isCLI()) { + const handler = result.success ? context.params.onSuccess : context.params.onFailed; + const execContext: ExecutionContext = { + connectionId, + timeElapsed, + success: result.success + }; + + await handler(execContext); + } + + return result; } From 18d54a05a55a3806ba6580a55711e453a766e710 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Mon, 23 Mar 2020 15:25:38 +0545 Subject: [PATCH 31/80] Update imports --- src/cli.ts | 2 +- src/service/migrator.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index e7dcb51b..3174bd75 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -2,10 +2,10 @@ import * as path from 'path'; import * as fs from './util/fs'; import { printLine } from './util/io'; +import { getElapsedTime } from './util/ts'; import SyncDbOptions from './domain/SyncDbOptions'; import { CONNECTIONS_FILENAME } from './constants'; import { resolveConnectionsFromEnv, loadConfig, resolveConnections } from './config'; -import { getElapsedTime } from './util/misc'; import SyncParams from './domain/SyncParams'; import { log } from './util/logger'; diff --git a/src/service/migrator.ts b/src/service/migrator.ts index 3fd9f804..cb45757a 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -3,12 +3,12 @@ import * as path from 'path'; import { glob, exists } from '../util/fs'; import { resolveFile } from './sqlRunner'; +import { getElapsedTime } from '../util/ts'; // import SqlMigrationContext, { MigrationContext } from '../migration/SqlMigrationContext'; import SqlMigrationEntry from '../domain/SqlMigrationEntry'; import SyncResult from '../domain/SyncResult'; // import SyncContext from '../domain/SyncContext'; import { dbLogger } from '../util/logger'; -import { getElapsedTime } from '../util/misc'; import { isCLI } from '../config'; import ExecutionContext from '../domain/ExecutionContext'; // import KnexMigrationSource from '../migration/KnexMigrationSource'; From 5ef7596f8cfe0f7fe351a8566cf1ccd359368a16 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 13:23:26 +0545 Subject: [PATCH 32/80] Move migration flags to commands --- src/cli.ts | 102 +------------------------------ src/commands/index.ts | 3 + src/commands/migrate-latest.ts | 75 +++++++++++++++++++++++ src/commands/migrate-list.ts | 72 ++++++++++++++++++++++ src/commands/migrate-rollback.ts | 74 ++++++++++++++++++++++ src/commands/synchronize.ts | 8 +-- src/domain/SyncDbOptions.ts | 3 - 7 files changed, 226 insertions(+), 111 deletions(-) create mode 100644 src/commands/index.ts create mode 100644 src/commands/migrate-latest.ts create mode 100644 src/commands/migrate-list.ts create mode 100644 src/commands/migrate-rollback.ts diff --git a/src/cli.ts b/src/cli.ts index 3174bd75..d413f086 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -2,12 +2,10 @@ import * as path from 'path'; import * as fs from './util/fs'; import { printLine } from './util/io'; -import { getElapsedTime } from './util/ts'; import SyncDbOptions from './domain/SyncDbOptions'; import { CONNECTIONS_FILENAME } from './constants'; -import { resolveConnectionsFromEnv, loadConfig, resolveConnections } from './config'; +import { resolveConnectionsFromEnv } from './config'; import SyncParams from './domain/SyncParams'; -import { log } from './util/logger'; /** * Generates connections.sync-db.json file. @@ -25,95 +23,6 @@ async function generateConnection(): Promise { await printLine(`Generated file: ${CONNECTIONS_FILENAME}\n`); } -async function migrate(params: SyncParams): Promise { - const config = await loadConfig(); - const connections = await resolveConnections(); - const { migrateLatest } = await import('./api'); - const timeStart = process.hrtime(); - - await printLine('Synchronizing...\n'); - - const results = await migrateLatest(config, connections, params); - - log('Results:', results); - console.log('Results', results); // tslint:disable-line - - const successfulCount = results.filter(item => item.success).length; - - if (successfulCount > 0) { - // Display output. - await printLine( - `Migration complete for ${successfulCount} / ${results.length} connection(s). ` + - `(${getElapsedTime(timeStart)}s)` - ); - } - - // If all completed successfully, exit gracefully. - if (results.length === successfulCount) { - return process.exit(0); - } - - throw new Error(`Synchronization failed for some connections.`); -} - -async function rollback(params: SyncParams): Promise { - const config = await loadConfig(); - const connections = await resolveConnections(); - const { migrateRollback } = await import('./api'); - const timeStart = process.hrtime(); - - await printLine('Rolling back...\n'); - - const results = await migrateRollback(config, connections, params); - - log('Results:', results); - console.log('Results', results); // tslint:disable-line - - const successfulCount = results.filter(item => item.success).length; - - if (successfulCount > 0) { - // Display output. - await printLine( - `Rollback complete for ${successfulCount} / ${results.length} connection(s). ` + `(${getElapsedTime(timeStart)}s)` - ); - } - - // If all completed successfully, exit gracefully. - if (results.length === successfulCount) { - return process.exit(0); - } - - throw new Error(`Synchronization failed for some connections.`); -} - -async function list(params: SyncParams): Promise { - const config = await loadConfig(); - const connections = await resolveConnections(); - const { migrateList } = await import('./api'); - const timeStart = process.hrtime(); - - const results = await migrateList(config, connections, params); - - log('Results:', results); - console.log('Results', results); // tslint:disable-line - - const successfulCount = results.filter(item => item.success).length; - - if (successfulCount > 0) { - // Display output. - await printLine( - `List complete for ${successfulCount} / ${results.length} connection(s). ` + `(${getElapsedTime(timeStart)}s)` - ); - } - - // If all completed successfully, exit gracefully. - if (results.length === successfulCount) { - return process.exit(0); - } - - throw new Error(`Synchronization failed for some connections.`); -} - /** * Handle the provided CLI flags. * @@ -124,14 +33,5 @@ export async function handleFlags(flags: SyncDbOptions, params: SyncParams): Pro if (flags['generate-connections']) { await generateConnection(); process.exit(0); - } else if (flags['migrate']) { - await migrate(params); - process.exit(0); - } else if (flags['rollback']) { - await rollback(params); - process.exit(0); - } else if (flags['list']) { - await list(params); - process.exit(0); } } diff --git a/src/commands/index.ts b/src/commands/index.ts new file mode 100644 index 00000000..78a0c754 --- /dev/null +++ b/src/commands/index.ts @@ -0,0 +1,3 @@ +import MigrateLatest from './migrate-latest'; + +export default MigrateLatest; diff --git a/src/commands/migrate-latest.ts b/src/commands/migrate-latest.ts new file mode 100644 index 00000000..f6b8a425 --- /dev/null +++ b/src/commands/migrate-latest.ts @@ -0,0 +1,75 @@ +import { Command } from '@oclif/command'; + +import { printLine } from '../util/io'; +import { loadConfig, resolveConnections } from '..'; +import { log } from '../util/logger'; +import { getElapsedTime } from '../util/ts'; +import SyncParams from '../domain/SyncParams'; +import ExecutionContext from '../domain/ExecutionContext'; + +/** + * Migration command handler. + */ +class MigrateLatest extends Command { + static description = 'Run the migrations up to the latest changes.'; + + /** + * Default CLI options for running synchronize. + * + * @param {*} userParams + * @returns {SyncParams} + */ + getSyncParams(userParams: any): SyncParams { + return { + ...userParams, + // Individual success handler + onSuccess: (context: ExecutionContext) => + printLine(` [✓] ${context.connectionId} - Successful (${context.timeElapsed}s)`), + + // Individual error handler + onFailed: (context: ExecutionContext) => + printLine(` [✖] ${context.connectionId} - Failed (${context.timeElapsed}s)`) + }; + } + + /** + * CLI command execution handler. + * + * @returns {Promise} + */ + async run(): Promise { + const { flags: parsedFlags } = this.parse(MigrateLatest); + const params = this.getSyncParams({ ...parsedFlags }); + + const config = await loadConfig(); + const connections = await resolveConnections(); + const { migrateLatest } = await import('../api'); + const timeStart = process.hrtime(); + + await printLine('Running Migrations\n'); + + const results = await migrateLatest(config, connections, params); + + log('Results:', results); + console.log('Results', results); // tslint:disable-line + + const successfulCount = results.filter(item => item.success).length; + + if (successfulCount > 0) { + // Display output. + await printLine( + `Migration complete for ${successfulCount} / ${results.length} connection(s). ` + + `(${getElapsedTime(timeStart)}s)` + ); + } + + // If all completed successfully, exit gracefully. + if (results.length === successfulCount) { + return process.exit(0); + } + + throw new Error(`Migration failed for some connections.`); + } +} + +export default MigrateLatest; diff --git a/src/commands/migrate-list.ts b/src/commands/migrate-list.ts new file mode 100644 index 00000000..4aa67761 --- /dev/null +++ b/src/commands/migrate-list.ts @@ -0,0 +1,72 @@ +import { Command } from '@oclif/command'; + +import { printLine } from '../util/io'; +import { loadConfig, resolveConnections } from '..'; +import { log } from '../util/logger'; +import { getElapsedTime } from '../util/ts'; +import SyncParams from '../domain/SyncParams'; +import ExecutionContext from '../domain/ExecutionContext'; + +/** + * Migration command handler. + */ +class MigrateList extends Command { + static description = 'List migrations.'; + + /** + * Default CLI options for running synchronize. + * + * @param {*} userParams + * @returns {SyncParams} + */ + getSyncParams(userParams: any): SyncParams { + return { + ...userParams, + // Individual success handler + onSuccess: (context: ExecutionContext) => + printLine(` [✓] ${context.connectionId} - Successful (${context.timeElapsed}s)`), + + // Individual error handler + onFailed: (context: ExecutionContext) => + printLine(` [✖] ${context.connectionId} - Failed (${context.timeElapsed}s)`) + }; + } + + /** + * CLI command execution handler. + * + * @returns {Promise} + */ + async run(): Promise { + const { flags: parsedFlags } = this.parse(MigrateList); + const params = this.getSyncParams({ ...parsedFlags }); + + const config = await loadConfig(); + const connections = await resolveConnections(); + const { migrateList } = await import('../api'); + const timeStart = process.hrtime(); + + const results = await migrateList(config, connections, params); + + log('Results:', results); + console.log('Results', results); // tslint:disable-line + + const successfulCount = results.filter(item => item.success).length; + + if (successfulCount > 0) { + // Display output. + await printLine( + `List complete for ${successfulCount} / ${results.length} connection(s). ` + `(${getElapsedTime(timeStart)}s)` + ); + } + + // If all completed successfully, exit gracefully. + if (results.length === successfulCount) { + return process.exit(0); + } + + throw new Error(`Failed to get the list for some connections.`); + } +} + +export default MigrateList; diff --git a/src/commands/migrate-rollback.ts b/src/commands/migrate-rollback.ts new file mode 100644 index 00000000..25fd2496 --- /dev/null +++ b/src/commands/migrate-rollback.ts @@ -0,0 +1,74 @@ +import { Command } from '@oclif/command'; + +import { printLine } from '../util/io'; +import { loadConfig, resolveConnections } from '..'; +import { log } from '../util/logger'; +import { getElapsedTime } from '../util/ts'; +import SyncParams from '../domain/SyncParams'; +import ExecutionContext from '../domain/ExecutionContext'; + +/** + * Migration command handler. + */ +class MigrateRollback extends Command { + static description = 'Rollback migrations up to the last run batch.'; + + /** + * Default CLI options for running synchronize. + * + * @param {*} userParams + * @returns {SyncParams} + */ + getSyncParams(userParams: any): SyncParams { + return { + ...userParams, + // Individual success handler + onSuccess: (context: ExecutionContext) => + printLine(` [✓] ${context.connectionId} - Successful (${context.timeElapsed}s)`), + + // Individual error handler + onFailed: (context: ExecutionContext) => + printLine(` [✖] ${context.connectionId} - Failed (${context.timeElapsed}s)`) + }; + } + + /** + * CLI command execution handler. + * + * @returns {Promise} + */ + async run(): Promise { + const { flags: parsedFlags } = this.parse(MigrateRollback); + const params = this.getSyncParams({ ...parsedFlags }); + const config = await loadConfig(); + const connections = await resolveConnections(); + const { migrateRollback } = await import('../api'); + const timeStart = process.hrtime(); + + await printLine('Rolling back migrations\n'); + + const results = await migrateRollback(config, connections, params); + + log('Results:', results); + console.log('Results', results); // tslint:disable-line + + const successfulCount = results.filter(item => item.success).length; + + if (successfulCount > 0) { + // Display output. + await printLine( + `Rollback complete for ${successfulCount} / ${results.length} connection(s). ` + + `(${getElapsedTime(timeStart)}s)` + ); + } + + // If all completed successfully, exit gracefully. + if (results.length === successfulCount) { + return process.exit(0); + } + + throw new Error(`Rollback failed for some connections.`); + } +} + +export default MigrateRollback; diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index d45c4752..869619b3 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -22,13 +22,7 @@ class Synchronize extends Command { version: flags.version({ char: 'v', description: 'Print version', name: 'sync-db' }), help: flags.help({ char: 'h', description: 'Print help information' }), force: flags.boolean({ char: 'f', description: 'Force synchronization' }), - 'generate-connections': flags.boolean({ char: 'c', description: 'Generate connections' }), - - // Note: These are temporary workaround using args. - // TODO: Move these into separate commands (multi-commands) - migrate: flags.boolean({ description: 'Run Migrations' }), - rollback: flags.boolean({ description: 'Run Rollback' }), - list: flags.boolean({ description: 'List migrations' }) + 'generate-connections': flags.boolean({ char: 'c', description: 'Generate connections' }) }; /** diff --git a/src/domain/SyncDbOptions.ts b/src/domain/SyncDbOptions.ts index de8484a3..4104dbf6 100644 --- a/src/domain/SyncDbOptions.ts +++ b/src/domain/SyncDbOptions.ts @@ -6,9 +6,6 @@ interface SyncDbOptions { help: void; force: boolean; 'generate-connections': boolean; - migrate: boolean; - rollback: boolean; - list: boolean; } export default SyncDbOptions; From 4a10e6cd1d04c8e8f6d99597bf79ef59891ef2c0 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 13:25:04 +0545 Subject: [PATCH 33/80] Remove commands/index file --- src/commands/index.ts | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 src/commands/index.ts diff --git a/src/commands/index.ts b/src/commands/index.ts deleted file mode 100644 index 78a0c754..00000000 --- a/src/commands/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -import MigrateLatest from './migrate-latest'; - -export default MigrateLatest; From 6ecfcd14554a8b86cb29e7398e73ba47cbe64c26 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 14:27:22 +0545 Subject: [PATCH 34/80] Simplify promise.runSequentially function --- src/util/promise.ts | 20 ++++---------------- test/util/promise.test.ts | 28 ++-------------------------- 2 files changed, 6 insertions(+), 42 deletions(-) diff --git a/src/util/promise.ts b/src/util/promise.ts index 7d147c1b..1774821a 100644 --- a/src/util/promise.ts +++ b/src/util/promise.ts @@ -14,14 +14,10 @@ export const timeout = promisify(setTimeout); * Run each of the promise sequentially and return their results in the same order. * * @param {Promiser[]} promisers - * @param {boolean} [failCascade=true] - * @returns {(Promise<(T | Error)[]>)} + * @returns {Promise} */ -export async function runSequentially( - promisers: Promiser[], - failCascade: boolean = true -): Promise<(T | Error)[]> { - const result: (T | Error)[] = []; +export async function runSequentially(promisers: Promiser[]): Promise { + const result: T[] = []; for (const promiser of promisers) { try { @@ -29,15 +25,7 @@ export async function runSequentially( result.push(value); } catch (err) { - // If failCascade = true, - // any error (promise rejection) will be cascaded thus halting the process. - if (failCascade) { - throw err; - } - - // If failCascade = false, - // the failed promise will be resolved with the rejected error as a value. - result.push(err instanceof Error ? err : new Error(err)); + throw err; } } diff --git a/test/util/promise.test.ts b/test/util/promise.test.ts index 6776b978..ef7e1cd1 100644 --- a/test/util/promise.test.ts +++ b/test/util/promise.test.ts @@ -50,7 +50,7 @@ describe('UTIL: promise', () => { expect(result).to.deep.equal(['one', 'two', 'three', 'four', 'five']); }); - it('should throw an error (reject) if any promise fails, when failCascade = true. (default)', async () => { + it('should throw an error (reject) if any promise fails.', async () => { const promisers = [ () => Promise.resolve('one'), () => Promise.reject('An error occurred.'), @@ -61,31 +61,7 @@ describe('UTIL: promise', () => { } ]; - expect(runSequentially(promisers, true)).to.be.eventually.rejectedWith('An error occurred.'); - }); - - it('should run and resolve all the promises when failCascade = false.', async () => { - const promisers = [ - () => Promise.resolve('one'), - () => Promise.reject('An error occurred.'), - () => Promise.resolve('three'), - () => Promise.reject(new Error('A second error occurred.')), - async () => { - throw new Error('Another error occurred.'); - } - ]; - - const result = await runSequentially(promisers, false); - - expect(result).to.have.lengthOf(5); - expect(result[0]).to.equal('one'); - expect(result[1]).to.instanceOf(Error); - expect(result[1].toString()).to.equal('Error: An error occurred.'); - expect(result[2]).to.equal('three'); - expect(result[3]).to.instanceOf(Error); - expect(result[3].toString()).to.equal('Error: A second error occurred.'); - expect(result[4]).to.instanceOf(Error); - expect(result[4].toString()).to.equal('Error: Another error occurred.'); + expect(runSequentially(promisers)).to.be.eventually.rejectedWith('An error occurred.'); }); }); }); From 0bf07f650c57f5af83a03521e61e8e3413aa5696 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 14:28:24 +0545 Subject: [PATCH 35/80] Simplify executePromisses() return type --- src/service/execution.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/execution.ts b/src/service/execution.ts index 7724d1ea..c9e8424e 100644 --- a/src/service/execution.ts +++ b/src/service/execution.ts @@ -9,7 +9,7 @@ import { Promiser, runSequentially } from '../util/promise'; * @param {SyncConfig} config * @returns {Promise} */ -export function executeProcesses(processes: Promiser[], config: SyncConfig): Promise<(T | Error)[]> { +export function executeProcesses(processes: Promiser[], config: SyncConfig): Promise { log(`Execution Strategy: ${config.execution}`); switch (config.execution) { From 3231c29aa6b3700915da267592b0e317372eaf6d Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 16:09:47 +0545 Subject: [PATCH 36/80] Make migrate-list command work --- src/api.ts | 5 ++-- src/service/migrator.ts | 55 +++++++++++++++++++++++++++++++---------- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/src/api.ts b/src/api.ts index 8ee602df..8b907ceb 100644 --- a/src/api.ts +++ b/src/api.ts @@ -19,6 +19,7 @@ import { executeProcesses } from './service/execution'; import * as migratorService from './service/migrator'; import SqlMigrationContext from './migration/SqlMigrationContext'; import KnexMigrationSource from './migration/KnexMigrationSource'; +import { MigrationListResult, MigrationListParams } from './service/migrator'; /** * Synchronize all the configured database connections. @@ -136,8 +137,8 @@ export async function migrateRollback( export async function migrateList( config: SyncConfig, conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, - options?: SyncParams -): Promise { + options?: MigrationListParams +): Promise { log('Starting to migrate.'); const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); diff --git a/src/service/migrator.ts b/src/service/migrator.ts index cb45757a..b90279b8 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -191,40 +191,69 @@ export async function migrateRollback(trx: Knex | Knex.Transaction, context: Mig return result; } -export async function migrateList(trx: Knex | Knex.Transaction, context: MigrationCommandContext): Promise { +export interface MigrationListResult { + connectionId: string; + success: boolean; + timeElapsed: number; + data: any; + error?: any; +} + +/** + * Synchronize parameters. + */ +export interface MigrationListParams { + onSuccess: (context: MigrationListResult) => Promise; + onFailed: (context: MigrationListResult) => Promise; +} + +export interface MCommandContext { + config: SyncConfig; + connectionId: string; + params: T; + knexMigrationConfig: Knex.MigratorConfig; +} + +export async function migrateList( + trx: Knex | Knex.Transaction, + context: MCommandContext +): Promise { const { connectionId, knexMigrationConfig } = context; const log = dbLogger(context.connectionId); - const result: SyncResult = { connectionId, success: false }; + + let error; + let data; const timeStart = process.hrtime(); try { log('BEGIN: migrate.list'); - const migrationResult = await trx.migrate.list(knexMigrationConfig); + data = await trx.migrate.list(knexMigrationConfig); log('END: migrate.list'); - log('Migration Result:\n%O', migrationResult); - - result.success = true; + log('Migration Result:\n%O', data); } catch (e) { log(`Error caught for connection ${connectionId}:`, e); - result.error = e; + error = e; } const timeElapsed = getElapsedTime(timeStart); log(`Execution completed in ${timeElapsed} s`); + const result: MigrationListResult = { + connectionId, + error, + data, + timeElapsed, + success: !error + }; + // If it's a CLI environment, invoke the handler. if (isCLI()) { const handler = result.success ? context.params.onSuccess : context.params.onFailed; - const execContext: ExecutionContext = { - connectionId, - timeElapsed, - success: result.success - }; - await handler(execContext); + await handler(result); } return result; From 797887c61ac06f067b4c5103f62089b096efabd9 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 16:10:09 +0545 Subject: [PATCH 37/80] Build UX for the migrate-list command --- src/commands/migrate-list.ts | 75 ++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/src/commands/migrate-list.ts b/src/commands/migrate-list.ts index 4aa67761..aca77820 100644 --- a/src/commands/migrate-list.ts +++ b/src/commands/migrate-list.ts @@ -3,9 +3,7 @@ import { Command } from '@oclif/command'; import { printLine } from '../util/io'; import { loadConfig, resolveConnections } from '..'; import { log } from '../util/logger'; -import { getElapsedTime } from '../util/ts'; -import SyncParams from '../domain/SyncParams'; -import ExecutionContext from '../domain/ExecutionContext'; +import { MigrationListParams, MigrationListResult } from '../service/migrator'; /** * Migration command handler. @@ -13,22 +11,43 @@ import ExecutionContext from '../domain/ExecutionContext'; class MigrateList extends Command { static description = 'List migrations.'; - /** - * Default CLI options for running synchronize. - * - * @param {*} userParams - * @returns {SyncParams} - */ - getSyncParams(userParams: any): SyncParams { + getParams(): MigrationListParams { return { - ...userParams, - // Individual success handler - onSuccess: (context: ExecutionContext) => - printLine(` [✓] ${context.connectionId} - Successful (${context.timeElapsed}s)`), - - // Individual error handler - onFailed: (context: ExecutionContext) => - printLine(` [✖] ${context.connectionId} - Failed (${context.timeElapsed}s)`) + onSuccess: async (result: MigrationListResult) => { + // ➜ + await printLine(` ▸ ${result.connectionId}`); + + log(result.data); + + const [list1, list2] = result.data; + const ranCount = list1.length; + const remainingCount = list2.length; + + // Completed migrations. + for (const item of list1) { + await printLine(` • ${item}`); + } + + // Remaining Migrations + for (const item of list2) { + await printLine(` - ${item}`); + } + + if (ranCount === 0 && remainingCount === 0) { + await printLine(' No migrations.'); + } else if (remainingCount === 0) { + await printLine(`\n All up to date.`); + } else { + await printLine(`\n ${list2.length} migrations yet to be run.`); + } + + await printLine(); + }, + onFailed: async (result: MigrationListResult) => { + printLine(` ▸ ${result.connectionId} - Failed (${result.timeElapsed}s)`); + + await printLine(` ${result.error}\n`); + } }; } @@ -38,34 +57,22 @@ class MigrateList extends Command { * @returns {Promise} */ async run(): Promise { - const { flags: parsedFlags } = this.parse(MigrateList); - const params = this.getSyncParams({ ...parsedFlags }); + const params = this.getParams(); const config = await loadConfig(); const connections = await resolveConnections(); const { migrateList } = await import('../api'); - const timeStart = process.hrtime(); const results = await migrateList(config, connections, params); log('Results:', results); - console.log('Results', results); // tslint:disable-line - - const successfulCount = results.filter(item => item.success).length; - if (successfulCount > 0) { - // Display output. - await printLine( - `List complete for ${successfulCount} / ${results.length} connection(s). ` + `(${getElapsedTime(timeStart)}s)` - ); - } + const failedCount = results.filter(({ success }) => !success).length; // If all completed successfully, exit gracefully. - if (results.length === successfulCount) { - return process.exit(0); - } + const exitCode = failedCount === 0 ? 0 : -1; - throw new Error(`Failed to get the list for some connections.`); + process.exit(exitCode); } } From 8bee2343357db352938aa21a455bf1649a2188e6 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 16:15:33 +0545 Subject: [PATCH 38/80] Add chalk --- package.json | 1 + yarn.lock | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 5512e857..4fb70c7b 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "@oclif/command": "^1", "@oclif/config": "^1", "@oclif/plugin-help": "^2", + "chalk": "^4.0.0", "debug": "^4.1.1", "globby": "^10.0.2", "knex": "^0.20.11", diff --git a/yarn.lock b/yarn.lock index da2e3457..921408e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -468,7 +468,7 @@ ansi-styles@^3.2.0, ansi-styles@^3.2.1: dependencies: color-convert "^1.9.0" -ansi-styles@^4.0.0: +ansi-styles@^4.0.0, ansi-styles@^4.1.0: version "4.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.2.1.tgz#90ae75c424d008d2624c5bf29ead3177ebfcf359" integrity sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA== @@ -677,6 +677,14 @@ chalk@^2.0.0, chalk@^2.3.0, chalk@^2.4.1, chalk@^2.4.2: escape-string-regexp "^1.0.5" supports-color "^5.3.0" +chalk@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.0.0.tgz#6e98081ed2d17faab615eb52ac66ec1fe6209e72" + integrity sha512-N9oWFcegS0sFr9oh1oz2d7Npos6vNoWW9HvtCg5N1KRFpUhaAhvTv5Y58g880fZaEYSNm3qDz8SU1UrGvp+n7A== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + check-error@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/check-error/-/check-error-1.0.2.tgz#574d312edd88bb5dd8912e9286dd6c0aed4aac82" From d672e3fc3ce3e9ba6c10cf399edb48c45b509b88 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 16:31:06 +0545 Subject: [PATCH 39/80] Print colored info & errors --- src/util/io.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/util/io.ts b/src/util/io.ts index a06151f8..82fc200a 100644 --- a/src/util/io.ts +++ b/src/util/io.ts @@ -1,3 +1,5 @@ +import { green, red } from 'chalk'; + /** * Prints a line into the console (stdout). * @@ -8,6 +10,16 @@ export function printLine(message: string = ''): Promise { return print(message.toString() + '\n'); } +/** + * Print an info line on the console (in green color). + * + * @param {string} [message=''] + * @returns {Promise} + */ +export function printInfo(message: string = ''): Promise { + return printLine(green(message)); +} + /** * Prints a message into the console (stdout). * @@ -26,5 +38,5 @@ export function print(message: string): Promise { * @returns {Promise} */ export function printError(error: Error): Promise { - return new Promise(resolve => process.stderr.write((error.stack || error).toString() + '\n', resolve)); + return new Promise(resolve => process.stderr.write(red((error.stack || error).toString()) + '\n', resolve)); } From 9407d5266dadabe1b1cb4701d5b0d498feb1f4c2 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 16:37:47 +0545 Subject: [PATCH 40/80] Support parametric modifiers on util/io --- src/util/io.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/util/io.ts b/src/util/io.ts index 82fc200a..25b03507 100644 --- a/src/util/io.ts +++ b/src/util/io.ts @@ -1,13 +1,16 @@ import { green, red } from 'chalk'; +export type ModifierFunc = (message: string) => string; + /** * Prints a line into the console (stdout). * * @param {string} [message=''] + * @param {ModifierFunc} [modifier=(str: string) => str] * @returns {Promise} */ -export function printLine(message: string = ''): Promise { - return print(message.toString() + '\n'); +export function printLine(message: string = '', modifier: ModifierFunc = (str: string) => str): Promise { + return print(modifier(message.toString()) + '\n'); } /** @@ -17,7 +20,7 @@ export function printLine(message: string = ''): Promise { * @returns {Promise} */ export function printInfo(message: string = ''): Promise { - return printLine(green(message)); + return printLine(message, green); } /** @@ -34,9 +37,12 @@ export function print(message: string): Promise { * Prints an error message with stack trace available * into the stderr. * - * @param {Error} error + * @param {any} error + * @param {boolean} [stacktrace=true] * @returns {Promise} */ -export function printError(error: Error): Promise { - return new Promise(resolve => process.stderr.write(red((error.stack || error).toString()) + '\n', resolve)); +export function printError(error: any, stacktrace: boolean = true): Promise { + const errorStr = stacktrace ? (error.stack || error).toString() : error.toString(); + + return new Promise(resolve => process.stderr.write(red(errorStr) + '\n', resolve)); } From cca64c809021a2c9c85aea2308bd3a48018c43a8 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 16:48:31 +0545 Subject: [PATCH 41/80] Improve UX for migrate-list command --- src/commands/migrate-list.ts | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/commands/migrate-list.ts b/src/commands/migrate-list.ts index aca77820..662d2e19 100644 --- a/src/commands/migrate-list.ts +++ b/src/commands/migrate-list.ts @@ -1,6 +1,7 @@ import { Command } from '@oclif/command'; +import { bold, grey, red, cyan, yellow } from 'chalk'; -import { printLine } from '../util/io'; +import { printLine, printError } from '../util/io'; import { loadConfig, resolveConnections } from '..'; import { log } from '../util/logger'; import { MigrationListParams, MigrationListResult } from '../service/migrator'; @@ -14,8 +15,7 @@ class MigrateList extends Command { getParams(): MigrationListParams { return { onSuccess: async (result: MigrationListResult) => { - // ➜ - await printLine(` ▸ ${result.connectionId}`); + await printLine(bold(` ▸ ${result.connectionId}`)); log(result.data); @@ -25,28 +25,26 @@ class MigrateList extends Command { // Completed migrations. for (const item of list1) { - await printLine(` • ${item}`); + await printLine(cyan(` • ${item}`)); } // Remaining Migrations for (const item of list2) { - await printLine(` - ${item}`); + await printLine(grey(` - ${item}`)); } if (ranCount === 0 && remainingCount === 0) { - await printLine(' No migrations.'); - } else if (remainingCount === 0) { - await printLine(`\n All up to date.`); - } else { - await printLine(`\n ${list2.length} migrations yet to be run.`); + await printLine(yellow(' No migrations.')); + } else if (remainingCount > 0) { + await printLine(yellow(`\n ${list2.length} migrations yet to be run.`)); } await printLine(); }, onFailed: async (result: MigrationListResult) => { - printLine(` ▸ ${result.connectionId} - Failed (${result.timeElapsed}s)`); + printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); - await printLine(` ${result.error}\n`); + await printError(` ${result.error}\n`); } }; } From 8910ba00bd02fe863010e22ee7aae5a3007a5a45 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 16:58:43 +0545 Subject: [PATCH 42/80] Clean up debug logs --- src/api.ts | 5 ++--- src/commands/migrate-list.ts | 6 ------ src/service/migrator.ts | 2 +- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/api.ts b/src/api.ts index 8b907ceb..f04cb370 100644 --- a/src/api.ts +++ b/src/api.ts @@ -139,7 +139,6 @@ export async function migrateList( conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, options?: MigrationListParams ): Promise { - log('Starting to migrate.'); const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); @@ -148,7 +147,7 @@ export async function migrateList( const migrations = await migratorService.resolveSqlMigrations(migrationPath); log('Migration Path:', migrationPath); - log('Available migrations:'); + log('Available migration sources:'); log('%O', migrations); // TODO: We'll need to support different types of migrations eg both sql & js @@ -169,7 +168,7 @@ export async function migrateList( const results = await executeProcesses(processes, config); - log('Migrations completed.'); + log('Finished retrieving lists.'); return results; } diff --git a/src/commands/migrate-list.ts b/src/commands/migrate-list.ts index 662d2e19..ff18ffa4 100644 --- a/src/commands/migrate-list.ts +++ b/src/commands/migrate-list.ts @@ -17,8 +17,6 @@ class MigrateList extends Command { onSuccess: async (result: MigrationListResult) => { await printLine(bold(` ▸ ${result.connectionId}`)); - log(result.data); - const [list1, list2] = result.data; const ranCount = list1.length; const remainingCount = list2.length; @@ -63,11 +61,7 @@ class MigrateList extends Command { const results = await migrateList(config, connections, params); - log('Results:', results); - const failedCount = results.filter(({ success }) => !success).length; - - // If all completed successfully, exit gracefully. const exitCode = failedCount === 0 ? 0 : -1; process.exit(exitCode); diff --git a/src/service/migrator.ts b/src/service/migrator.ts index b90279b8..3f593d23 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -231,7 +231,7 @@ export async function migrateList( data = await trx.migrate.list(knexMigrationConfig); log('END: migrate.list'); - log('Migration Result:\n%O', data); + log('Result:\n%O', data); } catch (e) { log(`Error caught for connection ${connectionId}:`, e); error = e; From 207eae0e47f82cd831e709575a70d829c3000eb1 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 17:09:08 +0545 Subject: [PATCH 43/80] Print final error message --- src/commands/migrate-list.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/commands/migrate-list.ts b/src/commands/migrate-list.ts index ff18ffa4..6961b89b 100644 --- a/src/commands/migrate-list.ts +++ b/src/commands/migrate-list.ts @@ -3,7 +3,6 @@ import { bold, grey, red, cyan, yellow } from 'chalk'; import { printLine, printError } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { log } from '../util/logger'; import { MigrationListParams, MigrationListResult } from '../service/migrator'; /** @@ -62,9 +61,13 @@ class MigrateList extends Command { const results = await migrateList(config, connections, params); const failedCount = results.filter(({ success }) => !success).length; - const exitCode = failedCount === 0 ? 0 : -1; - process.exit(exitCode); + if (failedCount === 0) { + return process.exit(0); + } + + printError(`Error: Failed retrieving list for ${failedCount} connection(s).`); + process.exit(-1); } } From c35b08a4afa1217c137148d884180aa1331e0323 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 19:13:55 +0545 Subject: [PATCH 44/80] Print info in the migrate-list command --- src/commands/migrate-list.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/commands/migrate-list.ts b/src/commands/migrate-list.ts index 6961b89b..b6fa588a 100644 --- a/src/commands/migrate-list.ts +++ b/src/commands/migrate-list.ts @@ -3,7 +3,7 @@ import { bold, grey, red, cyan, yellow } from 'chalk'; import { printLine, printError } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { MigrationListParams, MigrationListResult } from '../service/migrator'; +import { MigrationListResult, MigrationCommandParams } from '../service/migrator'; /** * Migration command handler. @@ -11,7 +11,7 @@ import { MigrationListParams, MigrationListResult } from '../service/migrator'; class MigrateList extends Command { static description = 'List migrations.'; - getParams(): MigrationListParams { + getParams(): MigrationCommandParams { return { onSuccess: async (result: MigrationListResult) => { await printLine(bold(` ▸ ${result.connectionId}`)); @@ -34,6 +34,8 @@ class MigrateList extends Command { await printLine(yellow(' No migrations.')); } else if (remainingCount > 0) { await printLine(yellow(`\n ${list2.length} migrations yet to be run.`)); + } else if (remainingCount === 0) { + await printLine('\n All up to date.'); } await printLine(); From 3fe284db1f25f2449b38d21f6e169861e3476578 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 19:15:54 +0545 Subject: [PATCH 45/80] Make migration command work --- src/api.ts | 6 +- src/commands/migrate-latest.ts | 77 ++++++++++++------------ src/service/migrator.ts | 106 ++++++++++++++------------------- 3 files changed, 85 insertions(+), 104 deletions(-) diff --git a/src/api.ts b/src/api.ts index f04cb370..8e171fc9 100644 --- a/src/api.ts +++ b/src/api.ts @@ -19,7 +19,7 @@ import { executeProcesses } from './service/execution'; import * as migratorService from './service/migrator'; import SqlMigrationContext from './migration/SqlMigrationContext'; import KnexMigrationSource from './migration/KnexMigrationSource'; -import { MigrationListResult, MigrationListParams } from './service/migrator'; +import { MigrationListResult, MigrationCommandParams, MigrationLatestResult } from './service/migrator'; /** * Synchronize all the configured database connections. @@ -57,7 +57,7 @@ export async function synchronize( export async function migrateLatest( config: SyncConfig, conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, - options?: SyncParams + options?: MigrationCommandParams ): Promise { log('Starting to migrate.'); const connections = mapToConnectionReferences(conn); @@ -137,7 +137,7 @@ export async function migrateRollback( export async function migrateList( config: SyncConfig, conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, - options?: MigrationListParams + options?: MigrationCommandParams ): Promise { const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); diff --git a/src/commands/migrate-latest.ts b/src/commands/migrate-latest.ts index f6b8a425..0e7d80a5 100644 --- a/src/commands/migrate-latest.ts +++ b/src/commands/migrate-latest.ts @@ -1,11 +1,10 @@ import { Command } from '@oclif/command'; +import { bold, red, cyan } from 'chalk'; -import { printLine } from '../util/io'; +import { printLine, printError, printInfo } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { log } from '../util/logger'; -import { getElapsedTime } from '../util/ts'; -import SyncParams from '../domain/SyncParams'; -import ExecutionContext from '../domain/ExecutionContext'; +import { MigrationCommandParams, MigrationLatestResult } from '../service/migrator'; +import { dbLogger } from '../util/logger'; /** * Migration command handler. @@ -13,22 +12,35 @@ import ExecutionContext from '../domain/ExecutionContext'; class MigrateLatest extends Command { static description = 'Run the migrations up to the latest changes.'; - /** - * Default CLI options for running synchronize. - * - * @param {*} userParams - * @returns {SyncParams} - */ - getSyncParams(userParams: any): SyncParams { + getParams(): MigrationCommandParams { return { - ...userParams, - // Individual success handler - onSuccess: (context: ExecutionContext) => - printLine(` [✓] ${context.connectionId} - Successful (${context.timeElapsed}s)`), - - // Individual error handler - onFailed: (context: ExecutionContext) => - printLine(` [✖] ${context.connectionId} - Failed (${context.timeElapsed}s)`) + onSuccess: async (result: MigrationLatestResult) => { + const log = dbLogger(result.connectionId); + const [num, list] = result.data; + const alreadyUpToDate = num && list.length === 0; + + log('Up to date: ', alreadyUpToDate); + + await printLine(bold(` ▸ ${result.connectionId} - Successful`) + ` (${result.timeElapsed}s)`); + + if (alreadyUpToDate) { + await printInfo(' Already up to date.\n'); + + return; + } + + // Completed migrations. + for (const item of list) { + await printLine(cyan(` - ${item}`)); + } + + await printInfo(`\n Ran ${list.length} migrations.\n`); + }, + onFailed: async (result: MigrationLatestResult) => { + printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); + + await printError(` ${result.error}\n`); + } }; } @@ -38,37 +50,22 @@ class MigrateLatest extends Command { * @returns {Promise} */ async run(): Promise { - const { flags: parsedFlags } = this.parse(MigrateLatest); - const params = this.getSyncParams({ ...parsedFlags }); + const params = this.getParams(); const config = await loadConfig(); const connections = await resolveConnections(); const { migrateLatest } = await import('../api'); - const timeStart = process.hrtime(); - - await printLine('Running Migrations\n'); const results = await migrateLatest(config, connections, params); - log('Results:', results); - console.log('Results', results); // tslint:disable-line - - const successfulCount = results.filter(item => item.success).length; - - if (successfulCount > 0) { - // Display output. - await printLine( - `Migration complete for ${successfulCount} / ${results.length} connection(s). ` + - `(${getElapsedTime(timeStart)}s)` - ); - } + const failedCount = results.filter(({ success }) => !success).length; - // If all completed successfully, exit gracefully. - if (results.length === successfulCount) { + if (failedCount === 0) { return process.exit(0); } - throw new Error(`Migration failed for some connections.`); + printError(`Error: Migration failed for ${failedCount} connection(s).`); + process.exit(-1); } } diff --git a/src/service/migrator.ts b/src/service/migrator.ts index 3f593d23..4729b666 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -69,32 +69,33 @@ export async function resolveSqlMigrations(migrationPath: string): Promise { -// const log = dbLogger(context.connectionId); -// log('Getting migration config'); - -// const { basePath, migration } = context.config; -// const migrationPath = path.join(basePath, migration.directory); - -// const migrations = await resolveSqlMigrations(migrationPath); +export interface MigrationListResult { + connectionId: string; + success: boolean; + timeElapsed: number; + data: any; + error?: any; +} -// // TODO: Multiple migration context based on type of migration (sql, js, etc) -// const migrationContext = new SqlMigrationContext(context.connectionId, migrations); +export interface MigrationLatestResult { + connectionId: string; + success: boolean; + timeElapsed: number; + data: any; + error?: any; +} -// log('Available Migrations:', migrations); -// log('Table Name:', migration.tableName); +export interface MigrationCommandParams { + onSuccess: (context: T) => Promise; + onFailed: (context: T) => Promise; +} -// return { -// tableName: migration.tableName, -// migrationSource: new KnexMigrationSource(migrationContext) -// }; -// } +export interface MCommandContext { + config: SyncConfig; + connectionId: string; + params: T; + knexMigrationConfig: Knex.MigratorConfig; +} /** * Synchronize context parameters for the current database connection. @@ -110,43 +111,49 @@ interface MigrationCommandContext { * Run migrations on a target database connection / transaction. * * @param {(Knex | Knex.Transaction)} trx - * @param {SyncConfig} context + * @param {MCommandContext>} context * @returns {Promise} */ -export async function migrateLatest(trx: Knex | Knex.Transaction, context: MigrationCommandContext): Promise { +export async function migrateLatest( + trx: Knex | Knex.Transaction, + context: MCommandContext> +): Promise { const { connectionId, knexMigrationConfig } = context; const log = dbLogger(context.connectionId); - const result: SyncResult = { connectionId, success: false }; + + let error; + let data; const timeStart = process.hrtime(); try { log('BEGIN: migrate.latest'); - const migrationResult = await trx.migrate.latest(knexMigrationConfig); + data = await trx.migrate.latest(knexMigrationConfig); log('END: migrate.latest'); - log('Migration Result:\n%O', migrationResult); - - result.success = true; + log('Result:\n%O', data); } catch (e) { log(`Error caught for connection ${connectionId}:`, e); - result.error = e; + error = e; } const timeElapsed = getElapsedTime(timeStart); log(`Execution completed in ${timeElapsed} s`); + const result: MigrationListResult = { + connectionId, + error, + data, + timeElapsed, + success: !error + }; + // If it's a CLI environment, invoke the handler. if (isCLI()) { const handler = result.success ? context.params.onSuccess : context.params.onFailed; - const execContext: ExecutionContext = { - connectionId, - timeElapsed, - success: result.success - }; - await handler(execContext); + await handler(result); } return result; @@ -191,32 +198,9 @@ export async function migrateRollback(trx: Knex | Knex.Transaction, context: Mig return result; } -export interface MigrationListResult { - connectionId: string; - success: boolean; - timeElapsed: number; - data: any; - error?: any; -} - -/** - * Synchronize parameters. - */ -export interface MigrationListParams { - onSuccess: (context: MigrationListResult) => Promise; - onFailed: (context: MigrationListResult) => Promise; -} - -export interface MCommandContext { - config: SyncConfig; - connectionId: string; - params: T; - knexMigrationConfig: Knex.MigratorConfig; -} - export async function migrateList( trx: Knex | Knex.Transaction, - context: MCommandContext + context: MCommandContext> ): Promise { const { connectionId, knexMigrationConfig } = context; const log = dbLogger(context.connectionId); From f1ebfa2d4b1822c2f51ad8ac7d5b9e8b2e08d601 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 18 Apr 2020 19:52:50 +0545 Subject: [PATCH 46/80] Make migration rollback work --- src/api.ts | 9 +++- src/commands/migrate-rollback.ts | 78 ++++++++++++++++---------------- src/service/migrator.ts | 64 ++++++++++++++------------ 3 files changed, 79 insertions(+), 72 deletions(-) diff --git a/src/api.ts b/src/api.ts index 8e171fc9..71f3f10a 100644 --- a/src/api.ts +++ b/src/api.ts @@ -19,7 +19,12 @@ import { executeProcesses } from './service/execution'; import * as migratorService from './service/migrator'; import SqlMigrationContext from './migration/SqlMigrationContext'; import KnexMigrationSource from './migration/KnexMigrationSource'; -import { MigrationListResult, MigrationCommandParams, MigrationLatestResult } from './service/migrator'; +import { + MigrationListResult, + MigrationCommandParams, + MigrationLatestResult, + MigrationResult +} from './service/migrator'; /** * Synchronize all the configured database connections. @@ -97,7 +102,7 @@ export async function migrateLatest( export async function migrateRollback( config: SyncConfig, conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, - options?: SyncParams + options?: MigrationCommandParams ): Promise { log('Starting to migrate.'); const connections = mapToConnectionReferences(conn); diff --git a/src/commands/migrate-rollback.ts b/src/commands/migrate-rollback.ts index 25fd2496..cf21c620 100644 --- a/src/commands/migrate-rollback.ts +++ b/src/commands/migrate-rollback.ts @@ -1,11 +1,10 @@ import { Command } from '@oclif/command'; +import { bold, red, cyan } from 'chalk'; -import { printLine } from '../util/io'; +import { printLine, printError, printInfo } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { log } from '../util/logger'; -import { getElapsedTime } from '../util/ts'; -import SyncParams from '../domain/SyncParams'; -import ExecutionContext from '../domain/ExecutionContext'; +import { MigrationCommandParams, MigrationResult } from '../service/migrator'; +import { dbLogger } from '../util/logger'; /** * Migration command handler. @@ -13,22 +12,35 @@ import ExecutionContext from '../domain/ExecutionContext'; class MigrateRollback extends Command { static description = 'Rollback migrations up to the last run batch.'; - /** - * Default CLI options for running synchronize. - * - * @param {*} userParams - * @returns {SyncParams} - */ - getSyncParams(userParams: any): SyncParams { + getParams(): MigrationCommandParams { return { - ...userParams, - // Individual success handler - onSuccess: (context: ExecutionContext) => - printLine(` [✓] ${context.connectionId} - Successful (${context.timeElapsed}s)`), - - // Individual error handler - onFailed: (context: ExecutionContext) => - printLine(` [✖] ${context.connectionId} - Failed (${context.timeElapsed}s)`) + onSuccess: async (result: MigrationResult) => { + const log = dbLogger(result.connectionId); + const [num, list] = result.data; + const allRolledBack = num === 0; + + log('Already on the top of migrations: ', allRolledBack); + + await printLine(bold(` ▸ ${result.connectionId} - Successful`) + ` (${result.timeElapsed}s)`); + + if (allRolledBack) { + await printLine(' No more migrations to rollback.\n'); + + return; + } + + // Completed migrations. + for (const item of list) { + await printLine(cyan(` - ${item}`)); + } + + await printInfo(`\n Rolled back ${list.length} migrations.\n`); + }, + onFailed: async (result: MigrationResult) => { + printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); + + await printError(` ${result.error}\n`); + } }; } @@ -38,36 +50,22 @@ class MigrateRollback extends Command { * @returns {Promise} */ async run(): Promise { - const { flags: parsedFlags } = this.parse(MigrateRollback); - const params = this.getSyncParams({ ...parsedFlags }); + const params = this.getParams(); + const config = await loadConfig(); const connections = await resolveConnections(); const { migrateRollback } = await import('../api'); - const timeStart = process.hrtime(); - - await printLine('Rolling back migrations\n'); const results = await migrateRollback(config, connections, params); - log('Results:', results); - console.log('Results', results); // tslint:disable-line - - const successfulCount = results.filter(item => item.success).length; - - if (successfulCount > 0) { - // Display output. - await printLine( - `Rollback complete for ${successfulCount} / ${results.length} connection(s). ` + - `(${getElapsedTime(timeStart)}s)` - ); - } + const failedCount = results.filter(({ success }) => !success).length; - // If all completed successfully, exit gracefully. - if (results.length === successfulCount) { + if (failedCount === 0) { return process.exit(0); } - throw new Error(`Rollback failed for some connections.`); + printError(`Error: Rollback failed for ${failedCount} connection(s).`); + process.exit(-1); } } diff --git a/src/service/migrator.ts b/src/service/migrator.ts index 4729b666..8064ad94 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -6,14 +6,14 @@ import { resolveFile } from './sqlRunner'; import { getElapsedTime } from '../util/ts'; // import SqlMigrationContext, { MigrationContext } from '../migration/SqlMigrationContext'; import SqlMigrationEntry from '../domain/SqlMigrationEntry'; -import SyncResult from '../domain/SyncResult'; +// import SyncResult from '../domain/SyncResult' // import SyncContext from '../domain/SyncContext'; import { dbLogger } from '../util/logger'; import { isCLI } from '../config'; -import ExecutionContext from '../domain/ExecutionContext'; +// import ExecutionContext from '../domain/ExecutionContext'; // import KnexMigrationSource from '../migration/KnexMigrationSource'; import SyncConfig from '../domain/SyncConfig'; -import SyncParams from '../domain/SyncParams'; +// import SyncParams from '../domain/SyncParams'; const FILE_PATTERN = /(.+)\.(up|down)\.sql$/; @@ -85,38 +85,36 @@ export interface MigrationLatestResult { error?: any; } +export interface MigrationResult { + connectionId: string; + success: boolean; + timeElapsed: number; + data: any; + error?: any; +} + export interface MigrationCommandParams { onSuccess: (context: T) => Promise; onFailed: (context: T) => Promise; } -export interface MCommandContext { +export interface MigrationCommandContext { config: SyncConfig; connectionId: string; params: T; knexMigrationConfig: Knex.MigratorConfig; } -/** - * Synchronize context parameters for the current database connection. - */ -interface MigrationCommandContext { - config: SyncConfig; - connectionId: string; - params: SyncParams; - knexMigrationConfig: Knex.MigratorConfig; -} - /** * Run migrations on a target database connection / transaction. * * @param {(Knex | Knex.Transaction)} trx - * @param {MCommandContext>} context + * @param {MigrationCommandContext>} context * @returns {Promise} */ export async function migrateLatest( trx: Knex | Knex.Transaction, - context: MCommandContext> + context: MigrationCommandContext> ): Promise { const { connectionId, knexMigrationConfig } = context; const log = dbLogger(context.connectionId); @@ -159,40 +157,46 @@ export async function migrateLatest( return result; } -export async function migrateRollback(trx: Knex | Knex.Transaction, context: MigrationCommandContext): Promise { +export async function migrateRollback( + trx: Knex | Knex.Transaction, + context: MigrationCommandContext> +): Promise { const { connectionId, knexMigrationConfig } = context; const log = dbLogger(context.connectionId); - const result: SyncResult = { connectionId, success: false }; + + let error; + let data; const timeStart = process.hrtime(); try { log('BEGIN: migrate.rollback'); - const migrationResult = await trx.migrate.rollback(knexMigrationConfig); + data = await trx.migrate.rollback(knexMigrationConfig); log('END: migrate.rollback'); - log('Migration Result:\n%O', migrationResult); - - result.success = true; + log('Migration Result:\n%O', data); } catch (e) { log(`Error caught for connection ${connectionId}:`, e); - result.error = e; + error = e; } const timeElapsed = getElapsedTime(timeStart); log(`Execution completed in ${timeElapsed} s`); + const result: MigrationResult = { + connectionId, + error, + data, + timeElapsed, + success: !error + }; + // If it's a CLI environment, invoke the handler. if (isCLI()) { const handler = result.success ? context.params.onSuccess : context.params.onFailed; - const execContext: ExecutionContext = { - connectionId, - timeElapsed, - success: result.success - }; - await handler(execContext); + await handler(result); } return result; @@ -200,7 +204,7 @@ export async function migrateRollback(trx: Knex | Knex.Transaction, context: Mig export async function migrateList( trx: Knex | Knex.Transaction, - context: MCommandContext> + context: MigrationCommandContext> ): Promise { const { connectionId, knexMigrationConfig } = context; const log = dbLogger(context.connectionId); From 1f1ed8607bf67e35620505cc353b89bfb65070d0 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 19 Apr 2020 03:16:34 +0545 Subject: [PATCH 47/80] Refactor migration code --- src/api.ts | 19 ++-- src/commands/migrate-latest.ts | 8 +- src/commands/migrate-list.ts | 8 +- src/commands/migrate-rollback.ts | 2 +- src/service/migrator.ts | 188 ++++++++----------------------- 5 files changed, 65 insertions(+), 160 deletions(-) diff --git a/src/api.ts b/src/api.ts index 71f3f10a..9c132e05 100644 --- a/src/api.ts +++ b/src/api.ts @@ -19,12 +19,7 @@ import { executeProcesses } from './service/execution'; import * as migratorService from './service/migrator'; import SqlMigrationContext from './migration/SqlMigrationContext'; import KnexMigrationSource from './migration/KnexMigrationSource'; -import { - MigrationListResult, - MigrationCommandParams, - MigrationLatestResult, - MigrationResult -} from './service/migrator'; +import { MigrationCommandParams, MigrationResult } from './service/migrator'; /** * Synchronize all the configured database connections. @@ -62,8 +57,8 @@ export async function synchronize( export async function migrateLatest( config: SyncConfig, conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, - options?: MigrationCommandParams -): Promise { + options?: MigrationCommandParams +): Promise { log('Starting to migrate.'); const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); @@ -102,8 +97,8 @@ export async function migrateLatest( export async function migrateRollback( config: SyncConfig, conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, - options?: MigrationCommandParams -): Promise { + options?: MigrationCommandParams +): Promise { log('Starting to migrate.'); const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); @@ -142,8 +137,8 @@ export async function migrateRollback( export async function migrateList( config: SyncConfig, conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, - options?: MigrationCommandParams -): Promise { + options?: MigrationCommandParams +): Promise { const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); diff --git a/src/commands/migrate-latest.ts b/src/commands/migrate-latest.ts index 0e7d80a5..32f6cd3a 100644 --- a/src/commands/migrate-latest.ts +++ b/src/commands/migrate-latest.ts @@ -3,7 +3,7 @@ import { bold, red, cyan } from 'chalk'; import { printLine, printError, printInfo } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { MigrationCommandParams, MigrationLatestResult } from '../service/migrator'; +import { MigrationCommandParams, MigrationResult } from '../service/migrator'; import { dbLogger } from '../util/logger'; /** @@ -12,9 +12,9 @@ import { dbLogger } from '../util/logger'; class MigrateLatest extends Command { static description = 'Run the migrations up to the latest changes.'; - getParams(): MigrationCommandParams { + getParams(): MigrationCommandParams { return { - onSuccess: async (result: MigrationLatestResult) => { + onSuccess: async (result: MigrationResult) => { const log = dbLogger(result.connectionId); const [num, list] = result.data; const alreadyUpToDate = num && list.length === 0; @@ -36,7 +36,7 @@ class MigrateLatest extends Command { await printInfo(`\n Ran ${list.length} migrations.\n`); }, - onFailed: async (result: MigrationLatestResult) => { + onFailed: async (result: MigrationResult) => { printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); await printError(` ${result.error}\n`); diff --git a/src/commands/migrate-list.ts b/src/commands/migrate-list.ts index b6fa588a..b6543520 100644 --- a/src/commands/migrate-list.ts +++ b/src/commands/migrate-list.ts @@ -3,7 +3,7 @@ import { bold, grey, red, cyan, yellow } from 'chalk'; import { printLine, printError } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { MigrationListResult, MigrationCommandParams } from '../service/migrator'; +import { MigrationResult, MigrationCommandParams } from '../service/migrator'; /** * Migration command handler. @@ -11,9 +11,9 @@ import { MigrationListResult, MigrationCommandParams } from '../service/migrator class MigrateList extends Command { static description = 'List migrations.'; - getParams(): MigrationCommandParams { + getParams(): MigrationCommandParams { return { - onSuccess: async (result: MigrationListResult) => { + onSuccess: async (result: MigrationResult) => { await printLine(bold(` ▸ ${result.connectionId}`)); const [list1, list2] = result.data; @@ -40,7 +40,7 @@ class MigrateList extends Command { await printLine(); }, - onFailed: async (result: MigrationListResult) => { + onFailed: async (result: MigrationResult) => { printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); await printError(` ${result.error}\n`); diff --git a/src/commands/migrate-rollback.ts b/src/commands/migrate-rollback.ts index cf21c620..1f1b6135 100644 --- a/src/commands/migrate-rollback.ts +++ b/src/commands/migrate-rollback.ts @@ -12,7 +12,7 @@ import { dbLogger } from '../util/logger'; class MigrateRollback extends Command { static description = 'Rollback migrations up to the last run batch.'; - getParams(): MigrationCommandParams { + getParams(): MigrationCommandParams { return { onSuccess: async (result: MigrationResult) => { const log = dbLogger(result.connectionId); diff --git a/src/service/migrator.ts b/src/service/migrator.ts index 8064ad94..89e1bf3b 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -4,19 +4,39 @@ import * as path from 'path'; import { glob, exists } from '../util/fs'; import { resolveFile } from './sqlRunner'; import { getElapsedTime } from '../util/ts'; -// import SqlMigrationContext, { MigrationContext } from '../migration/SqlMigrationContext'; import SqlMigrationEntry from '../domain/SqlMigrationEntry'; -// import SyncResult from '../domain/SyncResult' -// import SyncContext from '../domain/SyncContext'; import { dbLogger } from '../util/logger'; import { isCLI } from '../config'; -// import ExecutionContext from '../domain/ExecutionContext'; -// import KnexMigrationSource from '../migration/KnexMigrationSource'; import SyncConfig from '../domain/SyncConfig'; -// import SyncParams from '../domain/SyncParams'; const FILE_PATTERN = /(.+)\.(up|down)\.sql$/; +export interface MigrationResult { + connectionId: string; + success: boolean; + timeElapsed: number; + data: any; + error?: any; +} + +export interface MigrationCommandParams { + onSuccess: (result: MigrationResult) => Promise; + onFailed: (context: MigrationResult) => Promise; +} + +export interface MigrationCommandContext { + config: SyncConfig; + connectionId: string; + params: MigrationCommandParams; + knexMigrationConfig: Knex.MigratorConfig; +} + +const knexMigrationRunnersMap = { + 'migrate.latest': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.latest(config), + 'migrate.rollback': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.rollback(config), + 'migrate.list': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.list(config) +}; + /** * Glob the migration directory and retrieve all the migration entries (names) * that needs to be run. @@ -69,55 +89,14 @@ export async function resolveSqlMigrations(migrationPath: string): Promise { - onSuccess: (context: T) => Promise; - onFailed: (context: T) => Promise; -} - -export interface MigrationCommandContext { - config: SyncConfig; - connectionId: string; - params: T; - knexMigrationConfig: Knex.MigratorConfig; -} - -/** - * Run migrations on a target database connection / transaction. - * - * @param {(Knex | Knex.Transaction)} trx - * @param {MigrationCommandContext>} context - * @returns {Promise} - */ -export async function migrateLatest( +export async function migrateRun( trx: Knex | Knex.Transaction, - context: MigrationCommandContext> -): Promise { - const { connectionId, knexMigrationConfig } = context; + context: MigrationCommandContext, + func: (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => Promise +): Promise { const log = dbLogger(context.connectionId); + const { connectionId, knexMigrationConfig } = context; + const funcName = func.name || 'func'; let error; let data; @@ -125,10 +104,10 @@ export async function migrateLatest( const timeStart = process.hrtime(); try { - log('BEGIN: migrate.latest'); - data = await trx.migrate.latest(knexMigrationConfig); + log(`BEGIN: ${funcName}`); + data = await func(trx, knexMigrationConfig); - log('END: migrate.latest'); + log(`END: ${funcName}`); log('Result:\n%O', data); } catch (e) { log(`Error caught for connection ${connectionId}:`, e); @@ -139,7 +118,7 @@ export async function migrateLatest( log(`Execution completed in ${timeElapsed} s`); - const result: MigrationListResult = { + const result: MigrationResult = { connectionId, error, data, @@ -157,92 +136,23 @@ export async function migrateLatest( return result; } -export async function migrateRollback( +export async function migrateLatest( trx: Knex | Knex.Transaction, - context: MigrationCommandContext> -): Promise { - const { connectionId, knexMigrationConfig } = context; - const log = dbLogger(context.connectionId); - - let error; - let data; - - const timeStart = process.hrtime(); - - try { - log('BEGIN: migrate.rollback'); - data = await trx.migrate.rollback(knexMigrationConfig); - - log('END: migrate.rollback'); - log('Migration Result:\n%O', data); - } catch (e) { - log(`Error caught for connection ${connectionId}:`, e); - error = e; - } - - const timeElapsed = getElapsedTime(timeStart); - - log(`Execution completed in ${timeElapsed} s`); - - const result: MigrationResult = { - connectionId, - error, - data, - timeElapsed, - success: !error - }; - - // If it's a CLI environment, invoke the handler. - if (isCLI()) { - const handler = result.success ? context.params.onSuccess : context.params.onFailed; - - await handler(result); - } + context: MigrationCommandContext +): Promise { + return migrateRun(trx, context, knexMigrationRunnersMap['migrate.latest']); +} - return result; +export async function migrateRollback( + trx: Knex | Knex.Transaction, + context: MigrationCommandContext +): Promise { + return migrateRun(trx, context, knexMigrationRunnersMap['migrate.rollback']); } export async function migrateList( trx: Knex | Knex.Transaction, - context: MigrationCommandContext> -): Promise { - const { connectionId, knexMigrationConfig } = context; - const log = dbLogger(context.connectionId); - - let error; - let data; - - const timeStart = process.hrtime(); - - try { - log('BEGIN: migrate.list'); - data = await trx.migrate.list(knexMigrationConfig); - - log('END: migrate.list'); - log('Result:\n%O', data); - } catch (e) { - log(`Error caught for connection ${connectionId}:`, e); - error = e; - } - - const timeElapsed = getElapsedTime(timeStart); - - log(`Execution completed in ${timeElapsed} s`); - - const result: MigrationListResult = { - connectionId, - error, - data, - timeElapsed, - success: !error - }; - - // If it's a CLI environment, invoke the handler. - if (isCLI()) { - const handler = result.success ? context.params.onSuccess : context.params.onFailed; - - await handler(result); - } - - return result; + context: MigrationCommandContext +): Promise { + return migrateRun(trx, context, knexMigrationRunnersMap['migrate.list']); } From 9d2037a7c4da322016dd226d513dec83530818a9 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 19 Apr 2020 13:33:23 +0545 Subject: [PATCH 48/80] Deduplicate redundancy on migrations --- src/api.ts | 79 ++++++++++++---------- src/commands/migrate-latest.ts | 2 +- src/commands/migrate-list.ts | 2 +- src/commands/migrate-rollback.ts | 2 +- src/migration/KnexMigrationSource.ts | 1 + src/service/knexMigrator.ts | 90 +++++++++++++++++++++++++ src/service/migrator.ts | 99 ---------------------------- 7 files changed, 140 insertions(+), 135 deletions(-) create mode 100644 src/service/knexMigrator.ts diff --git a/src/api.ts b/src/api.ts index 9c132e05..f97df435 100644 --- a/src/api.ts +++ b/src/api.ts @@ -4,22 +4,23 @@ import { mergeDeepRight } from 'ramda'; import { log } from './util/logger'; import { getConnectionId } from './config'; +import { DEFAULT_SYNC_PARAMS } from './constants'; +import { isKnexInstance, getConfig, createInstance } from './util/db'; + import SyncParams from './domain/SyncParams'; import SyncConfig from './domain/SyncConfig'; import SyncResult from './domain/SyncResult'; -import { DEFAULT_SYNC_PARAMS } from './constants'; - import ConnectionConfig from './domain/ConnectionConfig'; import ConnectionReference from './domain/ConnectionReference'; -import { isKnexInstance, getConfig, createInstance } from './util/db'; +import SqlMigrationContext from './migration/SqlMigrationContext'; +import KnexMigrationSource from './migration/KnexMigrationSource'; // Services import { synchronizeDatabase } from './service/sync'; import { executeProcesses } from './service/execution'; import * as migratorService from './service/migrator'; -import SqlMigrationContext from './migration/SqlMigrationContext'; -import KnexMigrationSource from './migration/KnexMigrationSource'; -import { MigrationCommandParams, MigrationResult } from './service/migrator'; +import * as knexMigratorService from './service/knexMigrator'; +import { MigrationCommandParams, MigrationResult } from './service/knexMigrator'; /** * Synchronize all the configured database connections. @@ -76,15 +77,19 @@ export async function migrateLatest( const getMigrationContext = (connectionId: string) => new SqlMigrationContext(connectionId, migrations); const processes = connections.map(({ connection, id: connectionId }) => () => - migratorService.migrateLatest(connection, { - config, - params, - connectionId, - knexMigrationConfig: { - tableName: config.migration.tableName, - migrationSource: new KnexMigrationSource(getMigrationContext(connectionId)) - } - }) + knexMigratorService.runMigrateFunc( + connection, + { + config, + params, + connectionId, + knexMigrationConfig: { + tableName: config.migration.tableName, + migrationSource: new KnexMigrationSource(getMigrationContext(connectionId)) + } + }, + knexMigratorService.migrationApiMap['migrate.latest'] + ) ); const results = await executeProcesses(processes, config); @@ -116,15 +121,19 @@ export async function migrateRollback( const getMigrationContext = (connectionId: string) => new SqlMigrationContext(connectionId, migrations); const processes = connections.map(({ connection, id: connectionId }) => () => - migratorService.migrateRollback(connection, { - config, - params, - connectionId, - knexMigrationConfig: { - tableName: config.migration.tableName, - migrationSource: new KnexMigrationSource(getMigrationContext(connectionId)) - } - }) + knexMigratorService.runMigrateFunc( + connection, + { + config, + params, + connectionId, + knexMigrationConfig: { + tableName: config.migration.tableName, + migrationSource: new KnexMigrationSource(getMigrationContext(connectionId)) + } + }, + knexMigratorService.migrationApiMap['migrate.rollback'] + ) ); const results = await executeProcesses(processes, config); @@ -155,15 +164,19 @@ export async function migrateList( const getMigrationContext = (connectionId: string) => new SqlMigrationContext(connectionId, migrations); const processes = connections.map(({ connection, id: connectionId }) => () => - migratorService.migrateList(connection, { - config, - params, - connectionId, - knexMigrationConfig: { - tableName: config.migration.tableName, - migrationSource: new KnexMigrationSource(getMigrationContext(connectionId)) - } - }) + knexMigratorService.runMigrateFunc( + connection, + { + config, + params, + connectionId, + knexMigrationConfig: { + tableName: config.migration.tableName, + migrationSource: new KnexMigrationSource(getMigrationContext(connectionId)) + } + }, + knexMigratorService.migrationApiMap['migrate.list'] + ) ); const results = await executeProcesses(processes, config); diff --git a/src/commands/migrate-latest.ts b/src/commands/migrate-latest.ts index 32f6cd3a..7a88035f 100644 --- a/src/commands/migrate-latest.ts +++ b/src/commands/migrate-latest.ts @@ -3,7 +3,7 @@ import { bold, red, cyan } from 'chalk'; import { printLine, printError, printInfo } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { MigrationCommandParams, MigrationResult } from '../service/migrator'; +import { MigrationCommandParams, MigrationResult } from '../service/knexMigrator'; import { dbLogger } from '../util/logger'; /** diff --git a/src/commands/migrate-list.ts b/src/commands/migrate-list.ts index b6543520..9c101db4 100644 --- a/src/commands/migrate-list.ts +++ b/src/commands/migrate-list.ts @@ -3,7 +3,7 @@ import { bold, grey, red, cyan, yellow } from 'chalk'; import { printLine, printError } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { MigrationResult, MigrationCommandParams } from '../service/migrator'; +import { MigrationResult, MigrationCommandParams } from '../service/knexMigrator'; /** * Migration command handler. diff --git a/src/commands/migrate-rollback.ts b/src/commands/migrate-rollback.ts index 1f1b6135..fb113123 100644 --- a/src/commands/migrate-rollback.ts +++ b/src/commands/migrate-rollback.ts @@ -3,7 +3,7 @@ import { bold, red, cyan } from 'chalk'; import { printLine, printError, printInfo } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { MigrationCommandParams, MigrationResult } from '../service/migrator'; +import { MigrationCommandParams, MigrationResult } from '../service/knexMigrator'; import { dbLogger } from '../util/logger'; /** diff --git a/src/migration/KnexMigrationSource.ts b/src/migration/KnexMigrationSource.ts index 7074c903..0d879726 100644 --- a/src/migration/KnexMigrationSource.ts +++ b/src/migration/KnexMigrationSource.ts @@ -42,6 +42,7 @@ class KnexMigrationSource { return migration; } + /** * Get the migration runner. * diff --git a/src/service/knexMigrator.ts b/src/service/knexMigrator.ts new file mode 100644 index 00000000..fdec185c --- /dev/null +++ b/src/service/knexMigrator.ts @@ -0,0 +1,90 @@ +import Knex from 'knex'; + +import { isCLI } from '../config'; +import { dbLogger } from '../util/logger'; +import { getElapsedTime } from '../util/ts'; +import SyncConfig from '../domain/SyncConfig'; + +export interface MigrationResult { + connectionId: string; + success: boolean; + timeElapsed: number; + data: any; + error?: any; +} + +export interface MigrationCommandParams { + onSuccess: (result: MigrationResult) => Promise; + onFailed: (context: MigrationResult) => Promise; +} + +export interface MigrationCommandContext { + config: SyncConfig; + connectionId: string; + params: MigrationCommandParams; + knexMigrationConfig: Knex.MigratorConfig; +} + +/** + * A map of Knex's migration API functions. + */ +export const migrationApiMap = { + 'migrate.latest': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.latest(config), + 'migrate.rollback': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.rollback(config), + 'migrate.list': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.list(config) +}; + +/** + * Invoke Knex's migration API. + * + * @param {(Knex | Knex.Transaction)} trx + * @param {MigrationCommandContext} context + * @param {((trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => Promise)} func + * @returns {Promise} + */ +export async function runMigrateFunc( + trx: Knex | Knex.Transaction, + context: MigrationCommandContext, + func: (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => Promise +): Promise { + const log = dbLogger(context.connectionId); + const { connectionId, knexMigrationConfig } = context; + const funcName = func.name || 'func'; + + let error; + let data; + + const timeStart = process.hrtime(); + + try { + log(`BEGIN: ${funcName}`); + data = await func(trx, knexMigrationConfig); + + log(`END: ${funcName}`); + log('Result:\n%O', data); + } catch (e) { + log(`Error caught for connection ${connectionId}:`, e); + error = e; + } + + const timeElapsed = getElapsedTime(timeStart); + + log(`Execution completed in ${timeElapsed} s`); + + const result: MigrationResult = { + connectionId, + error, + data, + timeElapsed, + success: !error + }; + + // If it's a CLI environment, invoke the handler. + if (isCLI()) { + const handler = result.success ? context.params.onSuccess : context.params.onFailed; + + await handler(result); + } + + return result; +} diff --git a/src/service/migrator.ts b/src/service/migrator.ts index 89e1bf3b..576fbc4e 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -1,42 +1,11 @@ -import Knex from 'knex'; import * as path from 'path'; import { glob, exists } from '../util/fs'; import { resolveFile } from './sqlRunner'; -import { getElapsedTime } from '../util/ts'; import SqlMigrationEntry from '../domain/SqlMigrationEntry'; -import { dbLogger } from '../util/logger'; -import { isCLI } from '../config'; -import SyncConfig from '../domain/SyncConfig'; const FILE_PATTERN = /(.+)\.(up|down)\.sql$/; -export interface MigrationResult { - connectionId: string; - success: boolean; - timeElapsed: number; - data: any; - error?: any; -} - -export interface MigrationCommandParams { - onSuccess: (result: MigrationResult) => Promise; - onFailed: (context: MigrationResult) => Promise; -} - -export interface MigrationCommandContext { - config: SyncConfig; - connectionId: string; - params: MigrationCommandParams; - knexMigrationConfig: Knex.MigratorConfig; -} - -const knexMigrationRunnersMap = { - 'migrate.latest': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.latest(config), - 'migrate.rollback': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.rollback(config), - 'migrate.list': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.list(config) -}; - /** * Glob the migration directory and retrieve all the migration entries (names) * that needs to be run. @@ -88,71 +57,3 @@ export async function resolveSqlMigrations(migrationPath: string): Promise Promise -): Promise { - const log = dbLogger(context.connectionId); - const { connectionId, knexMigrationConfig } = context; - const funcName = func.name || 'func'; - - let error; - let data; - - const timeStart = process.hrtime(); - - try { - log(`BEGIN: ${funcName}`); - data = await func(trx, knexMigrationConfig); - - log(`END: ${funcName}`); - log('Result:\n%O', data); - } catch (e) { - log(`Error caught for connection ${connectionId}:`, e); - error = e; - } - - const timeElapsed = getElapsedTime(timeStart); - - log(`Execution completed in ${timeElapsed} s`); - - const result: MigrationResult = { - connectionId, - error, - data, - timeElapsed, - success: !error - }; - - // If it's a CLI environment, invoke the handler. - if (isCLI()) { - const handler = result.success ? context.params.onSuccess : context.params.onFailed; - - await handler(result); - } - - return result; -} - -export async function migrateLatest( - trx: Knex | Knex.Transaction, - context: MigrationCommandContext -): Promise { - return migrateRun(trx, context, knexMigrationRunnersMap['migrate.latest']); -} - -export async function migrateRollback( - trx: Knex | Knex.Transaction, - context: MigrationCommandContext -): Promise { - return migrateRun(trx, context, knexMigrationRunnersMap['migrate.rollback']); -} - -export async function migrateList( - trx: Knex | Knex.Transaction, - context: MigrationCommandContext -): Promise { - return migrateRun(trx, context, knexMigrationRunnersMap['migrate.list']); -} From fbb51da7c346377f1f9511bac0c67c178eb066f9 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 19 Apr 2020 18:38:19 +0545 Subject: [PATCH 49/80] Simplify codebase --- src/api.ts | 98 +++++++------------------------------ src/service/knexMigrator.ts | 32 +++++++++--- src/service/migrator.ts | 9 ++-- 3 files changed, 48 insertions(+), 91 deletions(-) diff --git a/src/api.ts b/src/api.ts index f97df435..74872265 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,5 +1,4 @@ import * as Knex from 'knex'; -import * as path from 'path'; import { mergeDeepRight } from 'ramda'; import { log } from './util/logger'; @@ -12,15 +11,12 @@ import SyncConfig from './domain/SyncConfig'; import SyncResult from './domain/SyncResult'; import ConnectionConfig from './domain/ConnectionConfig'; import ConnectionReference from './domain/ConnectionReference'; -import SqlMigrationContext from './migration/SqlMigrationContext'; -import KnexMigrationSource from './migration/KnexMigrationSource'; // Services import { synchronizeDatabase } from './service/sync'; import { executeProcesses } from './service/execution'; -import * as migratorService from './service/migrator'; -import * as knexMigratorService from './service/knexMigrator'; import { MigrationCommandParams, MigrationResult } from './service/knexMigrator'; +import { runMigrateFunc, resolveKnexMigrationConfig, migrationApiMap } from './service/knexMigrator'; /** * Synchronize all the configured database connections. @@ -46,13 +42,7 @@ export async function synchronize( }) ); - // Explicitly suppressing the `| Error` type since - // all errors are already caught inside synchronizeDatabase(). - const results = (await executeProcesses(processes, config)) as SyncResult[]; - - log('Synchronization completed.'); - - return results; + return executeProcesses(processes, config); } export async function migrateLatest( @@ -63,40 +53,22 @@ export async function migrateLatest( log('Starting to migrate.'); const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); - - const { basePath, migration } = config; - const migrationPath = path.join(basePath, migration.directory); - const migrations = await migratorService.resolveSqlMigrations(migrationPath); - - log('Migration Path:', migrationPath); - log('Available migrations:'); - log('%O', migrations); - - // TODO: We'll need to support different types of migrations eg both sql & js - // For instance migrations in JS would have different context like JavaScriptMigrationContext. - const getMigrationContext = (connectionId: string) => new SqlMigrationContext(connectionId, migrations); + const knexMigrationConfig = await resolveKnexMigrationConfig(config); const processes = connections.map(({ connection, id: connectionId }) => () => - knexMigratorService.runMigrateFunc( + runMigrateFunc( connection, { config, params, connectionId, - knexMigrationConfig: { - tableName: config.migration.tableName, - migrationSource: new KnexMigrationSource(getMigrationContext(connectionId)) - } + knexMigrationConfig: knexMigrationConfig(connectionId) }, - knexMigratorService.migrationApiMap['migrate.latest'] + migrationApiMap['migrate.latest'] ) ); - const results = await executeProcesses(processes, config); - - log('Migrations completed.'); - - return results; + return executeProcesses(processes, config); } export async function migrateRollback( @@ -107,40 +79,22 @@ export async function migrateRollback( log('Starting to migrate.'); const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); - - const { basePath, migration } = config; - const migrationPath = path.join(basePath, migration.directory); - const migrations = await migratorService.resolveSqlMigrations(migrationPath); - - log('Migration Path:', migrationPath); - log('Available migrations:'); - log('%O', migrations); - - // TODO: We'll need to support different types of migrations eg both sql & js - // For instance migrations in JS would have different context like JavaScriptMigrationContext. - const getMigrationContext = (connectionId: string) => new SqlMigrationContext(connectionId, migrations); + const knexMigrationConfig = await resolveKnexMigrationConfig(config); const processes = connections.map(({ connection, id: connectionId }) => () => - knexMigratorService.runMigrateFunc( + runMigrateFunc( connection, { config, params, connectionId, - knexMigrationConfig: { - tableName: config.migration.tableName, - migrationSource: new KnexMigrationSource(getMigrationContext(connectionId)) - } + knexMigrationConfig: knexMigrationConfig(connectionId) }, - knexMigratorService.migrationApiMap['migrate.rollback'] + migrationApiMap['migrate.rollback'] ) ); - const results = await executeProcesses(processes, config); - - log('Migrations completed.'); - - return results; + return executeProcesses(processes, config); } export async function migrateList( @@ -150,40 +104,22 @@ export async function migrateList( ): Promise { const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); - - const { basePath, migration } = config; - const migrationPath = path.join(basePath, migration.directory); - const migrations = await migratorService.resolveSqlMigrations(migrationPath); - - log('Migration Path:', migrationPath); - log('Available migration sources:'); - log('%O', migrations); - - // TODO: We'll need to support different types of migrations eg both sql & js - // For instance migrations in JS would have different context like JavaScriptMigrationContext. - const getMigrationContext = (connectionId: string) => new SqlMigrationContext(connectionId, migrations); + const knexMigrationConfig = await resolveKnexMigrationConfig(config); const processes = connections.map(({ connection, id: connectionId }) => () => - knexMigratorService.runMigrateFunc( + runMigrateFunc( connection, { config, params, connectionId, - knexMigrationConfig: { - tableName: config.migration.tableName, - migrationSource: new KnexMigrationSource(getMigrationContext(connectionId)) - } + knexMigrationConfig: knexMigrationConfig(connectionId) }, - knexMigratorService.migrationApiMap['migrate.list'] + migrationApiMap['migrate.list'] ) ); - const results = await executeProcesses(processes, config); - - log('Finished retrieving lists.'); - - return results; + return executeProcesses(processes, config); } /** diff --git a/src/service/knexMigrator.ts b/src/service/knexMigrator.ts index fdec185c..629fb21b 100644 --- a/src/service/knexMigrator.ts +++ b/src/service/knexMigrator.ts @@ -1,9 +1,12 @@ import Knex from 'knex'; import { isCLI } from '../config'; -import { dbLogger } from '../util/logger'; +import { dbLogger, log } from '../util/logger'; import { getElapsedTime } from '../util/ts'; import SyncConfig from '../domain/SyncConfig'; +import * as migratorService from './migrator'; +import KnexMigrationSource from '../migration/KnexMigrationSource'; +import SqlMigrationContext from '../migration/SqlMigrationContext'; export interface MigrationResult { connectionId: string; @@ -34,6 +37,21 @@ export const migrationApiMap = { 'migrate.list': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.list(config) }; +// TODO: Naming +export async function resolveKnexMigrationConfig(config: SyncConfig) { + const migrations = await migratorService.resolveSqlMigrations(config); + log('Available migrations:\n%O', migrations); + + return (connectionId: string) => ({ + tableName: config.migration.tableName, + migrationSource: new KnexMigrationSource( + // TODO: We'll need to support different types of migrations eg both sql & js + // For instance migrations in JS would have different context like JavaScriptMigrationContext. + new SqlMigrationContext(connectionId, migrations) + ) + }); +} + /** * Invoke Knex's migration API. * @@ -47,7 +65,7 @@ export async function runMigrateFunc( context: MigrationCommandContext, func: (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => Promise ): Promise { - const log = dbLogger(context.connectionId); + const dbLog = dbLogger(context.connectionId); const { connectionId, knexMigrationConfig } = context; const funcName = func.name || 'func'; @@ -57,19 +75,19 @@ export async function runMigrateFunc( const timeStart = process.hrtime(); try { - log(`BEGIN: ${funcName}`); + dbLog(`BEGIN: ${funcName}`); data = await func(trx, knexMigrationConfig); - log(`END: ${funcName}`); - log('Result:\n%O', data); + dbLog(`END: ${funcName}`); + dbLog('Result:\n%O', data); } catch (e) { - log(`Error caught for connection ${connectionId}:`, e); + dbLog(`Error caught for connection ${connectionId}:`, e); error = e; } const timeElapsed = getElapsedTime(timeStart); - log(`Execution completed in ${timeElapsed} s`); + dbLog(`Execution completed in ${timeElapsed} s`); const result: MigrationResult = { connectionId, diff --git a/src/service/migrator.ts b/src/service/migrator.ts index 576fbc4e..3a58444e 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -3,6 +3,7 @@ import * as path from 'path'; import { glob, exists } from '../util/fs'; import { resolveFile } from './sqlRunner'; import SqlMigrationEntry from '../domain/SqlMigrationEntry'; +import SyncConfig from '../domain/SyncConfig'; const FILE_PATTERN = /(.+)\.(up|down)\.sql$/; @@ -32,12 +33,14 @@ export async function getSqlMigrationNames(migrationPath: string): Promise} */ -export async function resolveSqlMigrations(migrationPath: string): Promise { +export async function resolveSqlMigrations(config: SyncConfig): Promise { + const { basePath, migration } = config; + const migrationPath = path.join(basePath, migration.directory); const migrationNames = await getSqlMigrationNames(migrationPath); const migrationPromises = migrationNames.map(async name => { const upFilename = `${name}.up.sql`; From a11ef737f7df8b488b47266a03b247cf83d0efe6 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 19 Apr 2020 21:57:16 +0545 Subject: [PATCH 50/80] Add migration.sourceType config option --- src/constants.ts | 3 ++- src/domain/SyncConfig.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/constants.ts b/src/constants.ts index 33d36991..892252eb 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -27,7 +27,8 @@ export const DEFAULT_CONFIG: SyncConfig = { }, migration: { directory: 'migration', - tableName: 'knex_migrations' // Note: This is Knex's default value. Just keeping it same. + tableName: 'knex_migrations', // Note: This is Knex's default value. Just keeping it same. + sourceType: 'sql' } }; diff --git a/src/domain/SyncConfig.ts b/src/domain/SyncConfig.ts index b951adad..e25d5527 100644 --- a/src/domain/SyncConfig.ts +++ b/src/domain/SyncConfig.ts @@ -17,6 +17,8 @@ interface SyncConfig { migration: { directory: string; tableName: string; + // TODO: Only 'sql' is supported sourceType now. JS will be supported later. + sourceType: 'sql' | 'javascript'; }; } From 3074396fa96194b5a2008adec303dc57e80ffe28 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 19 Apr 2020 23:20:17 +0545 Subject: [PATCH 51/80] Ability to lazy bind connectionId --- src/domain/MigrationContext.ts | 1 + src/migration/SqlMigrationContext.ts | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/domain/MigrationContext.ts b/src/domain/MigrationContext.ts index a7c798a4..fa1ed5ff 100644 --- a/src/domain/MigrationContext.ts +++ b/src/domain/MigrationContext.ts @@ -7,6 +7,7 @@ interface MigrationContext { connectionId: string; keys(): string[]; get(name: string): MigrationRunner; + bind(connectionId: string): MigrationContext; } export default MigrationContext; diff --git a/src/migration/SqlMigrationContext.ts b/src/migration/SqlMigrationContext.ts index 38497fa3..9629f9ea 100644 --- a/src/migration/SqlMigrationContext.ts +++ b/src/migration/SqlMigrationContext.ts @@ -1,7 +1,7 @@ import * as Knex from 'knex'; -import { dbLogger } from '../util/logger'; import MigrationRunner from '../domain/MigrationRunner'; +import { dbLogger, log as logger } from '../util/logger'; import MigrationContext from '../domain/MigrationContext'; import SqlMigrationEntry from '../domain/SqlMigrationEntry'; @@ -16,13 +16,25 @@ class SqlMigrationContext implements MigrationContext { /** * SqlMigrationContext constructor. * - * @param {string} connectionId * @param {SqlMigrationEntry[]} list */ - constructor(connectionId: string, list: SqlMigrationEntry[]) { + constructor(list: SqlMigrationEntry[]) { this.list = list; + this.log = logger; + this.connectionId = ''; + } + + /** + * Bind connectionId to the context. + * + * @param {string} connectionId + * @returns {MigrationContext} this + */ + bind(connectionId: string): MigrationContext { this.connectionId = connectionId; this.log = dbLogger(connectionId); + + return this; } /** From f200128bf64a7a0d9203706afbc51527b83c7166 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 19 Apr 2020 23:21:05 +0545 Subject: [PATCH 52/80] Move knex migration source resolution upstream --- src/init.ts | 71 +++++++++++++++++++++++++++++++++++++ src/service/knexMigrator.ts | 20 +---------- 2 files changed, 72 insertions(+), 19 deletions(-) create mode 100644 src/init.ts diff --git a/src/init.ts b/src/init.ts new file mode 100644 index 00000000..0d4e5a89 --- /dev/null +++ b/src/init.ts @@ -0,0 +1,71 @@ +import Knex from 'knex'; + +import { log } from './util/logger'; +import SyncConfig from './domain/SyncConfig'; +import * as migratorService from './service/migrator'; +import MigrationContext from './domain/MigrationContext'; +import KnexMigrationSource from './migration/KnexMigrationSource'; +import SqlMigrationContext from './migration/SqlMigrationContext'; +import { validate } from './config'; + +export interface PrepareOptions { + loadMigrations?: boolean; + loadSqlSources?: boolean; +} + +export interface PreparedRequirements { + knexMigrationConfig: (connectionId: string) => Knex.MigratorConfig; +} + +/** + * Prepare configurations, preload requirements and validate before proceeding further. + * + * @param {SyncConfig} config + * @param {PrepareOptions} options + * @returns {Promise} + */ +export async function prepare(config: SyncConfig, options: PrepareOptions): Promise { + log('Prepare: ', options); + + // Validate the config. + // This might be the first step for the provided configuration for the programmatic API. + validate(config); + + const migrationContext = await resolveMigrationContext(config, options); + + return { + knexMigrationConfig: (connectionId: string) => ({ + tableName: config.migration.tableName, + migrationSource: migrationContext ? new KnexMigrationSource(migrationContext.bind(connectionId)) : null + }) + }; +} + +/** + * Resolve migration context based on the migration configuration. + * + * @param {SyncConfig} config + * @param {PrepareOptions} options + * @returns {(Promise)} + */ +async function resolveMigrationContext(config: SyncConfig, options: PrepareOptions): Promise { + if (options.loadMigrations !== true) { + return null; + } + + log(`Initialize migration context for sourceType: ${config.migration.sourceType}`); + + switch (config.migration.sourceType) { + case 'sql': + const src = await migratorService.resolveSqlMigrations(config); + + log('Available migration sources:\n%O', src); + + return new SqlMigrationContext(src); + + default: + // TODO: We'll need to support different types of migrations eg both sql & js + // For instance migrations in JS would have different context like JavaScriptMigrationContext. + throw new Error(`Unsupported migration.sourceType value "${config.migration.sourceType}".`); + } +} diff --git a/src/service/knexMigrator.ts b/src/service/knexMigrator.ts index 629fb21b..86808356 100644 --- a/src/service/knexMigrator.ts +++ b/src/service/knexMigrator.ts @@ -1,12 +1,9 @@ import Knex from 'knex'; import { isCLI } from '../config'; -import { dbLogger, log } from '../util/logger'; +import { dbLogger } from '../util/logger'; import { getElapsedTime } from '../util/ts'; import SyncConfig from '../domain/SyncConfig'; -import * as migratorService from './migrator'; -import KnexMigrationSource from '../migration/KnexMigrationSource'; -import SqlMigrationContext from '../migration/SqlMigrationContext'; export interface MigrationResult { connectionId: string; @@ -37,21 +34,6 @@ export const migrationApiMap = { 'migrate.list': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.list(config) }; -// TODO: Naming -export async function resolveKnexMigrationConfig(config: SyncConfig) { - const migrations = await migratorService.resolveSqlMigrations(config); - log('Available migrations:\n%O', migrations); - - return (connectionId: string) => ({ - tableName: config.migration.tableName, - migrationSource: new KnexMigrationSource( - // TODO: We'll need to support different types of migrations eg both sql & js - // For instance migrations in JS would have different context like JavaScriptMigrationContext. - new SqlMigrationContext(connectionId, migrations) - ) - }); -} - /** * Invoke Knex's migration API. * From b0fb9087f26db99f8cca72ed55b9e4ef570b3aee Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 19 Apr 2020 23:21:36 +0545 Subject: [PATCH 53/80] Use init.prepare() --- src/api.ts | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/api.ts b/src/api.ts index 74872265..ec450d9c 100644 --- a/src/api.ts +++ b/src/api.ts @@ -6,17 +6,18 @@ import { getConnectionId } from './config'; import { DEFAULT_SYNC_PARAMS } from './constants'; import { isKnexInstance, getConfig, createInstance } from './util/db'; +import * as init from './init'; import SyncParams from './domain/SyncParams'; import SyncConfig from './domain/SyncConfig'; import SyncResult from './domain/SyncResult'; import ConnectionConfig from './domain/ConnectionConfig'; import ConnectionReference from './domain/ConnectionReference'; -// Services +// Service import { synchronizeDatabase } from './service/sync'; import { executeProcesses } from './service/execution'; +import { runMigrateFunc, migrationApiMap } from './service/knexMigrator'; import { MigrationCommandParams, MigrationResult } from './service/knexMigrator'; -import { runMigrateFunc, resolveKnexMigrationConfig, migrationApiMap } from './service/knexMigrator'; /** * Synchronize all the configured database connections. @@ -31,7 +32,12 @@ export async function synchronize( conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, options?: SyncParams ): Promise { - log('Starting to synchronize.'); + log('Synchronize'); + + // Note: This does nothing right now. + // TODO: Need to preload the SQL source code under this step. + await init.prepare(config, { loadSqlSources: true }); + const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); const processes = connections.map(({ connection, id: connectionId }) => () => @@ -50,10 +56,11 @@ export async function migrateLatest( conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, options?: MigrationCommandParams ): Promise { - log('Starting to migrate.'); + log('Migrate Latest'); + const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); - const knexMigrationConfig = await resolveKnexMigrationConfig(config); + const { knexMigrationConfig } = await init.prepare(config, { loadMigrations: true }); const processes = connections.map(({ connection, id: connectionId }) => () => runMigrateFunc( @@ -76,10 +83,11 @@ export async function migrateRollback( conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, options?: MigrationCommandParams ): Promise { - log('Starting to migrate.'); + log('Migrate Rollback'); + const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); - const knexMigrationConfig = await resolveKnexMigrationConfig(config); + const { knexMigrationConfig } = await init.prepare(config, { loadMigrations: true }); const processes = connections.map(({ connection, id: connectionId }) => () => runMigrateFunc( @@ -102,9 +110,11 @@ export async function migrateList( conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, options?: MigrationCommandParams ): Promise { + log('Migrate List'); + const connections = mapToConnectionReferences(conn); const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); - const knexMigrationConfig = await resolveKnexMigrationConfig(config); + const { knexMigrationConfig } = await init.prepare(config, { loadMigrations: true }); const processes = connections.map(({ connection, id: connectionId }) => () => runMigrateFunc( From c6a011497a7bf659635cb22d37e65e0c4af506b1 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 19 Apr 2020 23:29:40 +0545 Subject: [PATCH 54/80] Skip validation on prepare for CLI --- src/init.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/init.ts b/src/init.ts index 0d4e5a89..e8827121 100644 --- a/src/init.ts +++ b/src/init.ts @@ -1,12 +1,12 @@ import Knex from 'knex'; import { log } from './util/logger'; +import { validate, isCLI } from './config'; import SyncConfig from './domain/SyncConfig'; import * as migratorService from './service/migrator'; import MigrationContext from './domain/MigrationContext'; import KnexMigrationSource from './migration/KnexMigrationSource'; import SqlMigrationContext from './migration/SqlMigrationContext'; -import { validate } from './config'; export interface PrepareOptions { loadMigrations?: boolean; @@ -27,9 +27,11 @@ export interface PreparedRequirements { export async function prepare(config: SyncConfig, options: PrepareOptions): Promise { log('Prepare: ', options); - // Validate the config. - // This might be the first step for the provided configuration for the programmatic API. - validate(config); + // Validate the config for programmatic API access. + // CLI access validates the config while loading, for programmatic access it should be done here. + if (!isCLI()) { + validate(config); + } const migrationContext = await resolveMigrationContext(config, options); @@ -53,7 +55,7 @@ async function resolveMigrationContext(config: SyncConfig, options: PrepareOptio return null; } - log(`Initialize migration context for sourceType: ${config.migration.sourceType}`); + log(`Initialize migration context [sourceType=${config.migration.sourceType}]`); switch (config.migration.sourceType) { case 'sql': From a798f3d133aa288624fb8e4b93e505a148c724db Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 19 Apr 2020 23:34:41 +0545 Subject: [PATCH 55/80] Improve debug logging --- src/migration/SqlMigrationContext.ts | 2 ++ src/service/execution.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/migration/SqlMigrationContext.ts b/src/migration/SqlMigrationContext.ts index 9629f9ea..c9e3a840 100644 --- a/src/migration/SqlMigrationContext.ts +++ b/src/migration/SqlMigrationContext.ts @@ -22,6 +22,8 @@ class SqlMigrationContext implements MigrationContext { this.list = list; this.log = logger; this.connectionId = ''; + + this.log('Instantiated SqlMigrationContext.'); } /** diff --git a/src/service/execution.ts b/src/service/execution.ts index c9e8424e..3a00d7b9 100644 --- a/src/service/execution.ts +++ b/src/service/execution.ts @@ -10,7 +10,7 @@ import { Promiser, runSequentially } from '../util/promise'; * @returns {Promise} */ export function executeProcesses(processes: Promiser[], config: SyncConfig): Promise { - log(`Execution Strategy: ${config.execution}`); + log(`Executing ${processes.length} processes [strategy=${config.execution}]`); switch (config.execution) { case 'sequential': From ef8416fa1a1468e49e6a97b49aeaeb8e7391bc03 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 19 Apr 2020 23:51:03 +0545 Subject: [PATCH 56/80] Rename interface SyncConfig -> Configuration --- src/api.ts | 36 +++++++++++++++---- src/config.ts | 14 ++++---- src/constants.ts | 4 +-- .../{SyncConfig.ts => Configuration.ts} | 6 ++-- src/domain/SyncContext.ts | 4 +-- src/init.ts | 13 ++++--- src/service/execution.ts | 6 ++-- src/service/knexMigrator.ts | 4 +-- src/service/migrator.ts | 6 ++-- 9 files changed, 60 insertions(+), 33 deletions(-) rename src/domain/{SyncConfig.ts => Configuration.ts} (79%) diff --git a/src/api.ts b/src/api.ts index ec450d9c..77aae78b 100644 --- a/src/api.ts +++ b/src/api.ts @@ -8,7 +8,7 @@ import { isKnexInstance, getConfig, createInstance } from './util/db'; import * as init from './init'; import SyncParams from './domain/SyncParams'; -import SyncConfig from './domain/SyncConfig'; +import Configuration from './domain/Configuration'; import SyncResult from './domain/SyncResult'; import ConnectionConfig from './domain/ConnectionConfig'; import ConnectionReference from './domain/ConnectionReference'; @@ -22,13 +22,13 @@ import { MigrationCommandParams, MigrationResult } from './service/knexMigrator' /** * Synchronize all the configured database connections. * - * @param {SyncConfig} config + * @param {Configuration} config * @param {(ConnectionConfig[] | ConnectionConfig | Knex[] | Knex)} conn * @param {SyncParams} [options] * @returns {Promise} */ export async function synchronize( - config: SyncConfig, + config: Configuration, conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, options?: SyncParams ): Promise { @@ -51,8 +51,16 @@ export async function synchronize( return executeProcesses(processes, config); } +/** + * Migrate Latest + * + * @param {Configuration} config + * @param {(ConnectionConfig[] | ConnectionConfig | Knex[] | Knex)} conn + * @param {MigrationCommandParams} [options] + * @returns {Promise} + */ export async function migrateLatest( - config: SyncConfig, + config: Configuration, conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, options?: MigrationCommandParams ): Promise { @@ -78,8 +86,16 @@ export async function migrateLatest( return executeProcesses(processes, config); } +/** + * Migrate Rollback. + * + * @param {Configuration} config + * @param {(ConnectionConfig[] | ConnectionConfig | Knex[] | Knex)} conn + * @param {MigrationCommandParams} [options] + * @returns {Promise} + */ export async function migrateRollback( - config: SyncConfig, + config: Configuration, conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, options?: MigrationCommandParams ): Promise { @@ -105,8 +121,16 @@ export async function migrateRollback( return executeProcesses(processes, config); } +/** + * List Migrations. + * + * @param {Configuration} config + * @param {(ConnectionConfig[] | ConnectionConfig | Knex[] | Knex)} conn + * @param {MigrationCommandParams} [options] + * @returns {Promise} + */ export async function migrateList( - config: SyncConfig, + config: Configuration, conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, options?: MigrationCommandParams ): Promise { diff --git a/src/config.ts b/src/config.ts index e991ede1..2805ab67 100644 --- a/src/config.ts +++ b/src/config.ts @@ -6,7 +6,7 @@ import * as fs from './util/fs'; import { log } from './util/logger'; import { isObject } from './util/types'; import DbConfig from './domain/DbConfig'; -import SyncConfig from './domain/SyncConfig'; +import Configuration from './domain/Configuration'; import ConnectionConfig from './domain/ConnectionConfig'; import { prepareInjectionConfigVars } from './service/configInjection'; import { DEFAULT_CONFIG, CONFIG_FILENAME, CONNECTIONS_FILENAME, REQUIRED_ENV_KEYS } from './constants'; @@ -23,17 +23,17 @@ export function isCLI(): boolean { /** * Load config yaml file. * - * @returns {Promise} + * @returns {Promise} */ -export async function loadConfig(): Promise { +export async function loadConfig(): Promise { log('Resolving config file.'); const filename = path.resolve(process.cwd(), CONFIG_FILENAME); - const loadedConfig = (await yaml.load(filename)) as SyncConfig; + const loadedConfig = (await yaml.load(filename)) as Configuration; log('Resolved config file.'); - const loaded = mergeDeepRight(DEFAULT_CONFIG, loadedConfig) as SyncConfig; + const loaded = mergeDeepRight(DEFAULT_CONFIG, loadedConfig) as Configuration; validate(loaded); @@ -53,9 +53,9 @@ export async function loadConfig(): Promise { /** * Validate the loaded configuration. * - * @param {SyncConfig} config + * @param {Configuration} config */ -export function validate(config: SyncConfig) { +export function validate(config: Configuration) { const { injectedConfig } = config; log('Validating config.'); diff --git a/src/constants.ts b/src/constants.ts index 892252eb..f94c9380 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -2,7 +2,7 @@ import * as path from 'path'; -import SyncConfig from './domain/SyncConfig'; +import Configuration from './domain/Configuration'; import SyncParams from './domain/SyncParams'; import ExecutionContext from './domain/ExecutionContext'; @@ -14,7 +14,7 @@ export const CONFIG_FILENAME = 'sync-db.yml'; export const CONNECTIONS_FILENAME = 'connections.sync-db.json'; export const INJECTED_CONFIG_TABLE = '__sync_db_injected_config'; -export const DEFAULT_CONFIG: SyncConfig = { +export const DEFAULT_CONFIG: Configuration = { basePath: path.resolve(process.cwd(), 'src/sql'), execution: 'parallel', sql: [], diff --git a/src/domain/SyncConfig.ts b/src/domain/Configuration.ts similarity index 79% rename from src/domain/SyncConfig.ts rename to src/domain/Configuration.ts index e25d5527..ee89f744 100644 --- a/src/domain/SyncConfig.ts +++ b/src/domain/Configuration.ts @@ -1,9 +1,9 @@ import Mapping from './Mapping'; /** - * Interface for synchronization configuration sycn-db.yml. + * Configuration schema interface. */ -interface SyncConfig { +interface Configuration { basePath: string; execution: 'parallel' | 'sequential'; sql: string[]; @@ -22,4 +22,4 @@ interface SyncConfig { }; } -export default SyncConfig; +export default Configuration; diff --git a/src/domain/SyncContext.ts b/src/domain/SyncContext.ts index 8f86f6b7..f5301263 100644 --- a/src/domain/SyncContext.ts +++ b/src/domain/SyncContext.ts @@ -1,11 +1,11 @@ -import SyncConfig from './SyncConfig'; +import Configuration from './Configuration'; import SyncParams from './SyncParams'; /** * Synchronize context parameters for the current database connection. */ interface SyncContext { - config: SyncConfig; + config: Configuration; connectionId: string; params: SyncParams; } diff --git a/src/init.ts b/src/init.ts index e8827121..f6483986 100644 --- a/src/init.ts +++ b/src/init.ts @@ -2,7 +2,7 @@ import Knex from 'knex'; import { log } from './util/logger'; import { validate, isCLI } from './config'; -import SyncConfig from './domain/SyncConfig'; +import Configuration from './domain/Configuration'; import * as migratorService from './service/migrator'; import MigrationContext from './domain/MigrationContext'; import KnexMigrationSource from './migration/KnexMigrationSource'; @@ -20,11 +20,11 @@ export interface PreparedRequirements { /** * Prepare configurations, preload requirements and validate before proceeding further. * - * @param {SyncConfig} config + * @param {Configuration} config * @param {PrepareOptions} options * @returns {Promise} */ -export async function prepare(config: SyncConfig, options: PrepareOptions): Promise { +export async function prepare(config: Configuration, options: PrepareOptions): Promise { log('Prepare: ', options); // Validate the config for programmatic API access. @@ -46,11 +46,14 @@ export async function prepare(config: SyncConfig, options: PrepareOptions): Prom /** * Resolve migration context based on the migration configuration. * - * @param {SyncConfig} config + * @param {Configuration} config * @param {PrepareOptions} options * @returns {(Promise)} */ -async function resolveMigrationContext(config: SyncConfig, options: PrepareOptions): Promise { +async function resolveMigrationContext( + config: Configuration, + options: PrepareOptions +): Promise { if (options.loadMigrations !== true) { return null; } diff --git a/src/service/execution.ts b/src/service/execution.ts index 3a00d7b9..61560c06 100644 --- a/src/service/execution.ts +++ b/src/service/execution.ts @@ -1,15 +1,15 @@ import { log } from '../util/logger'; -import SyncConfig from '../domain/SyncConfig'; +import Configuration from '../domain/Configuration'; import { Promiser, runSequentially } from '../util/promise'; /** * Execute a list of processes according to the configuration. * * @param {Promiser[]} processes - * @param {SyncConfig} config + * @param {Configuration} config * @returns {Promise} */ -export function executeProcesses(processes: Promiser[], config: SyncConfig): Promise { +export function executeProcesses(processes: Promiser[], config: Configuration): Promise { log(`Executing ${processes.length} processes [strategy=${config.execution}]`); switch (config.execution) { diff --git a/src/service/knexMigrator.ts b/src/service/knexMigrator.ts index 86808356..9ebfe91c 100644 --- a/src/service/knexMigrator.ts +++ b/src/service/knexMigrator.ts @@ -3,7 +3,7 @@ import Knex from 'knex'; import { isCLI } from '../config'; import { dbLogger } from '../util/logger'; import { getElapsedTime } from '../util/ts'; -import SyncConfig from '../domain/SyncConfig'; +import Configuration from '../domain/Configuration'; export interface MigrationResult { connectionId: string; @@ -19,7 +19,7 @@ export interface MigrationCommandParams { } export interface MigrationCommandContext { - config: SyncConfig; + config: Configuration; connectionId: string; params: MigrationCommandParams; knexMigrationConfig: Knex.MigratorConfig; diff --git a/src/service/migrator.ts b/src/service/migrator.ts index 3a58444e..f199f7cc 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -3,7 +3,7 @@ import * as path from 'path'; import { glob, exists } from '../util/fs'; import { resolveFile } from './sqlRunner'; import SqlMigrationEntry from '../domain/SqlMigrationEntry'; -import SyncConfig from '../domain/SyncConfig'; +import Configuration from '../domain/Configuration'; const FILE_PATTERN = /(.+)\.(up|down)\.sql$/; @@ -35,10 +35,10 @@ export async function getSqlMigrationNames(migrationPath: string): Promise} */ -export async function resolveSqlMigrations(config: SyncConfig): Promise { +export async function resolveSqlMigrations(config: Configuration): Promise { const { basePath, migration } = config; const migrationPath = path.join(basePath, migration.directory); const migrationNames = await getSqlMigrationNames(migrationPath); From ce220b14c9cd250e0f7658889cc672b747cd3a74 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sun, 19 Apr 2020 23:58:03 +0545 Subject: [PATCH 57/80] Rename SyncParams -> SynchronizeParams --- src/api.ts | 6 +++--- src/cli.ts | 4 ++-- src/commands/synchronize.ts | 6 +++--- src/constants.ts | 4 ++-- src/domain/SyncContext.ts | 4 ++-- src/domain/SyncParams.ts | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/api.ts b/src/api.ts index 77aae78b..a448de91 100644 --- a/src/api.ts +++ b/src/api.ts @@ -7,7 +7,7 @@ import { DEFAULT_SYNC_PARAMS } from './constants'; import { isKnexInstance, getConfig, createInstance } from './util/db'; import * as init from './init'; -import SyncParams from './domain/SyncParams'; +import SynchronizeParams from './domain/SyncParams'; import Configuration from './domain/Configuration'; import SyncResult from './domain/SyncResult'; import ConnectionConfig from './domain/ConnectionConfig'; @@ -24,13 +24,13 @@ import { MigrationCommandParams, MigrationResult } from './service/knexMigrator' * * @param {Configuration} config * @param {(ConnectionConfig[] | ConnectionConfig | Knex[] | Knex)} conn - * @param {SyncParams} [options] + * @param {SynchronizeParams} [options] * @returns {Promise} */ export async function synchronize( config: Configuration, conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, - options?: SyncParams + options?: SynchronizeParams ): Promise { log('Synchronize'); diff --git a/src/cli.ts b/src/cli.ts index d413f086..5fb3d2c6 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -5,7 +5,7 @@ import { printLine } from './util/io'; import SyncDbOptions from './domain/SyncDbOptions'; import { CONNECTIONS_FILENAME } from './constants'; import { resolveConnectionsFromEnv } from './config'; -import SyncParams from './domain/SyncParams'; +import SynchronizeParams from './domain/SyncParams'; /** * Generates connections.sync-db.json file. @@ -29,7 +29,7 @@ async function generateConnection(): Promise { * @param {SyncDbOptions} flags * @returns {Promise} */ -export async function handleFlags(flags: SyncDbOptions, params: SyncParams): Promise { +export async function handleFlags(flags: SyncDbOptions, params: SynchronizeParams): Promise { if (flags['generate-connections']) { await generateConnection(); process.exit(0); diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index 869619b3..383dfbf1 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -4,7 +4,7 @@ import { log } from '../util/logger'; import { handleFlags } from '../cli'; import { getElapsedTime } from '../util/ts'; import SyncResult from '../domain/SyncResult'; -import SyncParams from '../domain/SyncParams'; +import SynchronizeParams from '../domain/SyncParams'; import { printError, printLine } from '../util/io'; import ExecutionContext from '../domain/ExecutionContext'; import { loadConfig, resolveConnections } from '../config'; @@ -29,9 +29,9 @@ class Synchronize extends Command { * Default CLI options for running synchronize. * * @param {*} userParams - * @returns {SyncParams} + * @returns {SynchronizeParams} */ - getSyncParams(userParams: any): SyncParams { + getSyncParams(userParams: any): SynchronizeParams { return { ...userParams, // Individual success handler diff --git a/src/constants.ts b/src/constants.ts index f94c9380..d6878cfc 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -3,7 +3,7 @@ import * as path from 'path'; import Configuration from './domain/Configuration'; -import SyncParams from './domain/SyncParams'; +import SynchronizeParams from './domain/SyncParams'; import ExecutionContext from './domain/ExecutionContext'; // General constants @@ -32,7 +32,7 @@ export const DEFAULT_CONFIG: Configuration = { } }; -export const DEFAULT_SYNC_PARAMS: SyncParams = { +export const DEFAULT_SYNC_PARAMS: SynchronizeParams = { force: false, onSuccess: (context: ExecutionContext) => Promise.resolve(), onFailed: (context: ExecutionContext) => Promise.resolve() diff --git a/src/domain/SyncContext.ts b/src/domain/SyncContext.ts index f5301263..15313edd 100644 --- a/src/domain/SyncContext.ts +++ b/src/domain/SyncContext.ts @@ -1,5 +1,5 @@ import Configuration from './Configuration'; -import SyncParams from './SyncParams'; +import SynchronizeParams from './SyncParams'; /** * Synchronize context parameters for the current database connection. @@ -7,7 +7,7 @@ import SyncParams from './SyncParams'; interface SyncContext { config: Configuration; connectionId: string; - params: SyncParams; + params: SynchronizeParams; } export default SyncContext; diff --git a/src/domain/SyncParams.ts b/src/domain/SyncParams.ts index 9b1624a0..a5e6c321 100644 --- a/src/domain/SyncParams.ts +++ b/src/domain/SyncParams.ts @@ -3,10 +3,10 @@ import ExecutionContext from './ExecutionContext'; /** * Synchronize parameters. */ -interface SyncParams { +interface SynchronizeParams { force: boolean; onSuccess: (context: ExecutionContext) => Promise; onFailed: (context: ExecutionContext) => Promise; } -export default SyncParams; +export default SynchronizeParams; From f5c887b8c20f679b951b2f6488692b01c9418df1 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Mon, 20 Apr 2020 00:06:38 +0545 Subject: [PATCH 58/80] Fix tests --- src/api.ts | 2 +- src/cli.ts | 2 +- src/commands/synchronize.ts | 2 +- src/constants.ts | 2 +- src/domain/SyncContext.ts | 2 +- src/domain/{SyncParams.ts => SynchronizeParams.ts} | 0 src/init.ts | 6 +++++- src/service/migrator.ts | 7 ++----- test/migration/KnexMigrationSource.test.ts | 6 ++++++ test/migration/SqlMigrationContext.test.ts | 2 +- test/service/execution.test.ts | 6 +++--- 11 files changed, 22 insertions(+), 15 deletions(-) rename src/domain/{SyncParams.ts => SynchronizeParams.ts} (100%) diff --git a/src/api.ts b/src/api.ts index a448de91..b205d8a2 100644 --- a/src/api.ts +++ b/src/api.ts @@ -7,7 +7,7 @@ import { DEFAULT_SYNC_PARAMS } from './constants'; import { isKnexInstance, getConfig, createInstance } from './util/db'; import * as init from './init'; -import SynchronizeParams from './domain/SyncParams'; +import SynchronizeParams from './domain/SynchronizeParams'; import Configuration from './domain/Configuration'; import SyncResult from './domain/SyncResult'; import ConnectionConfig from './domain/ConnectionConfig'; diff --git a/src/cli.ts b/src/cli.ts index 5fb3d2c6..7085e22e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -5,7 +5,7 @@ import { printLine } from './util/io'; import SyncDbOptions from './domain/SyncDbOptions'; import { CONNECTIONS_FILENAME } from './constants'; import { resolveConnectionsFromEnv } from './config'; -import SynchronizeParams from './domain/SyncParams'; +import SynchronizeParams from './domain/SynchronizeParams'; /** * Generates connections.sync-db.json file. diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index 383dfbf1..a21883ee 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -4,7 +4,7 @@ import { log } from '../util/logger'; import { handleFlags } from '../cli'; import { getElapsedTime } from '../util/ts'; import SyncResult from '../domain/SyncResult'; -import SynchronizeParams from '../domain/SyncParams'; +import SynchronizeParams from '../domain/SynchronizeParams'; import { printError, printLine } from '../util/io'; import ExecutionContext from '../domain/ExecutionContext'; import { loadConfig, resolveConnections } from '../config'; diff --git a/src/constants.ts b/src/constants.ts index d6878cfc..6e683974 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -3,7 +3,7 @@ import * as path from 'path'; import Configuration from './domain/Configuration'; -import SynchronizeParams from './domain/SyncParams'; +import SynchronizeParams from './domain/SynchronizeParams'; import ExecutionContext from './domain/ExecutionContext'; // General constants diff --git a/src/domain/SyncContext.ts b/src/domain/SyncContext.ts index 15313edd..3d27c762 100644 --- a/src/domain/SyncContext.ts +++ b/src/domain/SyncContext.ts @@ -1,5 +1,5 @@ import Configuration from './Configuration'; -import SynchronizeParams from './SyncParams'; +import SynchronizeParams from './SynchronizeParams'; /** * Synchronize context parameters for the current database connection. diff --git a/src/domain/SyncParams.ts b/src/domain/SynchronizeParams.ts similarity index 100% rename from src/domain/SyncParams.ts rename to src/domain/SynchronizeParams.ts diff --git a/src/init.ts b/src/init.ts index f6483986..faa7470b 100644 --- a/src/init.ts +++ b/src/init.ts @@ -1,4 +1,5 @@ import Knex from 'knex'; +import * as path from 'path'; import { log } from './util/logger'; import { validate, isCLI } from './config'; @@ -60,9 +61,12 @@ async function resolveMigrationContext( log(`Initialize migration context [sourceType=${config.migration.sourceType}]`); + const { basePath, migration } = config; + const migrationPath = path.join(basePath, migration.directory); + switch (config.migration.sourceType) { case 'sql': - const src = await migratorService.resolveSqlMigrations(config); + const src = await migratorService.resolveSqlMigrations(migrationPath); log('Available migration sources:\n%O', src); diff --git a/src/service/migrator.ts b/src/service/migrator.ts index f199f7cc..e6dbb183 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -3,7 +3,6 @@ import * as path from 'path'; import { glob, exists } from '../util/fs'; import { resolveFile } from './sqlRunner'; import SqlMigrationEntry from '../domain/SqlMigrationEntry'; -import Configuration from '../domain/Configuration'; const FILE_PATTERN = /(.+)\.(up|down)\.sql$/; @@ -35,12 +34,10 @@ export async function getSqlMigrationNames(migrationPath: string): Promise} */ -export async function resolveSqlMigrations(config: Configuration): Promise { - const { basePath, migration } = config; - const migrationPath = path.join(basePath, migration.directory); +export async function resolveSqlMigrations(migrationPath: string): Promise { const migrationNames = await getSqlMigrationNames(migrationPath); const migrationPromises = migrationNames.map(async name => { const upFilename = `${name}.up.sql`; diff --git a/test/migration/KnexMigrationSource.test.ts b/test/migration/KnexMigrationSource.test.ts index 0df6c260..ecd100d2 100644 --- a/test/migration/KnexMigrationSource.test.ts +++ b/test/migration/KnexMigrationSource.test.ts @@ -9,6 +9,12 @@ describe('UTIL: KnexMigrationSource', () => { const migrationContext: MigrationContext = new (class { connectionId = 'testdb1'; + bind(connectionId: string) { + this.connectionId = connectionId; + + return this; + } + keys(): string[] { return ['mgr_001', 'mgr_002', 'mgr_003', 'mgr_004']; } diff --git a/test/migration/SqlMigrationContext.test.ts b/test/migration/SqlMigrationContext.test.ts index 215f1416..2a985078 100644 --- a/test/migration/SqlMigrationContext.test.ts +++ b/test/migration/SqlMigrationContext.test.ts @@ -5,7 +5,7 @@ import SqlMigrationEntry from '../../src/domain/SqlMigrationEntry'; import SqlMigrationContext from '../../src/migration/SqlMigrationContext'; describe('UTIL: SqlMigrationContext', () => { - const getInstance = (list: SqlMigrationEntry[]) => new SqlMigrationContext('testdb1', list); + const getInstance = (list: SqlMigrationEntry[]) => new SqlMigrationContext(list); describe('keys', () => { it('should return an empty list if migrations are empty.', () => { diff --git a/test/service/execution.test.ts b/test/service/execution.test.ts index adf19681..d598d83e 100644 --- a/test/service/execution.test.ts +++ b/test/service/execution.test.ts @@ -2,7 +2,7 @@ import { expect } from 'chai'; import { it, describe } from 'mocha'; import { timeout } from '../../src/util/promise'; -import SyncConfig from '../../src/domain/SyncConfig'; +import Configuration from '../../src/domain/Configuration'; import * as executionService from '../../src/service/execution'; const getProcesses = (tracker: string[]) => [ @@ -32,7 +32,7 @@ describe('SERVICE: execution', () => { const result = await executionService.executeProcesses(processes, { execution: 'sequential' - } as SyncConfig); + } as Configuration); expect(result).to.deep.equal(['Task A', 'Task B', 'Task C', 'Task D']); expect(tracker).to.deep.equal(['Task A', 'Task B', 'Task C', 'Task D']); @@ -44,7 +44,7 @@ describe('SERVICE: execution', () => { const result = await executionService.executeProcesses(processes, { execution: 'parallel' - } as SyncConfig); + } as Configuration); expect(result).to.deep.equal(['Task A', 'Task B', 'Task C', 'Task D']); expect(tracker).to.deep.equal(['Task C', 'Task D', 'Task A', 'Task B']); From c27b31b56db9b7ca90852c92adcd71085cc35812 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Mon, 20 Apr 2020 00:28:08 +0545 Subject: [PATCH 59/80] Run migrations on sync unless skipped --- src/api.ts | 22 ++++++++++++++++++---- src/commands/synchronize.ts | 1 + src/constants.ts | 1 + src/domain/SyncContext.ts | 4 +++- src/domain/SynchronizeParams.ts | 1 + src/service/sync.ts | 10 +++++++++- 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/api.ts b/src/api.ts index b205d8a2..c15d86e5 100644 --- a/src/api.ts +++ b/src/api.ts @@ -34,17 +34,31 @@ export async function synchronize( ): Promise { log('Synchronize'); - // Note: This does nothing right now. + const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); + // TODO: Need to preload the SQL source code under this step. - await init.prepare(config, { loadSqlSources: true }); + const { knexMigrationConfig } = await init.prepare(config, { + loadSqlSources: true, + loadMigrations: !params['skip-migration'] + }); const connections = mapToConnectionReferences(conn); - const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); const processes = connections.map(({ connection, id: connectionId }) => () => synchronizeDatabase(connection, { config, params, - connectionId + connectionId, + migrateFunc: (trx: Knex.Transaction) => + runMigrateFunc( + trx, + { + config, + params, + connectionId, + knexMigrationConfig: knexMigrationConfig(connectionId) + }, + migrationApiMap['migrate.latest'] + ) }) ); diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index a21883ee..e8c04686 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -22,6 +22,7 @@ class Synchronize extends Command { version: flags.version({ char: 'v', description: 'Print version', name: 'sync-db' }), help: flags.help({ char: 'h', description: 'Print help information' }), force: flags.boolean({ char: 'f', description: 'Force synchronization' }), + 'skip-migration': flags.boolean({ description: 'Skip running migrations.' }), 'generate-connections': flags.boolean({ char: 'c', description: 'Generate connections' }) }; diff --git a/src/constants.ts b/src/constants.ts index 6e683974..ef6ec4df 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -34,6 +34,7 @@ export const DEFAULT_CONFIG: Configuration = { export const DEFAULT_SYNC_PARAMS: SynchronizeParams = { force: false, + 'skip-migration': false, onSuccess: (context: ExecutionContext) => Promise.resolve(), onFailed: (context: ExecutionContext) => Promise.resolve() }; diff --git a/src/domain/SyncContext.ts b/src/domain/SyncContext.ts index 3d27c762..a2aa10be 100644 --- a/src/domain/SyncContext.ts +++ b/src/domain/SyncContext.ts @@ -1,13 +1,15 @@ import Configuration from './Configuration'; import SynchronizeParams from './SynchronizeParams'; +import Knex from 'knex'; /** - * Synchronize context parameters for the current database connection. + * Synchronize context for a database connection. */ interface SyncContext { config: Configuration; connectionId: string; params: SynchronizeParams; + migrateFunc: (trx: Knex.Transaction) => Promise; } export default SyncContext; diff --git a/src/domain/SynchronizeParams.ts b/src/domain/SynchronizeParams.ts index a5e6c321..68f95450 100644 --- a/src/domain/SynchronizeParams.ts +++ b/src/domain/SynchronizeParams.ts @@ -5,6 +5,7 @@ import ExecutionContext from './ExecutionContext'; */ interface SynchronizeParams { force: boolean; + 'skip-migration': boolean; onSuccess: (context: ExecutionContext) => Promise; onFailed: (context: ExecutionContext) => Promise; } diff --git a/src/service/sync.ts b/src/service/sync.ts index 807298ca..48258736 100644 --- a/src/service/sync.ts +++ b/src/service/sync.ts @@ -88,7 +88,7 @@ async function teardown(trx: Knex.Transaction, context: SyncContext): Promise} */ export async function synchronizeDatabase(connection: Knex, context: SyncContext): Promise { - const { connectionId } = context; + const { connectionId, migrateFunc } = context; const log = dbLogger(connectionId); const result: SyncResult = { connectionId, success: false }; @@ -100,6 +100,14 @@ export async function synchronizeDatabase(connection: Knex, context: SyncContext // Run the process in a single transaction for a database connection. await connection.transaction(async trx => { await teardown(trx, context); + + if (context.params['skip-migration']) { + log('Skipped migrations.'); + } else { + log('Running migrations.'); + await migrateFunc(trx); + } + await setup(trx, context); }); From 70d2931f4a176b0e20ec6297f444a3effc24062b Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Wed, 22 Apr 2020 00:43:01 +0545 Subject: [PATCH 60/80] Make handlers optional --- src/service/knexMigrator.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/service/knexMigrator.ts b/src/service/knexMigrator.ts index 9ebfe91c..43696d85 100644 --- a/src/service/knexMigrator.ts +++ b/src/service/knexMigrator.ts @@ -1,6 +1,5 @@ import Knex from 'knex'; -import { isCLI } from '../config'; import { dbLogger } from '../util/logger'; import { getElapsedTime } from '../util/ts'; import Configuration from '../domain/Configuration'; @@ -14,8 +13,8 @@ export interface MigrationResult { } export interface MigrationCommandParams { - onSuccess: (result: MigrationResult) => Promise; - onFailed: (context: MigrationResult) => Promise; + onSuccess?: (result: MigrationResult) => Promise; + onFailed?: (context: MigrationResult) => Promise; } export interface MigrationCommandContext { @@ -79,11 +78,11 @@ export async function runMigrateFunc( success: !error }; - // If it's a CLI environment, invoke the handler. - if (isCLI()) { - const handler = result.success ? context.params.onSuccess : context.params.onFailed; - - await handler(result); + // Invoke corresponding handlers if they're sent. + if (result.success && context.params.onSuccess) { + await context.params.onSuccess(result); + } else if (!result.success && context.params.onFailed) { + await context.params.onFailed(result); } return result; From 798dd0c572ac864b3f59544337478b721d0ccd32 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Wed, 22 Apr 2020 00:53:24 +0545 Subject: [PATCH 61/80] Reorganize command classes --- src/commands/migrate-latest.ts | 60 +++++++++--------- src/commands/migrate-list.ts | 84 +++++++++++++------------ src/commands/migrate-rollback.ts | 60 +++++++++--------- src/commands/synchronize.ts | 103 +++++++++++++++---------------- 4 files changed, 157 insertions(+), 150 deletions(-) diff --git a/src/commands/migrate-latest.ts b/src/commands/migrate-latest.ts index 7a88035f..5b42b988 100644 --- a/src/commands/migrate-latest.ts +++ b/src/commands/migrate-latest.ts @@ -3,7 +3,7 @@ import { bold, red, cyan } from 'chalk'; import { printLine, printError, printInfo } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { MigrationCommandParams, MigrationResult } from '../service/knexMigrator'; +import { MigrationResult } from '../service/knexMigrator'; import { dbLogger } from '../util/logger'; /** @@ -12,37 +12,40 @@ import { dbLogger } from '../util/logger'; class MigrateLatest extends Command { static description = 'Run the migrations up to the latest changes.'; - getParams(): MigrationCommandParams { - return { - onSuccess: async (result: MigrationResult) => { - const log = dbLogger(result.connectionId); - const [num, list] = result.data; - const alreadyUpToDate = num && list.length === 0; + /** + * Success handler for each connection. + */ + onSuccess = async (result: MigrationResult) => { + const log = dbLogger(result.connectionId); + const [num, list] = result.data; + const alreadyUpToDate = num && list.length === 0; - log('Up to date: ', alreadyUpToDate); + log('Up to date: ', alreadyUpToDate); - await printLine(bold(` ▸ ${result.connectionId} - Successful`) + ` (${result.timeElapsed}s)`); + await printLine(bold(` ▸ ${result.connectionId} - Successful`) + ` (${result.timeElapsed}s)`); - if (alreadyUpToDate) { - await printInfo(' Already up to date.\n'); + if (alreadyUpToDate) { + await printInfo(' Already up to date.\n'); - return; - } + return; + } - // Completed migrations. - for (const item of list) { - await printLine(cyan(` - ${item}`)); - } + // Completed migrations. + for (const item of list) { + await printLine(cyan(` - ${item}`)); + } - await printInfo(`\n Ran ${list.length} migrations.\n`); - }, - onFailed: async (result: MigrationResult) => { - printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); + await printInfo(`\n Ran ${list.length} migrations.\n`); + }; - await printError(` ${result.error}\n`); - } - }; - } + /** + * Failure handler for each connection. + */ + onFailed = async (result: MigrationResult) => { + printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); + + await printError(` ${result.error}\n`); + }; /** * CLI command execution handler. @@ -50,13 +53,14 @@ class MigrateLatest extends Command { * @returns {Promise} */ async run(): Promise { - const params = this.getParams(); - const config = await loadConfig(); const connections = await resolveConnections(); const { migrateLatest } = await import('../api'); - const results = await migrateLatest(config, connections, params); + const results = await migrateLatest(config, connections, { + onSuccess: this.onSuccess, + onFailed: this.onFailed + }); const failedCount = results.filter(({ success }) => !success).length; diff --git a/src/commands/migrate-list.ts b/src/commands/migrate-list.ts index 9c101db4..4ef37d89 100644 --- a/src/commands/migrate-list.ts +++ b/src/commands/migrate-list.ts @@ -3,7 +3,7 @@ import { bold, grey, red, cyan, yellow } from 'chalk'; import { printLine, printError } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { MigrationResult, MigrationCommandParams } from '../service/knexMigrator'; +import { MigrationResult } from '../service/knexMigrator'; /** * Migration command handler. @@ -11,42 +11,45 @@ import { MigrationResult, MigrationCommandParams } from '../service/knexMigrator class MigrateList extends Command { static description = 'List migrations.'; - getParams(): MigrationCommandParams { - return { - onSuccess: async (result: MigrationResult) => { - await printLine(bold(` ▸ ${result.connectionId}`)); - - const [list1, list2] = result.data; - const ranCount = list1.length; - const remainingCount = list2.length; - - // Completed migrations. - for (const item of list1) { - await printLine(cyan(` • ${item}`)); - } - - // Remaining Migrations - for (const item of list2) { - await printLine(grey(` - ${item}`)); - } - - if (ranCount === 0 && remainingCount === 0) { - await printLine(yellow(' No migrations.')); - } else if (remainingCount > 0) { - await printLine(yellow(`\n ${list2.length} migrations yet to be run.`)); - } else if (remainingCount === 0) { - await printLine('\n All up to date.'); - } - - await printLine(); - }, - onFailed: async (result: MigrationResult) => { - printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); - - await printError(` ${result.error}\n`); - } - }; - } + /** + * Success handler for a connection. + */ + onSuccess = async (result: MigrationResult) => { + await printLine(bold(` ▸ ${result.connectionId}`)); + + const [list1, list2] = result.data; + const ranCount = list1.length; + const remainingCount = list2.length; + + // Completed migrations. + for (const item of list1) { + await printLine(cyan(` • ${item}`)); + } + + // Remaining Migrations + for (const item of list2) { + await printLine(grey(` - ${item}`)); + } + + if (ranCount === 0 && remainingCount === 0) { + await printLine(yellow(' No migrations.')); + } else if (remainingCount > 0) { + await printLine(yellow(`\n ${list2.length} migrations yet to be run.`)); + } else if (remainingCount === 0) { + await printLine('\n All up to date.'); + } + + await printLine(); + }; + + /** + * Failure handler for a connection. + */ + onFailed = async (result: MigrationResult) => { + printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); + + await printError(` ${result.error}\n`); + }; /** * CLI command execution handler. @@ -54,13 +57,14 @@ class MigrateList extends Command { * @returns {Promise} */ async run(): Promise { - const params = this.getParams(); - const config = await loadConfig(); const connections = await resolveConnections(); const { migrateList } = await import('../api'); - const results = await migrateList(config, connections, params); + const results = await migrateList(config, connections, { + onSuccess: this.onSuccess, + onFailed: this.onFailed + }); const failedCount = results.filter(({ success }) => !success).length; diff --git a/src/commands/migrate-rollback.ts b/src/commands/migrate-rollback.ts index fb113123..35e2c737 100644 --- a/src/commands/migrate-rollback.ts +++ b/src/commands/migrate-rollback.ts @@ -3,7 +3,7 @@ import { bold, red, cyan } from 'chalk'; import { printLine, printError, printInfo } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { MigrationCommandParams, MigrationResult } from '../service/knexMigrator'; +import { MigrationResult } from '../service/knexMigrator'; import { dbLogger } from '../util/logger'; /** @@ -12,37 +12,40 @@ import { dbLogger } from '../util/logger'; class MigrateRollback extends Command { static description = 'Rollback migrations up to the last run batch.'; - getParams(): MigrationCommandParams { - return { - onSuccess: async (result: MigrationResult) => { - const log = dbLogger(result.connectionId); - const [num, list] = result.data; - const allRolledBack = num === 0; + /** + * Success handler for each connection. + */ + onSuccess = async (result: MigrationResult) => { + const log = dbLogger(result.connectionId); + const [num, list] = result.data; + const allRolledBack = num === 0; - log('Already on the top of migrations: ', allRolledBack); + log('Already on the top of migrations: ', allRolledBack); - await printLine(bold(` ▸ ${result.connectionId} - Successful`) + ` (${result.timeElapsed}s)`); + await printLine(bold(` ▸ ${result.connectionId} - Successful`) + ` (${result.timeElapsed}s)`); - if (allRolledBack) { - await printLine(' No more migrations to rollback.\n'); + if (allRolledBack) { + await printLine(' No more migrations to rollback.\n'); - return; - } + return; + } - // Completed migrations. - for (const item of list) { - await printLine(cyan(` - ${item}`)); - } + // Completed migrations. + for (const item of list) { + await printLine(cyan(` - ${item}`)); + } - await printInfo(`\n Rolled back ${list.length} migrations.\n`); - }, - onFailed: async (result: MigrationResult) => { - printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); + await printInfo(`\n Rolled back ${list.length} migrations.\n`); + }; - await printError(` ${result.error}\n`); - } - }; - } + /** + * Failure handler for each connection. + */ + onFailed = async (result: MigrationResult) => { + printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); + + await printError(` ${result.error}\n`); + }; /** * CLI command execution handler. @@ -50,13 +53,14 @@ class MigrateRollback extends Command { * @returns {Promise} */ async run(): Promise { - const params = this.getParams(); - const config = await loadConfig(); const connections = await resolveConnections(); const { migrateRollback } = await import('../api'); - const results = await migrateRollback(config, connections, params); + const results = await migrateRollback(config, connections, { + onSuccess: this.onSuccess, + onFailed: this.onFailed + }); const failedCount = results.filter(({ success }) => !success).length; diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index e8c04686..78390d7d 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -4,7 +4,6 @@ import { log } from '../util/logger'; import { handleFlags } from '../cli'; import { getElapsedTime } from '../util/ts'; import SyncResult from '../domain/SyncResult'; -import SynchronizeParams from '../domain/SynchronizeParams'; import { printError, printLine } from '../util/io'; import ExecutionContext from '../domain/ExecutionContext'; import { loadConfig, resolveConnections } from '../config'; @@ -27,22 +26,51 @@ class Synchronize extends Command { }; /** - * Default CLI options for running synchronize. + * Success handler for each connection. + */ + onSuccess = (context: ExecutionContext) => + printLine(` [✓] ${context.connectionId} - Successful (${context.timeElapsed}s)`); + + /** + * Failure handler for each connection. + */ + onFailed = (context: ExecutionContext) => + printLine(` [✖] ${context.connectionId} - Failed (${context.timeElapsed}s)`); + + /** + * Check the results for each connection and display them. + * All the successful / failed attempts are displayed and errors are logged. * - * @param {*} userParams - * @returns {SynchronizeParams} + * @param {SyncResult[]} results + * @returns {Promise<{ totalCount: number, failedCount: number, successfulCount: number }>} */ - getSyncParams(userParams: any): SynchronizeParams { - return { - ...userParams, - // Individual success handler - onSuccess: (context: ExecutionContext) => - printLine(` [✓] ${context.connectionId} - Successful (${context.timeElapsed}s)`), - - // Individual error handler - onFailed: (context: ExecutionContext) => - printLine(` [✖] ${context.connectionId} - Failed (${context.timeElapsed}s)`) - }; + async processResults( + results: SyncResult[] + ): Promise<{ totalCount: number; failedCount: number; successfulCount: number }> { + const totalCount = results.length; + const failedAttempts = results.filter(result => !result.success); + const successfulCount = totalCount - failedAttempts.length; + const failedCount = totalCount - successfulCount; + const allComplete = failedCount === 0; + + await printLine(); + + // If there are errors, display all of them. + if (!allComplete) { + await printLine(`Synchronization failed for ${failedCount} connection(s):\n`); + + failedAttempts.forEach(async (attempt, index) => { + await printLine(`${index + 1}) ${attempt.connectionId}`); + await printError(attempt.error.toString()); + + // Send verbose error with stack trace to debug logs. + log(attempt.error); + + await printLine(); + }); + } + + return { totalCount, failedCount, successfulCount }; } /** @@ -52,9 +80,11 @@ class Synchronize extends Command { */ async run(): Promise { const { flags: parsedFlags } = this.parse(Synchronize); - const params = this.getSyncParams({ ...parsedFlags }); - - console.log({ parsedFlags }); // tslint:disable-line + const params = { + ...parsedFlags, + onSuccess: this.onSuccess, + onFailed: this.onFailed + }; try { await handleFlags(parsedFlags, params); @@ -67,6 +97,7 @@ class Synchronize extends Command { await printLine('Synchronizing...\n'); const results = await synchronize(config, connections, params); + const { totalCount, failedCount, successfulCount } = await this.processResults(results); if (successfulCount > 0) { @@ -92,42 +123,6 @@ class Synchronize extends Command { process.exit(-1); } } - - /** - * Check the results for each connection and display them. - * All the successful / failed attempts are displayed and errors are logged. - * - * @param {SyncResult[]} results - * @returns {Promise<{ totalCount: number, failedCount: number, successfulCount: number }>} - */ - async processResults( - results: SyncResult[] - ): Promise<{ totalCount: number; failedCount: number; successfulCount: number }> { - const totalCount = results.length; - const failedAttempts = results.filter(result => !result.success); - const successfulCount = totalCount - failedAttempts.length; - const failedCount = totalCount - successfulCount; - const allComplete = failedCount === 0; - - await printLine(); - - // If there are errors, display all of them. - if (!allComplete) { - await printLine(`Synchronization failed for ${failedCount} connection(s):\n`); - - failedAttempts.forEach(async (attempt, index) => { - await printLine(`${index + 1}) ${attempt.connectionId}`); - await printError(attempt.error.toString()); - - // Send verbose error with stack trace to debug logs. - log(attempt.error); - - await printLine(); - }); - } - - return { totalCount, failedCount, successfulCount }; - } } export default Synchronize; From 793aac249b7c0d4503a35b84fcc43618e19b0b23 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Wed, 22 Apr 2020 02:10:31 +0545 Subject: [PATCH 62/80] Handle cases gracefully --- src/api.ts | 2 +- src/domain/ExecutionContext.ts | 1 + src/domain/SynchronizeParams.ts | 9 ++++++-- src/service/sync.ts | 41 ++++++++++++++++++++++++--------- 4 files changed, 39 insertions(+), 14 deletions(-) diff --git a/src/api.ts b/src/api.ts index c15d86e5..5bd18043 100644 --- a/src/api.ts +++ b/src/api.ts @@ -53,8 +53,8 @@ export async function synchronize( trx, { config, - params, connectionId, + params: { ...params, onSuccess: params.onMigrationSuccess, onFailed: params.onMigrationFailed }, knexMigrationConfig: knexMigrationConfig(connectionId) }, migrationApiMap['migrate.latest'] diff --git a/src/domain/ExecutionContext.ts b/src/domain/ExecutionContext.ts index 4b1c0e8d..9d9c1340 100644 --- a/src/domain/ExecutionContext.ts +++ b/src/domain/ExecutionContext.ts @@ -5,6 +5,7 @@ interface ExecutionContext { connectionId: string; success: boolean; timeElapsed: number; + error?: any; } export default ExecutionContext; diff --git a/src/domain/SynchronizeParams.ts b/src/domain/SynchronizeParams.ts index 68f95450..fdc09f80 100644 --- a/src/domain/SynchronizeParams.ts +++ b/src/domain/SynchronizeParams.ts @@ -1,4 +1,5 @@ import ExecutionContext from './ExecutionContext'; +import { MigrationResult } from '../service/knexMigrator'; /** * Synchronize parameters. @@ -6,8 +7,12 @@ import ExecutionContext from './ExecutionContext'; interface SynchronizeParams { force: boolean; 'skip-migration': boolean; - onSuccess: (context: ExecutionContext) => Promise; - onFailed: (context: ExecutionContext) => Promise; + onStarted?: (context: ExecutionContext) => Promise; + onSuccess?: (context: ExecutionContext) => Promise; + onTeardownSuccess?: (context: ExecutionContext) => Promise; + onFailed?: (context: ExecutionContext) => Promise; + onMigrationSuccess?: (result: MigrationResult) => Promise; + onMigrationFailed?: (result: MigrationResult) => Promise; } export default SynchronizeParams; diff --git a/src/service/sync.ts b/src/service/sync.ts index 48258736..1ed4f6f1 100644 --- a/src/service/sync.ts +++ b/src/service/sync.ts @@ -1,6 +1,5 @@ import * as Knex from 'knex'; -import { isCLI } from '../config'; import * as sqlRunner from './sqlRunner'; import { dbLogger } from '../util/logger'; import { getElapsedTime } from '../util/ts'; @@ -97,10 +96,28 @@ export async function synchronizeDatabase(connection: Knex, context: SyncContext try { log('Starting synchronization.'); + // Trigger onStarted handler if bound. + if (context.params.onStarted) { + await context.params.onStarted({ + connectionId, + success: false, + timeElapsed: getElapsedTime(timeStart) + }); + } + // Run the process in a single transaction for a database connection. await connection.transaction(async trx => { await teardown(trx, context); + // Trigger onTeardownSuccess if bound. + if (context.params.onTeardownSuccess) { + await context.params.onTeardownSuccess({ + connectionId, + success: true, + timeElapsed: getElapsedTime(timeStart) + }); + } + if (context.params['skip-migration']) { log('Skipped migrations.'); } else { @@ -122,16 +139,18 @@ export async function synchronizeDatabase(connection: Knex, context: SyncContext log(`Execution completed in ${timeElapsed} s`); - // If it's a CLI environment, invoke the handler. - if (isCLI()) { - const handler = result.success ? context.params.onSuccess : context.params.onFailed; - const execContext: ExecutionContext = { - connectionId, - timeElapsed, - success: result.success - }; - - await handler(execContext); + const execContext: ExecutionContext = { + connectionId, + timeElapsed, + success: result.success, + error: result.error + }; + + // Invoke corresponding handlers if they're sent. + if (result.success && context.params.onSuccess) { + await context.params.onSuccess(execContext); + } else if (!result.success && context.params.onFailed) { + await context.params.onFailed(execContext); } return result; From e4169833c9663ecc6782acc6dcdfc544aae0a75d Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Wed, 22 Apr 2020 02:10:57 +0545 Subject: [PATCH 63/80] Error handling and better UX --- src/commands/synchronize.ts | 67 ++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index 78390d7d..e49291b2 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -1,12 +1,14 @@ import { Command, flags } from '@oclif/command'; -import { log } from '../util/logger'; +import { log, dbLogger } from '../util/logger'; import { handleFlags } from '../cli'; import { getElapsedTime } from '../util/ts'; import SyncResult from '../domain/SyncResult'; -import { printError, printLine } from '../util/io'; +import { printError, printLine, printInfo } from '../util/io'; import ExecutionContext from '../domain/ExecutionContext'; import { loadConfig, resolveConnections } from '../config'; +import { MigrationResult } from '../service/knexMigrator'; +import { bold, cyan, red, green } from 'chalk'; /** * Synchronize command handler. @@ -25,17 +27,60 @@ class Synchronize extends Command { 'generate-connections': flags.boolean({ char: 'c', description: 'Generate connections' }) }; + onStarted = async (context: ExecutionContext) => { + await printLine(bold(` ▸ ${context.connectionId}`)); + await printInfo(' [✓] Synchronization - started'); + }; + + onTeardownSuccess = (context: ExecutionContext) => + printLine(green(' [✓] Synchronization - teardown') + ` (${context.timeElapsed}s)`); + /** * Success handler for each connection. */ onSuccess = (context: ExecutionContext) => - printLine(` [✓] ${context.connectionId} - Successful (${context.timeElapsed}s)`); + printLine(green(' [✓] Synchronization - completed') + ` (${context.timeElapsed}s)`); + + /** + * Success handler for migration run during sync process. + */ + onMigrationSuccess = async (result: MigrationResult) => { + const logDb = dbLogger(result.connectionId); + const [num, list] = result.data; + const alreadyUpToDate = num && list.length === 0; + + logDb('Up to date: ', alreadyUpToDate); + + if (alreadyUpToDate) { + await printLine(green(' [✓] Migrations - up to date') + ` (${result.timeElapsed}s)`); + + return; + } + + await printLine(green(` [✓] Migrations - ${list.length} run`) + ` (${result.timeElapsed}s)`); + + // Completed migrations. + for (const item of list) { + await printLine(cyan(` - ${item}`)); + } + }; + + /** + * Failure handler for migration during sync process. + */ + onMigrationFailed = async (result: MigrationResult) => { + await printLine(red(` [✖] Migrations - failed (${result.timeElapsed}s)\n`)); + + // await printError(` ${result.error}\n`); + }; /** * Failure handler for each connection. */ - onFailed = (context: ExecutionContext) => - printLine(` [✖] ${context.connectionId} - Failed (${context.timeElapsed}s)`); + onFailed = async (result: ExecutionContext) => { + await printLine(red(` [✖] Synchronization - failed (${result.timeElapsed}s)\n`)); + // await printError(` ${result.error}\n`); + }; /** * Check the results for each connection and display them. @@ -59,15 +104,15 @@ class Synchronize extends Command { if (!allComplete) { await printLine(`Synchronization failed for ${failedCount} connection(s):\n`); - failedAttempts.forEach(async (attempt, index) => { - await printLine(`${index + 1}) ${attempt.connectionId}`); + for (const attempt of failedAttempts) { + await printLine(bold(` ▸ ${attempt.connectionId}\n`)); await printError(attempt.error.toString()); // Send verbose error with stack trace to debug logs. log(attempt.error); await printLine(); - }); + } } return { totalCount, failedCount, successfulCount }; @@ -82,8 +127,12 @@ class Synchronize extends Command { const { flags: parsedFlags } = this.parse(Synchronize); const params = { ...parsedFlags, + onStarted: this.onStarted, + onTeardownSuccess: this.onTeardownSuccess, onSuccess: this.onSuccess, - onFailed: this.onFailed + onFailed: this.onFailed, + onMigrationSuccess: this.onMigrationSuccess, + onMigrationFailed: this.onMigrationFailed }; try { From 698129dd9b2f60520cbe209103987a2fe8894386 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Thu, 23 Apr 2020 01:23:29 +0545 Subject: [PATCH 64/80] Remove unnecessary blank line --- src/commands/synchronize.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index e49291b2..2ddcb73a 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -35,12 +35,6 @@ class Synchronize extends Command { onTeardownSuccess = (context: ExecutionContext) => printLine(green(' [✓] Synchronization - teardown') + ` (${context.timeElapsed}s)`); - /** - * Success handler for each connection. - */ - onSuccess = (context: ExecutionContext) => - printLine(green(' [✓] Synchronization - completed') + ` (${context.timeElapsed}s)`); - /** * Success handler for migration run during sync process. */ @@ -74,12 +68,17 @@ class Synchronize extends Command { // await printError(` ${result.error}\n`); }; + /** + * Success handler for each connection. + */ + onSuccess = (context: ExecutionContext) => + printLine(green(' [✓] Synchronization - completed') + ` (${context.timeElapsed}s)\n`); + /** * Failure handler for each connection. */ onFailed = async (result: ExecutionContext) => { await printLine(red(` [✖] Synchronization - failed (${result.timeElapsed}s)\n`)); - // await printError(` ${result.error}\n`); }; /** @@ -98,8 +97,6 @@ class Synchronize extends Command { const failedCount = totalCount - successfulCount; const allComplete = failedCount === 0; - await printLine(); - // If there are errors, display all of them. if (!allComplete) { await printLine(`Synchronization failed for ${failedCount} connection(s):\n`); From cea9f19f32bafb6e7ef8d1e99bd4fe4436c12486 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Thu, 23 Apr 2020 01:30:31 +0545 Subject: [PATCH 65/80] Remove --generate-connections flag --- src/cli.ts | 37 ------------------------------------- src/commands/synchronize.ts | 33 ++++++++++++++------------------- 2 files changed, 14 insertions(+), 56 deletions(-) delete mode 100644 src/cli.ts diff --git a/src/cli.ts b/src/cli.ts deleted file mode 100644 index 7085e22e..00000000 --- a/src/cli.ts +++ /dev/null @@ -1,37 +0,0 @@ -import * as path from 'path'; - -import * as fs from './util/fs'; -import { printLine } from './util/io'; -import SyncDbOptions from './domain/SyncDbOptions'; -import { CONNECTIONS_FILENAME } from './constants'; -import { resolveConnectionsFromEnv } from './config'; -import SynchronizeParams from './domain/SynchronizeParams'; - -/** - * Generates connections.sync-db.json file. - * - * @returns {Promise} - */ -async function generateConnection(): Promise { - const filePath = path.resolve(process.cwd(), CONNECTIONS_FILENAME); - - const connections = resolveConnectionsFromEnv(); - const contents = JSON.stringify({ connections }); - - await fs.write(filePath, contents); - - await printLine(`Generated file: ${CONNECTIONS_FILENAME}\n`); -} - -/** - * Handle the provided CLI flags. - * - * @param {SyncDbOptions} flags - * @returns {Promise} - */ -export async function handleFlags(flags: SyncDbOptions, params: SynchronizeParams): Promise { - if (flags['generate-connections']) { - await generateConnection(); - process.exit(0); - } -} diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index 2ddcb73a..0a67ec8e 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -1,14 +1,13 @@ +import { bold, cyan, red, green } from 'chalk'; import { Command, flags } from '@oclif/command'; -import { log, dbLogger } from '../util/logger'; -import { handleFlags } from '../cli'; import { getElapsedTime } from '../util/ts'; import SyncResult from '../domain/SyncResult'; -import { printError, printLine, printInfo } from '../util/io'; +import { log, dbLogger } from '../util/logger'; +import { MigrationResult } from '../service/knexMigrator'; import ExecutionContext from '../domain/ExecutionContext'; import { loadConfig, resolveConnections } from '../config'; -import { MigrationResult } from '../service/knexMigrator'; -import { bold, cyan, red, green } from 'chalk'; +import { printError, printLine, printInfo } from '../util/io'; /** * Synchronize command handler. @@ -23,8 +22,7 @@ class Synchronize extends Command { version: flags.version({ char: 'v', description: 'Print version', name: 'sync-db' }), help: flags.help({ char: 'h', description: 'Print help information' }), force: flags.boolean({ char: 'f', description: 'Force synchronization' }), - 'skip-migration': flags.boolean({ description: 'Skip running migrations.' }), - 'generate-connections': flags.boolean({ char: 'c', description: 'Generate connections' }) + 'skip-migration': flags.boolean({ description: 'Skip running migrations' }) }; onStarted = async (context: ExecutionContext) => { @@ -122,19 +120,8 @@ class Synchronize extends Command { */ async run(): Promise { const { flags: parsedFlags } = this.parse(Synchronize); - const params = { - ...parsedFlags, - onStarted: this.onStarted, - onTeardownSuccess: this.onTeardownSuccess, - onSuccess: this.onSuccess, - onFailed: this.onFailed, - onMigrationSuccess: this.onMigrationSuccess, - onMigrationFailed: this.onMigrationFailed - }; try { - await handleFlags(parsedFlags, params); - const config = await loadConfig(); const connections = await resolveConnections(); const { synchronize } = await import('../api'); @@ -142,7 +129,15 @@ class Synchronize extends Command { await printLine('Synchronizing...\n'); - const results = await synchronize(config, connections, params); + const results = await synchronize(config, connections, { + ...parsedFlags, + onStarted: this.onStarted, + onTeardownSuccess: this.onTeardownSuccess, + onSuccess: this.onSuccess, + onFailed: this.onFailed, + onMigrationSuccess: this.onMigrationSuccess, + onMigrationFailed: this.onMigrationFailed + }); const { totalCount, failedCount, successfulCount } = await this.processResults(results); From 574f4c5da090b50aa4d23507e50727de77317144 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Thu, 23 Apr 2020 02:03:07 +0545 Subject: [PATCH 66/80] Rename teardown to prune --- src/commands/synchronize.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index 0a67ec8e..1f8aa14c 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -30,8 +30,8 @@ class Synchronize extends Command { await printInfo(' [✓] Synchronization - started'); }; - onTeardownSuccess = (context: ExecutionContext) => - printLine(green(' [✓] Synchronization - teardown') + ` (${context.timeElapsed}s)`); + onPruneSuccess = (context: ExecutionContext) => + printLine(green(' [✓] Synchronization - pruned') + ` (${context.timeElapsed}s)`); /** * Success handler for migration run during sync process. @@ -132,7 +132,7 @@ class Synchronize extends Command { const results = await synchronize(config, connections, { ...parsedFlags, onStarted: this.onStarted, - onTeardownSuccess: this.onTeardownSuccess, + onTeardownSuccess: this.onPruneSuccess, onSuccess: this.onSuccess, onFailed: this.onFailed, onMigrationSuccess: this.onMigrationSuccess, From a578336c9c095fe2f29c41d2868a51f61df4b001 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Thu, 23 Apr 2020 02:13:50 +0545 Subject: [PATCH 67/80] Update imports --- src/commands/migrate-latest.ts | 2 +- src/commands/migrate-list.ts | 2 +- src/commands/migrate-rollback.ts | 2 +- src/commands/synchronize.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/commands/migrate-latest.ts b/src/commands/migrate-latest.ts index 5b42b988..943e447f 100644 --- a/src/commands/migrate-latest.ts +++ b/src/commands/migrate-latest.ts @@ -1,6 +1,7 @@ import { Command } from '@oclif/command'; import { bold, red, cyan } from 'chalk'; +import { migrateLatest } from '../api'; import { printLine, printError, printInfo } from '../util/io'; import { loadConfig, resolveConnections } from '..'; import { MigrationResult } from '../service/knexMigrator'; @@ -55,7 +56,6 @@ class MigrateLatest extends Command { async run(): Promise { const config = await loadConfig(); const connections = await resolveConnections(); - const { migrateLatest } = await import('../api'); const results = await migrateLatest(config, connections, { onSuccess: this.onSuccess, diff --git a/src/commands/migrate-list.ts b/src/commands/migrate-list.ts index 4ef37d89..0bf6b049 100644 --- a/src/commands/migrate-list.ts +++ b/src/commands/migrate-list.ts @@ -1,6 +1,7 @@ import { Command } from '@oclif/command'; import { bold, grey, red, cyan, yellow } from 'chalk'; +import { migrateList } from '../api'; import { printLine, printError } from '../util/io'; import { loadConfig, resolveConnections } from '..'; import { MigrationResult } from '../service/knexMigrator'; @@ -59,7 +60,6 @@ class MigrateList extends Command { async run(): Promise { const config = await loadConfig(); const connections = await resolveConnections(); - const { migrateList } = await import('../api'); const results = await migrateList(config, connections, { onSuccess: this.onSuccess, diff --git a/src/commands/migrate-rollback.ts b/src/commands/migrate-rollback.ts index 35e2c737..5fa159b1 100644 --- a/src/commands/migrate-rollback.ts +++ b/src/commands/migrate-rollback.ts @@ -1,6 +1,7 @@ import { Command } from '@oclif/command'; import { bold, red, cyan } from 'chalk'; +import { migrateRollback } from '../api'; import { printLine, printError, printInfo } from '../util/io'; import { loadConfig, resolveConnections } from '..'; import { MigrationResult } from '../service/knexMigrator'; @@ -55,7 +56,6 @@ class MigrateRollback extends Command { async run(): Promise { const config = await loadConfig(); const connections = await resolveConnections(); - const { migrateRollback } = await import('../api'); const results = await migrateRollback(config, connections, { onSuccess: this.onSuccess, diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index 1f8aa14c..99cc8137 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -8,6 +8,7 @@ import { MigrationResult } from '../service/knexMigrator'; import ExecutionContext from '../domain/ExecutionContext'; import { loadConfig, resolveConnections } from '../config'; import { printError, printLine, printInfo } from '../util/io'; +import { synchronize } from '../api'; /** * Synchronize command handler. @@ -124,7 +125,6 @@ class Synchronize extends Command { try { const config = await loadConfig(); const connections = await resolveConnections(); - const { synchronize } = await import('../api'); const timeStart = process.hrtime(); await printLine('Synchronizing...\n'); From b0ce82f204ff0c0e45f13dbb5d2d1564fec4f703 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 25 Apr 2020 17:20:39 +0545 Subject: [PATCH 68/80] Extract general domain interfaces --- src/domain/CommandParams.ts | 8 ++++++++ src/domain/CommandResult.ts | 9 +++++++++ src/domain/MigrationCommandContext.ts | 14 ++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 src/domain/CommandParams.ts create mode 100644 src/domain/CommandResult.ts create mode 100644 src/domain/MigrationCommandContext.ts diff --git a/src/domain/CommandParams.ts b/src/domain/CommandParams.ts new file mode 100644 index 00000000..d8a70a2d --- /dev/null +++ b/src/domain/CommandParams.ts @@ -0,0 +1,8 @@ +import CommandResult from './CommandResult'; + +interface CommandParams { + onSuccess?: (result: CommandResult) => Promise; + onFailed?: (result: CommandResult) => Promise; +} + +export default CommandParams; diff --git a/src/domain/CommandResult.ts b/src/domain/CommandResult.ts new file mode 100644 index 00000000..7fc1dda5 --- /dev/null +++ b/src/domain/CommandResult.ts @@ -0,0 +1,9 @@ +interface CommandResult { + connectionId: string; + success: boolean; + timeElapsed: number; + data: T; + error?: any; +} + +export default CommandResult; diff --git a/src/domain/MigrationCommandContext.ts b/src/domain/MigrationCommandContext.ts new file mode 100644 index 00000000..8f4fa95b --- /dev/null +++ b/src/domain/MigrationCommandContext.ts @@ -0,0 +1,14 @@ +import * as Knex from 'knex'; + +import Configuration from './Configuration'; +import CommandParams from './CommandParams'; +import CommandResult from './CommandResult'; + +interface MigrationCommandContext { + config: Configuration; + connectionId: string; + params: CommandParams; + knexMigrationConfig: Knex.MigratorConfig; +} + +export default MigrationCommandContext; From d95217ffce84c264f9e5e9e7dfba7e3a299293d3 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 25 Apr 2020 17:22:07 +0545 Subject: [PATCH 69/80] Simplify the contract --- src/api.ts | 76 ++++++++++++++++++-------------- src/commands/migrate-latest.ts | 6 +-- src/commands/migrate-list.ts | 6 +-- src/commands/migrate-rollback.ts | 6 +-- src/commands/synchronize.ts | 24 +++++----- src/constants.ts | 9 ---- src/domain/ExecutionContext.ts | 11 ----- src/domain/SyncResult.ts | 10 ----- src/domain/SynchronizeParams.ts | 16 +++---- src/service/knexMigrator.ts | 29 +++--------- src/service/sync.ts | 29 +++++------- 11 files changed, 84 insertions(+), 138 deletions(-) delete mode 100644 src/domain/ExecutionContext.ts delete mode 100644 src/domain/SyncResult.ts diff --git a/src/api.ts b/src/api.ts index 5bd18043..dd9acee0 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,15 +1,13 @@ import * as Knex from 'knex'; -import { mergeDeepRight } from 'ramda'; import { log } from './util/logger'; import { getConnectionId } from './config'; -import { DEFAULT_SYNC_PARAMS } from './constants'; import { isKnexInstance, getConfig, createInstance } from './util/db'; import * as init from './init'; -import SynchronizeParams from './domain/SynchronizeParams'; import Configuration from './domain/Configuration'; -import SyncResult from './domain/SyncResult'; +import SynchronizeParams from './domain/SynchronizeParams'; + import ConnectionConfig from './domain/ConnectionConfig'; import ConnectionReference from './domain/ConnectionReference'; @@ -17,24 +15,34 @@ import ConnectionReference from './domain/ConnectionReference'; import { synchronizeDatabase } from './service/sync'; import { executeProcesses } from './service/execution'; import { runMigrateFunc, migrationApiMap } from './service/knexMigrator'; -import { MigrationCommandParams, MigrationResult } from './service/knexMigrator'; +import CommandResult from './domain/CommandResult'; +import CommandParams from './domain/CommandParams'; + +/** + * Database connections given by the user or the CLI frontend. + */ +export type DatabaseConnections = ConnectionConfig[] | ConnectionConfig | Knex[] | Knex; /** * Synchronize all the configured database connections. * * @param {Configuration} config - * @param {(ConnectionConfig[] | ConnectionConfig | Knex[] | Knex)} conn + * @param {DatabaseConnections} conn * @param {SynchronizeParams} [options] - * @returns {Promise} + * @returns {Promise} */ export async function synchronize( config: Configuration, - conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, + conn: DatabaseConnections, options?: SynchronizeParams -): Promise { +): Promise { log('Synchronize'); - const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); + const params: SynchronizeParams = { + force: false, + 'skip-migration': false, + ...options + }; // TODO: Need to preload the SQL source code under this step. const { knexMigrationConfig } = await init.prepare(config, { @@ -66,22 +74,22 @@ export async function synchronize( } /** - * Migrate Latest + * Migrate Latest. * * @param {Configuration} config - * @param {(ConnectionConfig[] | ConnectionConfig | Knex[] | Knex)} conn - * @param {MigrationCommandParams} [options] - * @returns {Promise} + * @param {(DatabaseConnections)} conn + * @param {CommandParams} [options] + * @returns {Promise} */ export async function migrateLatest( config: Configuration, - conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, - options?: MigrationCommandParams -): Promise { + conn: DatabaseConnections, + options?: CommandParams +): Promise { log('Migrate Latest'); + const params: CommandParams = { ...options }; const connections = mapToConnectionReferences(conn); - const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); const { knexMigrationConfig } = await init.prepare(config, { loadMigrations: true }); const processes = connections.map(({ connection, id: connectionId }) => () => @@ -104,19 +112,19 @@ export async function migrateLatest( * Migrate Rollback. * * @param {Configuration} config - * @param {(ConnectionConfig[] | ConnectionConfig | Knex[] | Knex)} conn - * @param {MigrationCommandParams} [options] - * @returns {Promise} + * @param {(DatabaseConnections)} conn + * @param {CommandParams} [options] + * @returns {Promise} */ export async function migrateRollback( config: Configuration, - conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, - options?: MigrationCommandParams -): Promise { + conn: DatabaseConnections, + options?: CommandParams +): Promise { log('Migrate Rollback'); + const params: CommandParams = { ...options }; const connections = mapToConnectionReferences(conn); - const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); const { knexMigrationConfig } = await init.prepare(config, { loadMigrations: true }); const processes = connections.map(({ connection, id: connectionId }) => () => @@ -139,19 +147,19 @@ export async function migrateRollback( * List Migrations. * * @param {Configuration} config - * @param {(ConnectionConfig[] | ConnectionConfig | Knex[] | Knex)} conn - * @param {MigrationCommandParams} [options] - * @returns {Promise} + * @param {(DatabaseConnections)} conn + * @param {CommandParams} [options] + * @returns {Promise} */ export async function migrateList( config: Configuration, - conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex, - options?: MigrationCommandParams -): Promise { + conn: DatabaseConnections, + options?: CommandParams +): Promise { log('Migrate List'); + const params: CommandParams = { ...options }; const connections = mapToConnectionReferences(conn); - const params = mergeDeepRight(DEFAULT_SYNC_PARAMS, options || {}); const { knexMigrationConfig } = await init.prepare(config, { loadMigrations: true }); const processes = connections.map(({ connection, id: connectionId }) => () => @@ -173,10 +181,10 @@ export async function migrateList( /** * Map user provided connection(s) to the connection instances. * - * @param {(ConnectionConfig[] | ConnectionConfig | Knex[] | Knex)} conn + * @param {(DatabaseConnections)} conn * @returns {ConnectionReference[]} */ -function mapToConnectionReferences(conn: ConnectionConfig[] | ConnectionConfig | Knex[] | Knex): ConnectionReference[] { +function mapToConnectionReferences(conn: DatabaseConnections): ConnectionReference[] { const connectionList = Array.isArray(conn) ? conn : [conn]; return connectionList.map(connection => { diff --git a/src/commands/migrate-latest.ts b/src/commands/migrate-latest.ts index 943e447f..602de495 100644 --- a/src/commands/migrate-latest.ts +++ b/src/commands/migrate-latest.ts @@ -4,8 +4,8 @@ import { bold, red, cyan } from 'chalk'; import { migrateLatest } from '../api'; import { printLine, printError, printInfo } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { MigrationResult } from '../service/knexMigrator'; import { dbLogger } from '../util/logger'; +import CommandResult from '../domain/CommandResult'; /** * Migration command handler. @@ -16,7 +16,7 @@ class MigrateLatest extends Command { /** * Success handler for each connection. */ - onSuccess = async (result: MigrationResult) => { + onSuccess = async (result: CommandResult) => { const log = dbLogger(result.connectionId); const [num, list] = result.data; const alreadyUpToDate = num && list.length === 0; @@ -42,7 +42,7 @@ class MigrateLatest extends Command { /** * Failure handler for each connection. */ - onFailed = async (result: MigrationResult) => { + onFailed = async (result: CommandResult) => { printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); await printError(` ${result.error}\n`); diff --git a/src/commands/migrate-list.ts b/src/commands/migrate-list.ts index 0bf6b049..32e28759 100644 --- a/src/commands/migrate-list.ts +++ b/src/commands/migrate-list.ts @@ -4,7 +4,7 @@ import { bold, grey, red, cyan, yellow } from 'chalk'; import { migrateList } from '../api'; import { printLine, printError } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { MigrationResult } from '../service/knexMigrator'; +import CommandResult from '../domain/CommandResult'; /** * Migration command handler. @@ -15,7 +15,7 @@ class MigrateList extends Command { /** * Success handler for a connection. */ - onSuccess = async (result: MigrationResult) => { + onSuccess = async (result: CommandResult) => { await printLine(bold(` ▸ ${result.connectionId}`)); const [list1, list2] = result.data; @@ -46,7 +46,7 @@ class MigrateList extends Command { /** * Failure handler for a connection. */ - onFailed = async (result: MigrationResult) => { + onFailed = async (result: CommandResult) => { printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); await printError(` ${result.error}\n`); diff --git a/src/commands/migrate-rollback.ts b/src/commands/migrate-rollback.ts index 5fa159b1..219f8200 100644 --- a/src/commands/migrate-rollback.ts +++ b/src/commands/migrate-rollback.ts @@ -4,8 +4,8 @@ import { bold, red, cyan } from 'chalk'; import { migrateRollback } from '../api'; import { printLine, printError, printInfo } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import { MigrationResult } from '../service/knexMigrator'; import { dbLogger } from '../util/logger'; +import CommandResult from '../domain/CommandResult'; /** * Migration command handler. @@ -16,7 +16,7 @@ class MigrateRollback extends Command { /** * Success handler for each connection. */ - onSuccess = async (result: MigrationResult) => { + onSuccess = async (result: CommandResult) => { const log = dbLogger(result.connectionId); const [num, list] = result.data; const allRolledBack = num === 0; @@ -42,7 +42,7 @@ class MigrateRollback extends Command { /** * Failure handler for each connection. */ - onFailed = async (result: MigrationResult) => { + onFailed = async (result: CommandResult) => { printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); await printError(` ${result.error}\n`); diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index 99cc8137..be1b0686 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -2,13 +2,11 @@ import { bold, cyan, red, green } from 'chalk'; import { Command, flags } from '@oclif/command'; import { getElapsedTime } from '../util/ts'; -import SyncResult from '../domain/SyncResult'; import { log, dbLogger } from '../util/logger'; -import { MigrationResult } from '../service/knexMigrator'; -import ExecutionContext from '../domain/ExecutionContext'; import { loadConfig, resolveConnections } from '../config'; import { printError, printLine, printInfo } from '../util/io'; import { synchronize } from '../api'; +import CommandResult from '../domain/CommandResult'; /** * Synchronize command handler. @@ -26,18 +24,18 @@ class Synchronize extends Command { 'skip-migration': flags.boolean({ description: 'Skip running migrations' }) }; - onStarted = async (context: ExecutionContext) => { - await printLine(bold(` ▸ ${context.connectionId}`)); + onStarted = async (result: CommandResult) => { + await printLine(bold(` ▸ ${result.connectionId}`)); await printInfo(' [✓] Synchronization - started'); }; - onPruneSuccess = (context: ExecutionContext) => - printLine(green(' [✓] Synchronization - pruned') + ` (${context.timeElapsed}s)`); + onPruneSuccess = (result: CommandResult) => + printLine(green(' [✓] Synchronization - pruned') + ` (${result.timeElapsed}s)`); /** * Success handler for migration run during sync process. */ - onMigrationSuccess = async (result: MigrationResult) => { + onMigrationSuccess = async (result: CommandResult) => { const logDb = dbLogger(result.connectionId); const [num, list] = result.data; const alreadyUpToDate = num && list.length === 0; @@ -61,7 +59,7 @@ class Synchronize extends Command { /** * Failure handler for migration during sync process. */ - onMigrationFailed = async (result: MigrationResult) => { + onMigrationFailed = async (result: CommandResult) => { await printLine(red(` [✖] Migrations - failed (${result.timeElapsed}s)\n`)); // await printError(` ${result.error}\n`); @@ -70,13 +68,13 @@ class Synchronize extends Command { /** * Success handler for each connection. */ - onSuccess = (context: ExecutionContext) => - printLine(green(' [✓] Synchronization - completed') + ` (${context.timeElapsed}s)\n`); + onSuccess = (result: CommandResult) => + printLine(green(' [✓] Synchronization - completed') + ` (${result.timeElapsed}s)\n`); /** * Failure handler for each connection. */ - onFailed = async (result: ExecutionContext) => { + onFailed = async (result: CommandResult) => { await printLine(red(` [✖] Synchronization - failed (${result.timeElapsed}s)\n`)); }; @@ -88,7 +86,7 @@ class Synchronize extends Command { * @returns {Promise<{ totalCount: number, failedCount: number, successfulCount: number }>} */ async processResults( - results: SyncResult[] + results: CommandResult[] ): Promise<{ totalCount: number; failedCount: number; successfulCount: number }> { const totalCount = results.length; const failedAttempts = results.filter(result => !result.success); diff --git a/src/constants.ts b/src/constants.ts index ef6ec4df..cc22c040 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -3,8 +3,6 @@ import * as path from 'path'; import Configuration from './domain/Configuration'; -import SynchronizeParams from './domain/SynchronizeParams'; -import ExecutionContext from './domain/ExecutionContext'; // General constants export const NS_PER_SEC = 1e9; @@ -32,11 +30,4 @@ export const DEFAULT_CONFIG: Configuration = { } }; -export const DEFAULT_SYNC_PARAMS: SynchronizeParams = { - force: false, - 'skip-migration': false, - onSuccess: (context: ExecutionContext) => Promise.resolve(), - onFailed: (context: ExecutionContext) => Promise.resolve() -}; - export const REQUIRED_ENV_KEYS = ['DB_HOST', 'DB_PASSWORD', 'DB_NAME', 'DB_USERNAME', 'DB_PORT', 'DB_CLIENT']; diff --git a/src/domain/ExecutionContext.ts b/src/domain/ExecutionContext.ts deleted file mode 100644 index 9d9c1340..00000000 --- a/src/domain/ExecutionContext.ts +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Execution context for each connection. - */ -interface ExecutionContext { - connectionId: string; - success: boolean; - timeElapsed: number; - error?: any; -} - -export default ExecutionContext; diff --git a/src/domain/SyncResult.ts b/src/domain/SyncResult.ts deleted file mode 100644 index 56db330b..00000000 --- a/src/domain/SyncResult.ts +++ /dev/null @@ -1,10 +0,0 @@ -/** - * Synchronize result for a database connection. - */ -interface SyncResult { - success: boolean; - connectionId: string; - error?: any; -} - -export default SyncResult; diff --git a/src/domain/SynchronizeParams.ts b/src/domain/SynchronizeParams.ts index fdc09f80..0e91d527 100644 --- a/src/domain/SynchronizeParams.ts +++ b/src/domain/SynchronizeParams.ts @@ -1,18 +1,16 @@ -import ExecutionContext from './ExecutionContext'; -import { MigrationResult } from '../service/knexMigrator'; +import CommandParams from './CommandParams'; +import CommandResult from './CommandResult'; /** * Synchronize parameters. */ -interface SynchronizeParams { +interface SynchronizeParams extends CommandParams { force: boolean; 'skip-migration': boolean; - onStarted?: (context: ExecutionContext) => Promise; - onSuccess?: (context: ExecutionContext) => Promise; - onTeardownSuccess?: (context: ExecutionContext) => Promise; - onFailed?: (context: ExecutionContext) => Promise; - onMigrationSuccess?: (result: MigrationResult) => Promise; - onMigrationFailed?: (result: MigrationResult) => Promise; + onStarted?: (result: CommandResult) => Promise; + onTeardownSuccess?: (result: CommandResult) => Promise; + onMigrationSuccess?: (result: CommandResult) => Promise; + onMigrationFailed?: (result: CommandResult) => Promise; } export default SynchronizeParams; diff --git a/src/service/knexMigrator.ts b/src/service/knexMigrator.ts index 43696d85..9f3eec2c 100644 --- a/src/service/knexMigrator.ts +++ b/src/service/knexMigrator.ts @@ -2,27 +2,8 @@ import Knex from 'knex'; import { dbLogger } from '../util/logger'; import { getElapsedTime } from '../util/ts'; -import Configuration from '../domain/Configuration'; - -export interface MigrationResult { - connectionId: string; - success: boolean; - timeElapsed: number; - data: any; - error?: any; -} - -export interface MigrationCommandParams { - onSuccess?: (result: MigrationResult) => Promise; - onFailed?: (context: MigrationResult) => Promise; -} - -export interface MigrationCommandContext { - config: Configuration; - connectionId: string; - params: MigrationCommandParams; - knexMigrationConfig: Knex.MigratorConfig; -} +import MigrationCommandContext from '../domain/MigrationCommandContext'; +import CommandResult from '../domain/CommandResult'; /** * A map of Knex's migration API functions. @@ -39,13 +20,13 @@ export const migrationApiMap = { * @param {(Knex | Knex.Transaction)} trx * @param {MigrationCommandContext} context * @param {((trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => Promise)} func - * @returns {Promise} + * @returns {Promise} */ export async function runMigrateFunc( trx: Knex | Knex.Transaction, context: MigrationCommandContext, func: (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => Promise -): Promise { +): Promise { const dbLog = dbLogger(context.connectionId); const { connectionId, knexMigrationConfig } = context; const funcName = func.name || 'func'; @@ -70,7 +51,7 @@ export async function runMigrateFunc( dbLog(`Execution completed in ${timeElapsed} s`); - const result: MigrationResult = { + const result: CommandResult = { connectionId, error, data, diff --git a/src/service/sync.ts b/src/service/sync.ts index 1ed4f6f1..9a884a4d 100644 --- a/src/service/sync.ts +++ b/src/service/sync.ts @@ -3,10 +3,9 @@ import * as Knex from 'knex'; import * as sqlRunner from './sqlRunner'; import { dbLogger } from '../util/logger'; import { getElapsedTime } from '../util/ts'; -import SyncResult from '../domain/SyncResult'; import SyncContext from '../domain/SyncContext'; import * as configInjection from './configInjection'; -import ExecutionContext from '../domain/ExecutionContext'; +import CommandResult from '../domain/CommandResult'; /** * Migrate SQL on a database. @@ -84,12 +83,12 @@ async function teardown(trx: Knex.Transaction, context: SyncContext): Promise} + * @returns {Promise>} */ -export async function synchronizeDatabase(connection: Knex, context: SyncContext): Promise { +export async function synchronizeDatabase(connection: Knex, context: SyncContext): Promise> { const { connectionId, migrateFunc } = context; const log = dbLogger(connectionId); - const result: SyncResult = { connectionId, success: false }; + const result: CommandResult = { connectionId, success: false, data: null, timeElapsed: 0 }; const timeStart = process.hrtime(); @@ -99,8 +98,7 @@ export async function synchronizeDatabase(connection: Knex, context: SyncContext // Trigger onStarted handler if bound. if (context.params.onStarted) { await context.params.onStarted({ - connectionId, - success: false, + ...result, timeElapsed: getElapsedTime(timeStart) }); } @@ -112,7 +110,7 @@ export async function synchronizeDatabase(connection: Knex, context: SyncContext // Trigger onTeardownSuccess if bound. if (context.params.onTeardownSuccess) { await context.params.onTeardownSuccess({ - connectionId, + ...result, success: true, timeElapsed: getElapsedTime(timeStart) }); @@ -135,22 +133,15 @@ export async function synchronizeDatabase(connection: Knex, context: SyncContext result.error = e; } - const timeElapsed = getElapsedTime(timeStart); + result.timeElapsed = getElapsedTime(timeStart); - log(`Execution completed in ${timeElapsed} s`); - - const execContext: ExecutionContext = { - connectionId, - timeElapsed, - success: result.success, - error: result.error - }; + log(`Execution completed in ${result.timeElapsed} s`); // Invoke corresponding handlers if they're sent. if (result.success && context.params.onSuccess) { - await context.params.onSuccess(execContext); + await context.params.onSuccess(result); } else if (!result.success && context.params.onFailed) { - await context.params.onFailed(execContext); + await context.params.onFailed(result); } return result; From 905ac4cd4cab06ad04639c8f0189b94ff67f0646 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 25 Apr 2020 20:41:05 +0545 Subject: [PATCH 70/80] Add executeOperation function --- src/service/execution.ts | 49 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src/service/execution.ts b/src/service/execution.ts index 61560c06..84e21c55 100644 --- a/src/service/execution.ts +++ b/src/service/execution.ts @@ -1,6 +1,9 @@ -import { log } from '../util/logger'; +import { log, dbLogger } from '../util/logger'; import Configuration from '../domain/Configuration'; import { Promiser, runSequentially } from '../util/promise'; +import CommandContext from '../domain/CommandContext'; +import CommandResult from '../domain/CommandResult'; +import { getElapsedTime } from '../util/ts'; /** * Execute a list of processes according to the configuration. @@ -23,3 +26,47 @@ export function executeProcesses(processes: Promiser[], config: Configurat throw new Error(`Execution strategy should be "sequential" or "parallel" found: "${config.execution}".`); } } + +/** + * Execute a unit operation. + * + * @param {T} context + * @param {(options: any) => Promise} func + * @returns {Promise} + */ +export async function executeOperation( + context: T, + func: (options: any) => Promise +): Promise { + const { connectionId } = context; + const logDb = dbLogger(connectionId); + const result: CommandResult = { connectionId, success: false, data: null, timeElapsed: 0 }; + + const timeStart = process.hrtime(); + + try { + logDb('BEGIN: operation'); + + result.data = await func({ timeStart }); + + logDb(`END: operation`); + result.success = true; + } catch (e) { + logDb('FAILED: operation'); + logDb(`Error caught for connection ${connectionId}:`, e); + result.error = e; + } + + result.timeElapsed = getElapsedTime(timeStart); + + logDb(`Execution completed in ${result.timeElapsed} s`); + + // Invoke corresponding handlers if they're sent. + if (result.success && context.params.onSuccess) { + await context.params.onSuccess(result); + } else if (!result.success && context.params.onFailed) { + await context.params.onFailed(result); + } + + return result; +} From cec5886735343aa6844655768239a9196eac2e1c Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 25 Apr 2020 20:41:45 +0545 Subject: [PATCH 71/80] Extract a base interface --- src/domain/CommandContext.ts | 10 ++++++++++ src/domain/MigrationCommandContext.ts | 9 +++------ src/domain/SyncContext.ts | 9 ++++----- 3 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 src/domain/CommandContext.ts diff --git a/src/domain/CommandContext.ts b/src/domain/CommandContext.ts new file mode 100644 index 00000000..a8532689 --- /dev/null +++ b/src/domain/CommandContext.ts @@ -0,0 +1,10 @@ +import Configuration from './Configuration'; +import CommandParams from './CommandParams'; + +interface CommandContext { + config: Configuration; + connectionId: string; + params: CommandParams; +} + +export default CommandContext; diff --git a/src/domain/MigrationCommandContext.ts b/src/domain/MigrationCommandContext.ts index 8f4fa95b..0d5c030c 100644 --- a/src/domain/MigrationCommandContext.ts +++ b/src/domain/MigrationCommandContext.ts @@ -1,13 +1,10 @@ import * as Knex from 'knex'; -import Configuration from './Configuration'; import CommandParams from './CommandParams'; -import CommandResult from './CommandResult'; +import CommandContext from './CommandContext'; -interface MigrationCommandContext { - config: Configuration; - connectionId: string; - params: CommandParams; +interface MigrationCommandContext extends CommandContext { + params: CommandParams; knexMigrationConfig: Knex.MigratorConfig; } diff --git a/src/domain/SyncContext.ts b/src/domain/SyncContext.ts index a2aa10be..c177191e 100644 --- a/src/domain/SyncContext.ts +++ b/src/domain/SyncContext.ts @@ -1,13 +1,12 @@ -import Configuration from './Configuration'; +import * as Knex from 'knex'; + import SynchronizeParams from './SynchronizeParams'; -import Knex from 'knex'; +import CommandContext from './CommandContext'; /** * Synchronize context for a database connection. */ -interface SyncContext { - config: Configuration; - connectionId: string; +interface SyncContext extends CommandContext { params: SynchronizeParams; migrateFunc: (trx: Knex.Transaction) => Promise; } From 83c81d41b4d0cc81419636087d14ddc281ad1e81 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 25 Apr 2020 20:42:40 +0545 Subject: [PATCH 72/80] Use executeOperation function --- src/service/knexMigrator.ts | 43 +++++++------------------------------ 1 file changed, 8 insertions(+), 35 deletions(-) diff --git a/src/service/knexMigrator.ts b/src/service/knexMigrator.ts index 9f3eec2c..15a041df 100644 --- a/src/service/knexMigrator.ts +++ b/src/service/knexMigrator.ts @@ -1,9 +1,9 @@ import Knex from 'knex'; import { dbLogger } from '../util/logger'; -import { getElapsedTime } from '../util/ts'; import MigrationCommandContext from '../domain/MigrationCommandContext'; import CommandResult from '../domain/CommandResult'; +import { executeOperation } from './execution'; /** * A map of Knex's migration API functions. @@ -27,44 +27,17 @@ export async function runMigrateFunc( context: MigrationCommandContext, func: (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => Promise ): Promise { - const dbLog = dbLogger(context.connectionId); - const { connectionId, knexMigrationConfig } = context; - const funcName = func.name || 'func'; + return executeOperation(context, async () => { + const { knexMigrationConfig } = context; + const funcName = func.name || 'func'; - let error; - let data; - - const timeStart = process.hrtime(); - - try { + const dbLog = dbLogger(context.connectionId); dbLog(`BEGIN: ${funcName}`); - data = await func(trx, knexMigrationConfig); + const data = await func(trx, knexMigrationConfig); dbLog(`END: ${funcName}`); dbLog('Result:\n%O', data); - } catch (e) { - dbLog(`Error caught for connection ${connectionId}:`, e); - error = e; - } - - const timeElapsed = getElapsedTime(timeStart); - - dbLog(`Execution completed in ${timeElapsed} s`); - - const result: CommandResult = { - connectionId, - error, - data, - timeElapsed, - success: !error - }; - - // Invoke corresponding handlers if they're sent. - if (result.success && context.params.onSuccess) { - await context.params.onSuccess(result); - } else if (!result.success && context.params.onFailed) { - await context.params.onFailed(result); - } - return result; + return data; + }); } From 77281bac0f48385d3e18fd67ddca13693ce9f74a Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 25 Apr 2020 21:00:47 +0545 Subject: [PATCH 73/80] Add prune command --- src/api.ts | 34 ++++++++++++++- src/commands/migrate-rollback.ts | 2 +- src/commands/prune.ts | 53 +++++++++++++++++++++++ src/service/sync.ts | 74 ++++++++++++++------------------ 4 files changed, 120 insertions(+), 43 deletions(-) create mode 100644 src/commands/prune.ts diff --git a/src/api.ts b/src/api.ts index dd9acee0..74ea8248 100644 --- a/src/api.ts +++ b/src/api.ts @@ -12,7 +12,7 @@ import ConnectionConfig from './domain/ConnectionConfig'; import ConnectionReference from './domain/ConnectionReference'; // Service -import { synchronizeDatabase } from './service/sync'; +import { synchronizeDatabase, pruneDatabase } from './service/sync'; import { executeProcesses } from './service/execution'; import { runMigrateFunc, migrationApiMap } from './service/knexMigrator'; import CommandResult from './domain/CommandResult'; @@ -73,6 +73,38 @@ export async function synchronize( return executeProcesses(processes, config); } +/** + * Prune all synchronized objects from the databases (except the ones like tables made via migrations). + * + * TODO: An ability to prune only a handful of objects from the last. + * + * @param {Configuration} config + * @param {(DatabaseConnections)} conn + * @param {CommandParams} [options] + * @returns {Promise} + */ +export async function prune( + config: Configuration, + conn: DatabaseConnections, + options?: CommandParams +): Promise { + log('Prune'); + + const params: CommandParams = { ...options }; + const connections = mapToConnectionReferences(conn); + await init.prepare(config, {}); + + const processes = connections.map(({ connection, id: connectionId }) => () => + pruneDatabase(connection, { + config, + params, + connectionId + }) + ); + + return executeProcesses(processes, config); +} + /** * Migrate Latest. * diff --git a/src/commands/migrate-rollback.ts b/src/commands/migrate-rollback.ts index 219f8200..db37bd6f 100644 --- a/src/commands/migrate-rollback.ts +++ b/src/commands/migrate-rollback.ts @@ -43,7 +43,7 @@ class MigrateRollback extends Command { * Failure handler for each connection. */ onFailed = async (result: CommandResult) => { - printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); + await printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); await printError(` ${result.error}\n`); }; diff --git a/src/commands/prune.ts b/src/commands/prune.ts new file mode 100644 index 00000000..77e22237 --- /dev/null +++ b/src/commands/prune.ts @@ -0,0 +1,53 @@ +import { Command } from '@oclif/command'; +import { bold, red } from 'chalk'; + +import { printLine, printError } from '../util/io'; + +import CommandResult from '../domain/CommandResult'; +import { prune } from '../api'; +import { loadConfig, resolveConnections } from '..'; + +class Prune extends Command { + static description = 'Drop all the synchronized db objects except the ones created via migrations.'; + /** + * Success handler for each connection. + */ + onSuccess = async (result: CommandResult) => { + await printLine(bold(` ▸ ${result.connectionId} - Successful`) + ` (${result.timeElapsed}s)`); + }; + + /** + * Failure handler for each connection. + */ + onFailed = async (result: CommandResult) => { + await printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); + + await printError(` ${result.error}\n`); + }; + + /** + * CLI command execution handler. + * + * @returns {Promise} + */ + async run(): Promise { + const config = await loadConfig(); + const connections = await resolveConnections(); + + const results = await prune(config, connections, { + onSuccess: this.onSuccess, + onFailed: this.onFailed + }); + + const failedCount = results.filter(({ success }) => !success).length; + + if (failedCount === 0) { + return process.exit(0); + } + + printError(`Error: Prune failed for ${failedCount} connection(s).`); + process.exit(-1); + } +} + +export default Prune; diff --git a/src/service/sync.ts b/src/service/sync.ts index 9a884a4d..6452a86a 100644 --- a/src/service/sync.ts +++ b/src/service/sync.ts @@ -6,6 +6,8 @@ import { getElapsedTime } from '../util/ts'; import SyncContext from '../domain/SyncContext'; import * as configInjection from './configInjection'; import CommandResult from '../domain/CommandResult'; +import CommandContext from '../domain/CommandContext'; +import { executeOperation } from './execution'; /** * Migrate SQL on a database. @@ -62,10 +64,10 @@ async function setup(trx: Knex.Transaction, context: SyncContext): Promise * They're executed in the reverse order of their creation. * * @param {Knex.Transaction} trx - * @param {SyncContext} context + * @param {CommandContext} context * @returns {Promise} */ -async function teardown(trx: Knex.Transaction, context: SyncContext): Promise { +async function teardown(trx: Knex.Transaction, context: CommandContext): Promise { const { basePath, sql } = context.config; const log = dbLogger(context.connectionId); @@ -86,31 +88,29 @@ async function teardown(trx: Knex.Transaction, context: SyncContext): Promise>} */ export async function synchronizeDatabase(connection: Knex, context: SyncContext): Promise> { - const { connectionId, migrateFunc } = context; - const log = dbLogger(connectionId); - const result: CommandResult = { connectionId, success: false, data: null, timeElapsed: 0 }; - - const timeStart = process.hrtime(); - - try { - log('Starting synchronization.'); - - // Trigger onStarted handler if bound. - if (context.params.onStarted) { - await context.params.onStarted({ - ...result, - timeElapsed: getElapsedTime(timeStart) - }); - } + return connection.transaction(trx => + executeOperation(context, async options => { + const { connectionId, migrateFunc } = context; + const { timeStart } = options; + const log = dbLogger(connectionId); + + // Trigger onStarted handler if bound. + if (context.params.onStarted) { + await context.params.onStarted({ + connectionId, + success: false, + data: null, + timeElapsed: getElapsedTime(timeStart) + }); + } - // Run the process in a single transaction for a database connection. - await connection.transaction(async trx => { await teardown(trx, context); // Trigger onTeardownSuccess if bound. if (context.params.onTeardownSuccess) { await context.params.onTeardownSuccess({ - ...result, + connectionId, + data: null, success: true, timeElapsed: getElapsedTime(timeStart) }); @@ -124,25 +124,17 @@ export async function synchronizeDatabase(connection: Knex, context: SyncContext } await setup(trx, context); - }); - - log(`Synchronization successful.`); - result.success = true; - } catch (e) { - log(`Error caught for connection ${connectionId}:`, e); - result.error = e; - } - - result.timeElapsed = getElapsedTime(timeStart); - - log(`Execution completed in ${result.timeElapsed} s`); - - // Invoke corresponding handlers if they're sent. - if (result.success && context.params.onSuccess) { - await context.params.onSuccess(result); - } else if (!result.success && context.params.onFailed) { - await context.params.onFailed(result); - } + }) + ); +} - return result; +/** + * Prune on a single database connection. + * + * @param {Knex} connection + * @param {CommandContext} context + * @returns {Promise>} + */ +export async function pruneDatabase(connection: Knex, context: CommandContext): Promise> { + return connection.transaction(trx => executeOperation(context, () => teardown(trx, context))); } From 76f3f0202905dd44d5462f485150ab9eae4b40fc Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 25 Apr 2020 22:04:57 +0545 Subject: [PATCH 74/80] Refactor to simplify the api --- src/api.ts | 131 +++++++++++++++++++----------------- src/service/execution.ts | 4 -- src/service/knexMigrator.ts | 8 +-- src/service/sync.ts | 88 ++++++++++++------------ src/util/db.ts | 26 +++++++ 5 files changed, 143 insertions(+), 114 deletions(-) diff --git a/src/api.ts b/src/api.ts index 74ea8248..8b60684f 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,22 +1,21 @@ import * as Knex from 'knex'; +import * as init from './init'; import { log } from './util/logger'; import { getConnectionId } from './config'; -import { isKnexInstance, getConfig, createInstance } from './util/db'; +import { isKnexInstance, getConfig, createInstance, withTransaction } from './util/db'; -import * as init from './init'; import Configuration from './domain/Configuration'; -import SynchronizeParams from './domain/SynchronizeParams'; - +import CommandResult from './domain/CommandResult'; +import CommandParams from './domain/CommandParams'; import ConnectionConfig from './domain/ConnectionConfig'; +import SynchronizeParams from './domain/SynchronizeParams'; import ConnectionReference from './domain/ConnectionReference'; // Service -import { synchronizeDatabase, pruneDatabase } from './service/sync'; import { executeProcesses } from './service/execution'; -import { runMigrateFunc, migrationApiMap } from './service/knexMigrator'; -import CommandResult from './domain/CommandResult'; -import CommandParams from './domain/CommandParams'; +import { synchronizeDatabase, pruneDatabase } from './service/sync'; +import { invokeMigrationApi, migrationApiMap } from './service/knexMigrator'; /** * Database connections given by the user or the CLI frontend. @@ -51,23 +50,25 @@ export async function synchronize( }); const connections = mapToConnectionReferences(conn); - const processes = connections.map(({ connection, id: connectionId }) => () => - synchronizeDatabase(connection, { - config, - params, - connectionId, - migrateFunc: (trx: Knex.Transaction) => - runMigrateFunc( - trx, - { - config, - connectionId, - params: { ...params, onSuccess: params.onMigrationSuccess, onFailed: params.onMigrationFailed }, - knexMigrationConfig: knexMigrationConfig(connectionId) - }, - migrationApiMap['migrate.latest'] - ) - }) + const processes = connections.map(connection => () => + withTransaction(connection, trx => + synchronizeDatabase(trx, { + config, + params, + connectionId: connection.id, + migrateFunc: t => + invokeMigrationApi( + t, + { + config, + connectionId: connection.id, + knexMigrationConfig: knexMigrationConfig(connection.id), + params: { ...params, onSuccess: params.onMigrationSuccess, onFailed: params.onMigrationFailed } + }, + migrationApiMap['migrate.latest'] + ) + }) + ) ); return executeProcesses(processes, config); @@ -94,12 +95,14 @@ export async function prune( const connections = mapToConnectionReferences(conn); await init.prepare(config, {}); - const processes = connections.map(({ connection, id: connectionId }) => () => - pruneDatabase(connection, { - config, - params, - connectionId - }) + const processes = connections.map(connection => () => + withTransaction(connection, trx => + pruneDatabase(trx, { + config, + params, + connectionId: connection.id + }) + ) ); return executeProcesses(processes, config); @@ -124,16 +127,18 @@ export async function migrateLatest( const connections = mapToConnectionReferences(conn); const { knexMigrationConfig } = await init.prepare(config, { loadMigrations: true }); - const processes = connections.map(({ connection, id: connectionId }) => () => - runMigrateFunc( - connection, - { - config, - params, - connectionId, - knexMigrationConfig: knexMigrationConfig(connectionId) - }, - migrationApiMap['migrate.latest'] + const processes = connections.map(connection => () => + withTransaction(connection, trx => + invokeMigrationApi( + trx, + { + config, + params, + connectionId: connection.id, + knexMigrationConfig: knexMigrationConfig(connection.id) + }, + migrationApiMap['migrate.latest'] + ) ) ); @@ -159,16 +164,18 @@ export async function migrateRollback( const connections = mapToConnectionReferences(conn); const { knexMigrationConfig } = await init.prepare(config, { loadMigrations: true }); - const processes = connections.map(({ connection, id: connectionId }) => () => - runMigrateFunc( - connection, - { - config, - params, - connectionId, - knexMigrationConfig: knexMigrationConfig(connectionId) - }, - migrationApiMap['migrate.rollback'] + const processes = connections.map(connection => () => + withTransaction(connection, trx => + invokeMigrationApi( + trx, + { + config, + params, + connectionId: connection.id, + knexMigrationConfig: knexMigrationConfig(connection.id) + }, + migrationApiMap['migrate.rollback'] + ) ) ); @@ -194,16 +201,18 @@ export async function migrateList( const connections = mapToConnectionReferences(conn); const { knexMigrationConfig } = await init.prepare(config, { loadMigrations: true }); - const processes = connections.map(({ connection, id: connectionId }) => () => - runMigrateFunc( - connection, - { - config, - params, - connectionId, - knexMigrationConfig: knexMigrationConfig(connectionId) - }, - migrationApiMap['migrate.list'] + const processes = connections.map(connection => () => + withTransaction(connection, trx => + invokeMigrationApi( + trx, + { + config, + params, + connectionId: connection.id, + knexMigrationConfig: knexMigrationConfig(connection.id) + }, + migrationApiMap['migrate.list'] + ) ) ); diff --git a/src/service/execution.ts b/src/service/execution.ts index 84e21c55..d98e9b86 100644 --- a/src/service/execution.ts +++ b/src/service/execution.ts @@ -45,14 +45,10 @@ export async function executeOperation( const timeStart = process.hrtime(); try { - logDb('BEGIN: operation'); - result.data = await func({ timeStart }); - logDb(`END: operation`); result.success = true; } catch (e) { - logDb('FAILED: operation'); logDb(`Error caught for connection ${connectionId}:`, e); result.error = e; } diff --git a/src/service/knexMigrator.ts b/src/service/knexMigrator.ts index 15a041df..48c1a0b9 100644 --- a/src/service/knexMigrator.ts +++ b/src/service/knexMigrator.ts @@ -15,15 +15,15 @@ export const migrationApiMap = { }; /** - * Invoke Knex's migration API. + * Invoke Knex's migration API for given function. * - * @param {(Knex | Knex.Transaction)} trx + * @param {Knex.Transaction} trx * @param {MigrationCommandContext} context * @param {((trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => Promise)} func * @returns {Promise} */ -export async function runMigrateFunc( - trx: Knex | Knex.Transaction, +export async function invokeMigrationApi( + trx: Knex.Transaction, context: MigrationCommandContext, func: (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => Promise ): Promise { diff --git a/src/service/sync.ts b/src/service/sync.ts index 6452a86a..cfc47e35 100644 --- a/src/service/sync.ts +++ b/src/service/sync.ts @@ -83,58 +83,56 @@ async function teardown(trx: Knex.Transaction, context: CommandContext): Promise /** * Synchronize on a single database connection. * - * @param {Knex} connection + * @param {Knex.Transaction} trx * @param {SyncContext} context - * @returns {Promise>} + * @returns {Promise} */ -export async function synchronizeDatabase(connection: Knex, context: SyncContext): Promise> { - return connection.transaction(trx => - executeOperation(context, async options => { - const { connectionId, migrateFunc } = context; - const { timeStart } = options; - const log = dbLogger(connectionId); - - // Trigger onStarted handler if bound. - if (context.params.onStarted) { - await context.params.onStarted({ - connectionId, - success: false, - data: null, - timeElapsed: getElapsedTime(timeStart) - }); - } - - await teardown(trx, context); - - // Trigger onTeardownSuccess if bound. - if (context.params.onTeardownSuccess) { - await context.params.onTeardownSuccess({ - connectionId, - data: null, - success: true, - timeElapsed: getElapsedTime(timeStart) - }); - } - - if (context.params['skip-migration']) { - log('Skipped migrations.'); - } else { - log('Running migrations.'); - await migrateFunc(trx); - } - - await setup(trx, context); - }) - ); +export async function synchronizeDatabase(trx: Knex.Transaction, context: SyncContext): Promise { + return executeOperation(context, async options => { + const { connectionId, migrateFunc } = context; + const { timeStart } = options; + const log = dbLogger(connectionId); + + // Trigger onStarted handler if bound. + if (context.params.onStarted) { + await context.params.onStarted({ + connectionId, + success: false, + data: null, + timeElapsed: getElapsedTime(timeStart) + }); + } + + await teardown(trx, context); + + // Trigger onTeardownSuccess if bound. + if (context.params.onTeardownSuccess) { + await context.params.onTeardownSuccess({ + connectionId, + data: null, + success: true, + timeElapsed: getElapsedTime(timeStart) + }); + } + + if (context.params['skip-migration']) { + log('Skipped migrations.'); + } else { + log('Running migrations.'); + await migrateFunc(trx); + } + + await setup(trx, context); + }); } /** * Prune on a single database connection. * - * @param {Knex} connection + * @param {Knex.Transaction} trx * @param {CommandContext} context - * @returns {Promise>} + * @returns {Promise} */ -export async function pruneDatabase(connection: Knex, context: CommandContext): Promise> { - return connection.transaction(trx => executeOperation(context, () => teardown(trx, context))); +export async function pruneDatabase(trx: Knex.Transaction, context: CommandContext): Promise { + return executeOperation(context, () => teardown(trx, context)); } diff --git a/src/util/db.ts b/src/util/db.ts index fbeaeca8..ba786cc4 100644 --- a/src/util/db.ts +++ b/src/util/db.ts @@ -1,6 +1,8 @@ import * as Knex from 'knex'; import { getConnectionId } from '../config'; import ConnectionConfig from '../domain/ConnectionConfig'; +import { dbLogger } from './logger'; +import ConnectionReference from '../domain/ConnectionReference'; /** * Returns true if the provided object is a knex connection instance. @@ -38,3 +40,27 @@ export function getConfig(db: Knex): ConnectionConfig { export function createInstance(config: ConnectionConfig): Knex { return Knex({ connection: config, client: config.client }); } + +/** + * Run a callback function with in a transaction. + * + * @param {ConnectionReference} db + * @param {(trx: Knex.Transaction) => Promise} callback + * @returns {Promise} + */ +export function withTransaction( + db: ConnectionReference, + callback: (trx: Knex.Transaction) => Promise +): Promise { + const log = dbLogger(db.id); + + return db.connection.transaction(async trx => { + log('BEGIN: transaction'); + + const result = await callback(trx); + + log('END: transaction'); + + return result; + }); +} From 7ca11a3eafbe4d0e2c7d85c66c34f1663aaa82c0 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 25 Apr 2020 22:09:06 +0545 Subject: [PATCH 75/80] Move mapToConnectionReferences() to util/db --- src/api.ts | 37 +------------------------------------ src/util/db.ts | 38 ++++++++++++++++++++++++++++++++++---- 2 files changed, 35 insertions(+), 40 deletions(-) diff --git a/src/api.ts b/src/api.ts index 8b60684f..89c5bff0 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,27 +1,17 @@ -import * as Knex from 'knex'; - import * as init from './init'; import { log } from './util/logger'; -import { getConnectionId } from './config'; -import { isKnexInstance, getConfig, createInstance, withTransaction } from './util/db'; +import { withTransaction, mapToConnectionReferences, DatabaseConnections } from './util/db'; import Configuration from './domain/Configuration'; import CommandResult from './domain/CommandResult'; import CommandParams from './domain/CommandParams'; -import ConnectionConfig from './domain/ConnectionConfig'; import SynchronizeParams from './domain/SynchronizeParams'; -import ConnectionReference from './domain/ConnectionReference'; // Service import { executeProcesses } from './service/execution'; import { synchronizeDatabase, pruneDatabase } from './service/sync'; import { invokeMigrationApi, migrationApiMap } from './service/knexMigrator'; -/** - * Database connections given by the user or the CLI frontend. - */ -export type DatabaseConnections = ConnectionConfig[] | ConnectionConfig | Knex[] | Knex; - /** * Synchronize all the configured database connections. * @@ -218,28 +208,3 @@ export async function migrateList( return executeProcesses(processes, config); } - -/** - * Map user provided connection(s) to the connection instances. - * - * @param {(DatabaseConnections)} conn - * @returns {ConnectionReference[]} - */ -function mapToConnectionReferences(conn: DatabaseConnections): ConnectionReference[] { - const connectionList = Array.isArray(conn) ? conn : [conn]; - - return connectionList.map(connection => { - if (isKnexInstance(connection)) { - log(`Received connection instance to database: ${connection.client.config.connection.database}`); - - // TODO: Ask for `id` explicitly in for programmatic API, - // when Knex instance is passed directly. - // This implies a breaking change with the programmatic API. - return { connection, id: getConnectionId(getConfig(connection)) }; - } - - log(`Creating a connection to database: ${connection.host}/${connection.database}`); - - return { connection: createInstance(connection), id: getConnectionId(connection) }; - }); -} diff --git a/src/util/db.ts b/src/util/db.ts index ba786cc4..469c501a 100644 --- a/src/util/db.ts +++ b/src/util/db.ts @@ -1,9 +1,14 @@ import * as Knex from 'knex'; import { getConnectionId } from '../config'; import ConnectionConfig from '../domain/ConnectionConfig'; -import { dbLogger } from './logger'; +import { dbLogger, log } from './logger'; import ConnectionReference from '../domain/ConnectionReference'; +/** + * Database connections given by the user or the CLI frontend. + */ +export type DatabaseConnections = ConnectionConfig[] | ConnectionConfig | Knex[] | Knex; + /** * Returns true if the provided object is a knex connection instance. * @@ -52,15 +57,40 @@ export function withTransaction( db: ConnectionReference, callback: (trx: Knex.Transaction) => Promise ): Promise { - const log = dbLogger(db.id); + const dbLog = dbLogger(db.id); return db.connection.transaction(async trx => { - log('BEGIN: transaction'); + dbLog('BEGIN: transaction'); const result = await callback(trx); - log('END: transaction'); + dbLog('END: transaction'); return result; }); } + +/** + * Map user provided connection(s) to the connection instances. + * + * @param {(DatabaseConnections)} conn + * @returns {ConnectionReference[]} + */ +export function mapToConnectionReferences(conn: DatabaseConnections): ConnectionReference[] { + const connectionList = Array.isArray(conn) ? conn : [conn]; + + return connectionList.map(connection => { + if (isKnexInstance(connection)) { + log(`Received connection instance to database: ${connection.client.config.connection.database}`); + + // TODO: Ask for `id` explicitly in for programmatic API, + // when Knex instance is passed directly. + // This implies a breaking change with the programmatic API. + return { connection, id: getConnectionId(getConfig(connection)) }; + } + + log(`Creating a connection to database: ${connection.host}/${connection.database}`); + + return { connection: createInstance(connection), id: getConnectionId(connection) }; + }); +} From 53edbd20266ddecb69b104be19608fb39570ce45 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 25 Apr 2020 22:16:17 +0545 Subject: [PATCH 76/80] Add note for the programmatic api --- src/api.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/api.ts b/src/api.ts index 89c5bff0..9fe8b21d 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,3 +1,11 @@ +/** + * Programmatic API + * ---------------- + * This module defines the Programmatic API of sync-db. + * The functions exposed here are used by the CLI frontend and + * are also meant to be the public interface for the developers using it as a package. + */ + import * as init from './init'; import { log } from './util/logger'; import { withTransaction, mapToConnectionReferences, DatabaseConnections } from './util/db'; @@ -83,7 +91,9 @@ export async function prune( const params: CommandParams = { ...options }; const connections = mapToConnectionReferences(conn); - await init.prepare(config, {}); + + // TODO: Need to preload the SQL source code under this step. + await init.prepare(config, { loadSqlSources: true }); const processes = connections.map(connection => () => withTransaction(connection, trx => From 3975f2ba23918546be1e710160e96041207a3fa0 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 25 Apr 2020 22:22:28 +0545 Subject: [PATCH 77/80] Revert util/io change --- src/util/io.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/util/io.ts b/src/util/io.ts index 25b03507..dfd6ddfd 100644 --- a/src/util/io.ts +++ b/src/util/io.ts @@ -1,16 +1,13 @@ import { green, red } from 'chalk'; -export type ModifierFunc = (message: string) => string; - /** * Prints a line into the console (stdout). * * @param {string} [message=''] - * @param {ModifierFunc} [modifier=(str: string) => str] * @returns {Promise} */ -export function printLine(message: string = '', modifier: ModifierFunc = (str: string) => str): Promise { - return print(modifier(message.toString()) + '\n'); +export function printLine(message: string = ''): Promise { + return print(message.toString() + '\n'); } /** @@ -20,7 +17,7 @@ export function printLine(message: string = '', modifier: ModifierFunc = (str: s * @returns {Promise} */ export function printInfo(message: string = ''): Promise { - return printLine(message, green); + return printLine(green(message)); } /** From d644b77da5d84cc7233717a3e02e562680463520 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 25 Apr 2020 22:33:16 +0545 Subject: [PATCH 78/80] Organize interfaces --- src/api.ts | 48 +++++++++---------- src/commands/migrate-latest.ts | 6 +-- src/commands/migrate-list.ts | 6 +-- src/commands/migrate-rollback.ts | 6 +-- src/commands/prune.ts | 6 +-- src/commands/synchronize.ts | 16 +++---- src/domain/CommandContext.ts | 10 ---- src/domain/CommandParams.ts | 8 ---- src/domain/MigrationCommandContext.ts | 8 ++-- src/domain/SyncContext.ts | 4 +- src/domain/SyncDbOptions.ts | 11 ----- src/domain/SynchronizeParams.ts | 14 +++--- .../{ => migration}/MigrationContext.ts | 0 src/domain/{ => migration}/MigrationRunner.ts | 0 .../{ => migration}/SqlMigrationEntry.ts | 2 +- src/domain/operation/OperationContext.ts | 10 ++++ src/domain/operation/OperationParams.ts | 8 ++++ .../OperationResult.ts} | 4 +- src/init.ts | 2 +- src/migration/KnexMigrationSource.ts | 2 +- src/migration/SqlMigrationContext.ts | 6 +-- src/service/execution.ts | 12 ++--- src/service/knexMigrator.ts | 6 +-- src/service/migrator.ts | 2 +- src/service/sync.ts | 18 +++---- 25 files changed, 102 insertions(+), 113 deletions(-) delete mode 100644 src/domain/CommandContext.ts delete mode 100644 src/domain/CommandParams.ts delete mode 100644 src/domain/SyncDbOptions.ts rename src/domain/{ => migration}/MigrationContext.ts (100%) rename src/domain/{ => migration}/MigrationRunner.ts (100%) rename src/domain/{ => migration}/SqlMigrationEntry.ts (80%) create mode 100644 src/domain/operation/OperationContext.ts create mode 100644 src/domain/operation/OperationParams.ts rename src/domain/{CommandResult.ts => operation/OperationResult.ts} (58%) diff --git a/src/api.ts b/src/api.ts index 9fe8b21d..0ff0bbaa 100644 --- a/src/api.ts +++ b/src/api.ts @@ -11,8 +11,8 @@ import { log } from './util/logger'; import { withTransaction, mapToConnectionReferences, DatabaseConnections } from './util/db'; import Configuration from './domain/Configuration'; -import CommandResult from './domain/CommandResult'; -import CommandParams from './domain/CommandParams'; +import OperationResult from './domain/operation/OperationResult'; +import OperationParams from './domain/operation/OperationParams'; import SynchronizeParams from './domain/SynchronizeParams'; // Service @@ -26,13 +26,13 @@ import { invokeMigrationApi, migrationApiMap } from './service/knexMigrator'; * @param {Configuration} config * @param {DatabaseConnections} conn * @param {SynchronizeParams} [options] - * @returns {Promise} + * @returns {Promise} */ export async function synchronize( config: Configuration, conn: DatabaseConnections, options?: SynchronizeParams -): Promise { +): Promise { log('Synchronize'); const params: SynchronizeParams = { @@ -79,17 +79,17 @@ export async function synchronize( * * @param {Configuration} config * @param {(DatabaseConnections)} conn - * @param {CommandParams} [options] - * @returns {Promise} + * @param {OperationParams} [options] + * @returns {Promise} */ export async function prune( config: Configuration, conn: DatabaseConnections, - options?: CommandParams -): Promise { + options?: OperationParams +): Promise { log('Prune'); - const params: CommandParams = { ...options }; + const params: OperationParams = { ...options }; const connections = mapToConnectionReferences(conn); // TODO: Need to preload the SQL source code under this step. @@ -113,17 +113,17 @@ export async function prune( * * @param {Configuration} config * @param {(DatabaseConnections)} conn - * @param {CommandParams} [options] - * @returns {Promise} + * @param {OperationParams} [options] + * @returns {Promise} */ export async function migrateLatest( config: Configuration, conn: DatabaseConnections, - options?: CommandParams -): Promise { + options?: OperationParams +): Promise { log('Migrate Latest'); - const params: CommandParams = { ...options }; + const params: OperationParams = { ...options }; const connections = mapToConnectionReferences(conn); const { knexMigrationConfig } = await init.prepare(config, { loadMigrations: true }); @@ -150,17 +150,17 @@ export async function migrateLatest( * * @param {Configuration} config * @param {(DatabaseConnections)} conn - * @param {CommandParams} [options] - * @returns {Promise} + * @param {OperationParams} [options] + * @returns {Promise} */ export async function migrateRollback( config: Configuration, conn: DatabaseConnections, - options?: CommandParams -): Promise { + options?: OperationParams +): Promise { log('Migrate Rollback'); - const params: CommandParams = { ...options }; + const params: OperationParams = { ...options }; const connections = mapToConnectionReferences(conn); const { knexMigrationConfig } = await init.prepare(config, { loadMigrations: true }); @@ -187,17 +187,17 @@ export async function migrateRollback( * * @param {Configuration} config * @param {(DatabaseConnections)} conn - * @param {CommandParams} [options] - * @returns {Promise} + * @param {OperationParams} [options] + * @returns {Promise} */ export async function migrateList( config: Configuration, conn: DatabaseConnections, - options?: CommandParams -): Promise { + options?: OperationParams +): Promise { log('Migrate List'); - const params: CommandParams = { ...options }; + const params: OperationParams = { ...options }; const connections = mapToConnectionReferences(conn); const { knexMigrationConfig } = await init.prepare(config, { loadMigrations: true }); diff --git a/src/commands/migrate-latest.ts b/src/commands/migrate-latest.ts index 602de495..73e9e7aa 100644 --- a/src/commands/migrate-latest.ts +++ b/src/commands/migrate-latest.ts @@ -5,7 +5,7 @@ import { migrateLatest } from '../api'; import { printLine, printError, printInfo } from '../util/io'; import { loadConfig, resolveConnections } from '..'; import { dbLogger } from '../util/logger'; -import CommandResult from '../domain/CommandResult'; +import OperationResult from '../domain/operation/OperationResult'; /** * Migration command handler. @@ -16,7 +16,7 @@ class MigrateLatest extends Command { /** * Success handler for each connection. */ - onSuccess = async (result: CommandResult) => { + onSuccess = async (result: OperationResult) => { const log = dbLogger(result.connectionId); const [num, list] = result.data; const alreadyUpToDate = num && list.length === 0; @@ -42,7 +42,7 @@ class MigrateLatest extends Command { /** * Failure handler for each connection. */ - onFailed = async (result: CommandResult) => { + onFailed = async (result: OperationResult) => { printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); await printError(` ${result.error}\n`); diff --git a/src/commands/migrate-list.ts b/src/commands/migrate-list.ts index 32e28759..c48b2ccc 100644 --- a/src/commands/migrate-list.ts +++ b/src/commands/migrate-list.ts @@ -4,7 +4,7 @@ import { bold, grey, red, cyan, yellow } from 'chalk'; import { migrateList } from '../api'; import { printLine, printError } from '../util/io'; import { loadConfig, resolveConnections } from '..'; -import CommandResult from '../domain/CommandResult'; +import OperationResult from '../domain/operation/OperationResult'; /** * Migration command handler. @@ -15,7 +15,7 @@ class MigrateList extends Command { /** * Success handler for a connection. */ - onSuccess = async (result: CommandResult) => { + onSuccess = async (result: OperationResult) => { await printLine(bold(` ▸ ${result.connectionId}`)); const [list1, list2] = result.data; @@ -46,7 +46,7 @@ class MigrateList extends Command { /** * Failure handler for a connection. */ - onFailed = async (result: CommandResult) => { + onFailed = async (result: OperationResult) => { printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); await printError(` ${result.error}\n`); diff --git a/src/commands/migrate-rollback.ts b/src/commands/migrate-rollback.ts index db37bd6f..be4b426c 100644 --- a/src/commands/migrate-rollback.ts +++ b/src/commands/migrate-rollback.ts @@ -5,7 +5,7 @@ import { migrateRollback } from '../api'; import { printLine, printError, printInfo } from '../util/io'; import { loadConfig, resolveConnections } from '..'; import { dbLogger } from '../util/logger'; -import CommandResult from '../domain/CommandResult'; +import OperationResult from '../domain/operation/OperationResult'; /** * Migration command handler. @@ -16,7 +16,7 @@ class MigrateRollback extends Command { /** * Success handler for each connection. */ - onSuccess = async (result: CommandResult) => { + onSuccess = async (result: OperationResult) => { const log = dbLogger(result.connectionId); const [num, list] = result.data; const allRolledBack = num === 0; @@ -42,7 +42,7 @@ class MigrateRollback extends Command { /** * Failure handler for each connection. */ - onFailed = async (result: CommandResult) => { + onFailed = async (result: OperationResult) => { await printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); await printError(` ${result.error}\n`); diff --git a/src/commands/prune.ts b/src/commands/prune.ts index 77e22237..7b6976c2 100644 --- a/src/commands/prune.ts +++ b/src/commands/prune.ts @@ -3,7 +3,7 @@ import { bold, red } from 'chalk'; import { printLine, printError } from '../util/io'; -import CommandResult from '../domain/CommandResult'; +import OperationResult from '../domain/operation/OperationResult'; import { prune } from '../api'; import { loadConfig, resolveConnections } from '..'; @@ -12,14 +12,14 @@ class Prune extends Command { /** * Success handler for each connection. */ - onSuccess = async (result: CommandResult) => { + onSuccess = async (result: OperationResult) => { await printLine(bold(` ▸ ${result.connectionId} - Successful`) + ` (${result.timeElapsed}s)`); }; /** * Failure handler for each connection. */ - onFailed = async (result: CommandResult) => { + onFailed = async (result: OperationResult) => { await printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); await printError(` ${result.error}\n`); diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index be1b0686..bbed8536 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -6,7 +6,7 @@ import { log, dbLogger } from '../util/logger'; import { loadConfig, resolveConnections } from '../config'; import { printError, printLine, printInfo } from '../util/io'; import { synchronize } from '../api'; -import CommandResult from '../domain/CommandResult'; +import OperationResult from '../domain/operation/OperationResult'; /** * Synchronize command handler. @@ -24,18 +24,18 @@ class Synchronize extends Command { 'skip-migration': flags.boolean({ description: 'Skip running migrations' }) }; - onStarted = async (result: CommandResult) => { + onStarted = async (result: OperationResult) => { await printLine(bold(` ▸ ${result.connectionId}`)); await printInfo(' [✓] Synchronization - started'); }; - onPruneSuccess = (result: CommandResult) => + onPruneSuccess = (result: OperationResult) => printLine(green(' [✓] Synchronization - pruned') + ` (${result.timeElapsed}s)`); /** * Success handler for migration run during sync process. */ - onMigrationSuccess = async (result: CommandResult) => { + onMigrationSuccess = async (result: OperationResult) => { const logDb = dbLogger(result.connectionId); const [num, list] = result.data; const alreadyUpToDate = num && list.length === 0; @@ -59,7 +59,7 @@ class Synchronize extends Command { /** * Failure handler for migration during sync process. */ - onMigrationFailed = async (result: CommandResult) => { + onMigrationFailed = async (result: OperationResult) => { await printLine(red(` [✖] Migrations - failed (${result.timeElapsed}s)\n`)); // await printError(` ${result.error}\n`); @@ -68,13 +68,13 @@ class Synchronize extends Command { /** * Success handler for each connection. */ - onSuccess = (result: CommandResult) => + onSuccess = (result: OperationResult) => printLine(green(' [✓] Synchronization - completed') + ` (${result.timeElapsed}s)\n`); /** * Failure handler for each connection. */ - onFailed = async (result: CommandResult) => { + onFailed = async (result: OperationResult) => { await printLine(red(` [✖] Synchronization - failed (${result.timeElapsed}s)\n`)); }; @@ -86,7 +86,7 @@ class Synchronize extends Command { * @returns {Promise<{ totalCount: number, failedCount: number, successfulCount: number }>} */ async processResults( - results: CommandResult[] + results: OperationResult[] ): Promise<{ totalCount: number; failedCount: number; successfulCount: number }> { const totalCount = results.length; const failedAttempts = results.filter(result => !result.success); diff --git a/src/domain/CommandContext.ts b/src/domain/CommandContext.ts deleted file mode 100644 index a8532689..00000000 --- a/src/domain/CommandContext.ts +++ /dev/null @@ -1,10 +0,0 @@ -import Configuration from './Configuration'; -import CommandParams from './CommandParams'; - -interface CommandContext { - config: Configuration; - connectionId: string; - params: CommandParams; -} - -export default CommandContext; diff --git a/src/domain/CommandParams.ts b/src/domain/CommandParams.ts deleted file mode 100644 index d8a70a2d..00000000 --- a/src/domain/CommandParams.ts +++ /dev/null @@ -1,8 +0,0 @@ -import CommandResult from './CommandResult'; - -interface CommandParams { - onSuccess?: (result: CommandResult) => Promise; - onFailed?: (result: CommandResult) => Promise; -} - -export default CommandParams; diff --git a/src/domain/MigrationCommandContext.ts b/src/domain/MigrationCommandContext.ts index 0d5c030c..38cab9c1 100644 --- a/src/domain/MigrationCommandContext.ts +++ b/src/domain/MigrationCommandContext.ts @@ -1,10 +1,10 @@ import * as Knex from 'knex'; -import CommandParams from './CommandParams'; -import CommandContext from './CommandContext'; +import OperationParams from './operation/OperationParams'; +import OperationContext from './operation/OperationContext'; -interface MigrationCommandContext extends CommandContext { - params: CommandParams; +interface MigrationCommandContext extends OperationContext { + params: OperationParams; knexMigrationConfig: Knex.MigratorConfig; } diff --git a/src/domain/SyncContext.ts b/src/domain/SyncContext.ts index c177191e..4016448b 100644 --- a/src/domain/SyncContext.ts +++ b/src/domain/SyncContext.ts @@ -1,12 +1,12 @@ import * as Knex from 'knex'; import SynchronizeParams from './SynchronizeParams'; -import CommandContext from './CommandContext'; +import OperationContext from './operation/OperationContext'; /** * Synchronize context for a database connection. */ -interface SyncContext extends CommandContext { +interface SyncContext extends OperationContext { params: SynchronizeParams; migrateFunc: (trx: Knex.Transaction) => Promise; } diff --git a/src/domain/SyncDbOptions.ts b/src/domain/SyncDbOptions.ts deleted file mode 100644 index 4104dbf6..00000000 --- a/src/domain/SyncDbOptions.ts +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Interface for sync-db options(flags). - */ -interface SyncDbOptions { - version: void; - help: void; - force: boolean; - 'generate-connections': boolean; -} - -export default SyncDbOptions; diff --git a/src/domain/SynchronizeParams.ts b/src/domain/SynchronizeParams.ts index 0e91d527..0ff82f5a 100644 --- a/src/domain/SynchronizeParams.ts +++ b/src/domain/SynchronizeParams.ts @@ -1,16 +1,16 @@ -import CommandParams from './CommandParams'; -import CommandResult from './CommandResult'; +import OperationParams from './operation/OperationParams'; +import OperationResult from './operation/OperationResult'; /** * Synchronize parameters. */ -interface SynchronizeParams extends CommandParams { +interface SynchronizeParams extends OperationParams { force: boolean; 'skip-migration': boolean; - onStarted?: (result: CommandResult) => Promise; - onTeardownSuccess?: (result: CommandResult) => Promise; - onMigrationSuccess?: (result: CommandResult) => Promise; - onMigrationFailed?: (result: CommandResult) => Promise; + onStarted?: (result: OperationResult) => Promise; + onTeardownSuccess?: (result: OperationResult) => Promise; + onMigrationSuccess?: (result: OperationResult) => Promise; + onMigrationFailed?: (result: OperationResult) => Promise; } export default SynchronizeParams; diff --git a/src/domain/MigrationContext.ts b/src/domain/migration/MigrationContext.ts similarity index 100% rename from src/domain/MigrationContext.ts rename to src/domain/migration/MigrationContext.ts diff --git a/src/domain/MigrationRunner.ts b/src/domain/migration/MigrationRunner.ts similarity index 100% rename from src/domain/MigrationRunner.ts rename to src/domain/migration/MigrationRunner.ts diff --git a/src/domain/SqlMigrationEntry.ts b/src/domain/migration/SqlMigrationEntry.ts similarity index 80% rename from src/domain/SqlMigrationEntry.ts rename to src/domain/migration/SqlMigrationEntry.ts index e08dc084..0332fae9 100644 --- a/src/domain/SqlMigrationEntry.ts +++ b/src/domain/migration/SqlMigrationEntry.ts @@ -1,4 +1,4 @@ -import SqlCode from './SqlCode'; +import SqlCode from '../SqlCode'; interface SqlMigrationEntry { name: string; diff --git a/src/domain/operation/OperationContext.ts b/src/domain/operation/OperationContext.ts new file mode 100644 index 00000000..73997276 --- /dev/null +++ b/src/domain/operation/OperationContext.ts @@ -0,0 +1,10 @@ +import Configuration from '../Configuration'; +import OperationParams from './OperationParams'; + +interface OperationContext { + config: Configuration; + connectionId: string; + params: OperationParams; +} + +export default OperationContext; diff --git a/src/domain/operation/OperationParams.ts b/src/domain/operation/OperationParams.ts new file mode 100644 index 00000000..7c8048da --- /dev/null +++ b/src/domain/operation/OperationParams.ts @@ -0,0 +1,8 @@ +import OperationResult from './OperationResult'; + +interface OperationParams { + onSuccess?: (result: OperationResult) => Promise; + onFailed?: (result: OperationResult) => Promise; +} + +export default OperationParams; diff --git a/src/domain/CommandResult.ts b/src/domain/operation/OperationResult.ts similarity index 58% rename from src/domain/CommandResult.ts rename to src/domain/operation/OperationResult.ts index 7fc1dda5..1d790f25 100644 --- a/src/domain/CommandResult.ts +++ b/src/domain/operation/OperationResult.ts @@ -1,4 +1,4 @@ -interface CommandResult { +interface OperationResult { connectionId: string; success: boolean; timeElapsed: number; @@ -6,4 +6,4 @@ interface CommandResult { error?: any; } -export default CommandResult; +export default OperationResult; diff --git a/src/init.ts b/src/init.ts index faa7470b..7c1d7fbd 100644 --- a/src/init.ts +++ b/src/init.ts @@ -5,7 +5,7 @@ import { log } from './util/logger'; import { validate, isCLI } from './config'; import Configuration from './domain/Configuration'; import * as migratorService from './service/migrator'; -import MigrationContext from './domain/MigrationContext'; +import MigrationContext from './domain/migration/MigrationContext'; import KnexMigrationSource from './migration/KnexMigrationSource'; import SqlMigrationContext from './migration/SqlMigrationContext'; diff --git a/src/migration/KnexMigrationSource.ts b/src/migration/KnexMigrationSource.ts index 0d879726..93346c94 100644 --- a/src/migration/KnexMigrationSource.ts +++ b/src/migration/KnexMigrationSource.ts @@ -1,5 +1,5 @@ import { dbLogger } from '../util/logger'; -import MigrationContext from '../domain/MigrationContext'; +import MigrationContext from '../domain/migration/MigrationContext'; /** * MigrationSource class for the Knex Migration API. diff --git a/src/migration/SqlMigrationContext.ts b/src/migration/SqlMigrationContext.ts index c9e3a840..8b895cea 100644 --- a/src/migration/SqlMigrationContext.ts +++ b/src/migration/SqlMigrationContext.ts @@ -1,9 +1,9 @@ import * as Knex from 'knex'; -import MigrationRunner from '../domain/MigrationRunner'; +import MigrationRunner from '../domain/migration/MigrationRunner'; import { dbLogger, log as logger } from '../util/logger'; -import MigrationContext from '../domain/MigrationContext'; -import SqlMigrationEntry from '../domain/SqlMigrationEntry'; +import MigrationContext from '../domain/migration/MigrationContext'; +import SqlMigrationEntry from '../domain/migration/SqlMigrationEntry'; /** * SQL source migration context for KnexMigrationSource. diff --git a/src/service/execution.ts b/src/service/execution.ts index d98e9b86..8ba5eefb 100644 --- a/src/service/execution.ts +++ b/src/service/execution.ts @@ -1,8 +1,8 @@ import { log, dbLogger } from '../util/logger'; import Configuration from '../domain/Configuration'; import { Promiser, runSequentially } from '../util/promise'; -import CommandContext from '../domain/CommandContext'; -import CommandResult from '../domain/CommandResult'; +import OperationContext from '../domain/operation/OperationContext'; +import OperationResult from '../domain/operation/OperationResult'; import { getElapsedTime } from '../util/ts'; /** @@ -32,15 +32,15 @@ export function executeProcesses(processes: Promiser[], config: Configurat * * @param {T} context * @param {(options: any) => Promise} func - * @returns {Promise} + * @returns {Promise} */ -export async function executeOperation( +export async function executeOperation( context: T, func: (options: any) => Promise -): Promise { +): Promise { const { connectionId } = context; const logDb = dbLogger(connectionId); - const result: CommandResult = { connectionId, success: false, data: null, timeElapsed: 0 }; + const result: OperationResult = { connectionId, success: false, data: null, timeElapsed: 0 }; const timeStart = process.hrtime(); diff --git a/src/service/knexMigrator.ts b/src/service/knexMigrator.ts index 48c1a0b9..6b4212bd 100644 --- a/src/service/knexMigrator.ts +++ b/src/service/knexMigrator.ts @@ -2,7 +2,7 @@ import Knex from 'knex'; import { dbLogger } from '../util/logger'; import MigrationCommandContext from '../domain/MigrationCommandContext'; -import CommandResult from '../domain/CommandResult'; +import OperationResult from '../domain/operation/OperationResult'; import { executeOperation } from './execution'; /** @@ -20,13 +20,13 @@ export const migrationApiMap = { * @param {Knex.Transaction} trx * @param {MigrationCommandContext} context * @param {((trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => Promise)} func - * @returns {Promise} + * @returns {Promise} */ export async function invokeMigrationApi( trx: Knex.Transaction, context: MigrationCommandContext, func: (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => Promise -): Promise { +): Promise { return executeOperation(context, async () => { const { knexMigrationConfig } = context; const funcName = func.name || 'func'; diff --git a/src/service/migrator.ts b/src/service/migrator.ts index e6dbb183..1bed8b3d 100644 --- a/src/service/migrator.ts +++ b/src/service/migrator.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import { glob, exists } from '../util/fs'; import { resolveFile } from './sqlRunner'; -import SqlMigrationEntry from '../domain/SqlMigrationEntry'; +import SqlMigrationEntry from '../domain/migration/SqlMigrationEntry'; const FILE_PATTERN = /(.+)\.(up|down)\.sql$/; diff --git a/src/service/sync.ts b/src/service/sync.ts index cfc47e35..bb173b8a 100644 --- a/src/service/sync.ts +++ b/src/service/sync.ts @@ -5,8 +5,8 @@ import { dbLogger } from '../util/logger'; import { getElapsedTime } from '../util/ts'; import SyncContext from '../domain/SyncContext'; import * as configInjection from './configInjection'; -import CommandResult from '../domain/CommandResult'; -import CommandContext from '../domain/CommandContext'; +import OperationResult from '../domain/operation/OperationResult'; +import OperationContext from '../domain/operation/OperationContext'; import { executeOperation } from './execution'; /** @@ -64,10 +64,10 @@ async function setup(trx: Knex.Transaction, context: SyncContext): Promise * They're executed in the reverse order of their creation. * * @param {Knex.Transaction} trx - * @param {CommandContext} context + * @param {OperationContext} context * @returns {Promise} */ -async function teardown(trx: Knex.Transaction, context: CommandContext): Promise { +async function teardown(trx: Knex.Transaction, context: OperationContext): Promise { const { basePath, sql } = context.config; const log = dbLogger(context.connectionId); @@ -85,9 +85,9 @@ async function teardown(trx: Knex.Transaction, context: CommandContext): Promise * * @param {Knex.Transaction} trx * @param {SyncContext} context - * @returns {Promise} + * @returns {Promise} */ -export async function synchronizeDatabase(trx: Knex.Transaction, context: SyncContext): Promise { +export async function synchronizeDatabase(trx: Knex.Transaction, context: SyncContext): Promise { return executeOperation(context, async options => { const { connectionId, migrateFunc } = context; const { timeStart } = options; @@ -130,9 +130,9 @@ export async function synchronizeDatabase(trx: Knex.Transaction, context: SyncCo * Prune on a single database connection. * * @param {Knex.Transaction} trx - * @param {CommandContext} context - * @returns {Promise} + * @param {OperationContext} context + * @returns {Promise} */ -export async function pruneDatabase(trx: Knex.Transaction, context: CommandContext): Promise { +export async function pruneDatabase(trx: Knex.Transaction, context: OperationContext): Promise { return executeOperation(context, () => teardown(trx, context)); } From 284ed37aeaeee457e20fd593ec524bf6d3ce4ff5 Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 25 Apr 2020 22:57:35 +0545 Subject: [PATCH 79/80] Fix tests --- src/init.ts | 2 +- src/service/knexMigrator.ts | 2 +- test/migration/KnexMigrationSource.test.ts | 2 +- test/migration/SqlMigrationContext.test.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/init.ts b/src/init.ts index 7c1d7fbd..d51d7e64 100644 --- a/src/init.ts +++ b/src/init.ts @@ -1,4 +1,4 @@ -import Knex from 'knex'; +import * as Knex from 'knex'; import * as path from 'path'; import { log } from './util/logger'; diff --git a/src/service/knexMigrator.ts b/src/service/knexMigrator.ts index 6b4212bd..22545915 100644 --- a/src/service/knexMigrator.ts +++ b/src/service/knexMigrator.ts @@ -1,4 +1,4 @@ -import Knex from 'knex'; +import * as Knex from 'knex'; import { dbLogger } from '../util/logger'; import MigrationCommandContext from '../domain/MigrationCommandContext'; diff --git a/test/migration/KnexMigrationSource.test.ts b/test/migration/KnexMigrationSource.test.ts index ecd100d2..836b1b42 100644 --- a/test/migration/KnexMigrationSource.test.ts +++ b/test/migration/KnexMigrationSource.test.ts @@ -1,7 +1,7 @@ import { expect } from 'chai'; import { describe, it } from 'mocha'; -import MigrationContext from '../../src/domain/MigrationContext'; +import MigrationContext from '../../src/domain/migration/MigrationContext'; import KnexMigrationSource from '../../src/migration/KnexMigrationSource'; describe('UTIL: KnexMigrationSource', () => { diff --git a/test/migration/SqlMigrationContext.test.ts b/test/migration/SqlMigrationContext.test.ts index 2a985078..a5ee09f1 100644 --- a/test/migration/SqlMigrationContext.test.ts +++ b/test/migration/SqlMigrationContext.test.ts @@ -1,7 +1,7 @@ import { expect } from 'chai'; import { describe, it } from 'mocha'; -import SqlMigrationEntry from '../../src/domain/SqlMigrationEntry'; +import SqlMigrationEntry from '../../src/domain/migration/SqlMigrationEntry'; import SqlMigrationContext from '../../src/migration/SqlMigrationContext'; describe('UTIL: SqlMigrationContext', () => { From 10fd58e21eb4ff4e6e019e569e1cf179715a62ac Mon Sep 17 00:00:00 2001 From: Kabir Baidhya Date: Sat, 25 Apr 2020 23:08:10 +0545 Subject: [PATCH 80/80] Tests, aesthetics and improvements --- src/api.ts | 76 ++++++++----------- src/commands/migrate-latest.ts | 11 +-- src/commands/migrate-list.ts | 21 +++-- src/commands/migrate-rollback.ts | 13 ++-- src/commands/prune.ts | 12 +-- src/commands/synchronize.ts | 19 +++-- ...nCommandContext.ts => MigrationContext.ts} | 4 +- .../{SyncContext.ts => SynchronizeContext.ts} | 4 +- src/domain/migration/MigrationContext.ts | 13 ---- src/init.ts | 12 +-- src/migration/KnexMigrationSource.ts | 11 ++- .../domain}/MigrationRunner.ts | 0 .../domain/MigrationSourceContext.ts | 14 ++++ .../domain}/SqlMigrationEntry.ts | 2 +- src/migration/service/knexMigrator.ts | 58 ++++++++++++++ src/{ => migration}/service/migrator.ts | 6 +- .../SqlMigrationSourceContext.ts} | 16 ++-- src/service/configInjection.ts | 10 +-- src/service/execution.ts | 4 +- src/service/knexMigrator.ts | 43 ----------- src/service/sync.ts | 16 ++-- test/migration/KnexMigrationSource.test.ts | 6 +- test/migration/SqlMigrationContext.test.ts | 8 +- test/{service => migration}/migrator.test.ts | 4 +- 24 files changed, 190 insertions(+), 193 deletions(-) rename src/domain/{MigrationCommandContext.ts => MigrationContext.ts} (69%) rename src/domain/{SyncContext.ts => SynchronizeContext.ts} (76%) delete mode 100644 src/domain/migration/MigrationContext.ts rename src/{domain/migration => migration/domain}/MigrationRunner.ts (100%) create mode 100644 src/migration/domain/MigrationSourceContext.ts rename src/{domain/migration => migration/domain}/SqlMigrationEntry.ts (76%) create mode 100644 src/migration/service/knexMigrator.ts rename src/{ => migration}/service/migrator.ts (91%) rename src/migration/{SqlMigrationContext.ts => source-types/SqlMigrationSourceContext.ts} (78%) delete mode 100644 src/service/knexMigrator.ts rename test/{service => migration}/migrator.test.ts (97%) diff --git a/src/api.ts b/src/api.ts index 0ff0bbaa..01bd7aa2 100644 --- a/src/api.ts +++ b/src/api.ts @@ -11,14 +11,14 @@ import { log } from './util/logger'; import { withTransaction, mapToConnectionReferences, DatabaseConnections } from './util/db'; import Configuration from './domain/Configuration'; -import OperationResult from './domain/operation/OperationResult'; -import OperationParams from './domain/operation/OperationParams'; import SynchronizeParams from './domain/SynchronizeParams'; +import OperationParams from './domain/operation/OperationParams'; +import OperationResult from './domain/operation/OperationResult'; // Service import { executeProcesses } from './service/execution'; -import { synchronizeDatabase, pruneDatabase } from './service/sync'; -import { invokeMigrationApi, migrationApiMap } from './service/knexMigrator'; +import { runSynchronize, runPrune } from './service/sync'; +import { invokeMigrationApi, KnexMigrationAPI } from './migration/service/knexMigrator'; /** * Synchronize all the configured database connections. @@ -50,21 +50,17 @@ export async function synchronize( const connections = mapToConnectionReferences(conn); const processes = connections.map(connection => () => withTransaction(connection, trx => - synchronizeDatabase(trx, { + runSynchronize(trx, { config, params, connectionId: connection.id, migrateFunc: t => - invokeMigrationApi( - t, - { - config, - connectionId: connection.id, - knexMigrationConfig: knexMigrationConfig(connection.id), - params: { ...params, onSuccess: params.onMigrationSuccess, onFailed: params.onMigrationFailed } - }, - migrationApiMap['migrate.latest'] - ) + invokeMigrationApi(t, KnexMigrationAPI.MIGRATE_LATEST, { + config, + connectionId: connection.id, + knexMigrationConfig: knexMigrationConfig(connection.id), + params: { ...params, onSuccess: params.onMigrationSuccess, onFailed: params.onMigrationFailed } + }) }) ) ); @@ -97,7 +93,7 @@ export async function prune( const processes = connections.map(connection => () => withTransaction(connection, trx => - pruneDatabase(trx, { + runPrune(trx, { config, params, connectionId: connection.id @@ -129,16 +125,12 @@ export async function migrateLatest( const processes = connections.map(connection => () => withTransaction(connection, trx => - invokeMigrationApi( - trx, - { - config, - params, - connectionId: connection.id, - knexMigrationConfig: knexMigrationConfig(connection.id) - }, - migrationApiMap['migrate.latest'] - ) + invokeMigrationApi(trx, KnexMigrationAPI.MIGRATE_LATEST, { + config, + params, + connectionId: connection.id, + knexMigrationConfig: knexMigrationConfig(connection.id) + }) ) ); @@ -166,16 +158,12 @@ export async function migrateRollback( const processes = connections.map(connection => () => withTransaction(connection, trx => - invokeMigrationApi( - trx, - { - config, - params, - connectionId: connection.id, - knexMigrationConfig: knexMigrationConfig(connection.id) - }, - migrationApiMap['migrate.rollback'] - ) + invokeMigrationApi(trx, KnexMigrationAPI.MIGRATE_ROLLBACK, { + config, + params, + connectionId: connection.id, + knexMigrationConfig: knexMigrationConfig(connection.id) + }) ) ); @@ -203,16 +191,12 @@ export async function migrateList( const processes = connections.map(connection => () => withTransaction(connection, trx => - invokeMigrationApi( - trx, - { - config, - params, - connectionId: connection.id, - knexMigrationConfig: knexMigrationConfig(connection.id) - }, - migrationApiMap['migrate.list'] - ) + invokeMigrationApi(trx, KnexMigrationAPI.MIGRATE_LIST, { + config, + params, + connectionId: connection.id, + knexMigrationConfig: knexMigrationConfig(connection.id) + }) ) ); diff --git a/src/commands/migrate-latest.ts b/src/commands/migrate-latest.ts index 73e9e7aa..72c69c04 100644 --- a/src/commands/migrate-latest.ts +++ b/src/commands/migrate-latest.ts @@ -2,19 +2,16 @@ import { Command } from '@oclif/command'; import { bold, red, cyan } from 'chalk'; import { migrateLatest } from '../api'; -import { printLine, printError, printInfo } from '../util/io'; -import { loadConfig, resolveConnections } from '..'; import { dbLogger } from '../util/logger'; +import { loadConfig, resolveConnections } from '..'; +import { printLine, printError, printInfo } from '../util/io'; import OperationResult from '../domain/operation/OperationResult'; -/** - * Migration command handler. - */ class MigrateLatest extends Command { static description = 'Run the migrations up to the latest changes.'; /** - * Success handler for each connection. + * Success handler. */ onSuccess = async (result: OperationResult) => { const log = dbLogger(result.connectionId); @@ -40,7 +37,7 @@ class MigrateLatest extends Command { }; /** - * Failure handler for each connection. + * Failure handler. */ onFailed = async (result: OperationResult) => { printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); diff --git a/src/commands/migrate-list.ts b/src/commands/migrate-list.ts index c48b2ccc..17f0258c 100644 --- a/src/commands/migrate-list.ts +++ b/src/commands/migrate-list.ts @@ -6,36 +6,33 @@ import { printLine, printError } from '../util/io'; import { loadConfig, resolveConnections } from '..'; import OperationResult from '../domain/operation/OperationResult'; -/** - * Migration command handler. - */ class MigrateList extends Command { - static description = 'List migrations.'; + static description = 'List all the migrations.'; /** - * Success handler for a connection. + * Success handler. */ onSuccess = async (result: OperationResult) => { await printLine(bold(` ▸ ${result.connectionId}`)); - const [list1, list2] = result.data; - const ranCount = list1.length; - const remainingCount = list2.length; + const [completedList, remainingList] = result.data; + const ranCount = completedList.length; + const remainingCount = remainingList.length; // Completed migrations. - for (const item of list1) { + for (const item of completedList) { await printLine(cyan(` • ${item}`)); } // Remaining Migrations - for (const item of list2) { + for (const item of remainingList) { await printLine(grey(` - ${item}`)); } if (ranCount === 0 && remainingCount === 0) { await printLine(yellow(' No migrations.')); } else if (remainingCount > 0) { - await printLine(yellow(`\n ${list2.length} migrations yet to be run.`)); + await printLine(yellow(`\n ${remainingList.length} migrations yet to be run.`)); } else if (remainingCount === 0) { await printLine('\n All up to date.'); } @@ -44,7 +41,7 @@ class MigrateList extends Command { }; /** - * Failure handler for a connection. + * Failure handler. */ onFailed = async (result: OperationResult) => { printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); diff --git a/src/commands/migrate-rollback.ts b/src/commands/migrate-rollback.ts index be4b426c..a7628db2 100644 --- a/src/commands/migrate-rollback.ts +++ b/src/commands/migrate-rollback.ts @@ -2,19 +2,16 @@ import { Command } from '@oclif/command'; import { bold, red, cyan } from 'chalk'; import { migrateRollback } from '../api'; -import { printLine, printError, printInfo } from '../util/io'; -import { loadConfig, resolveConnections } from '..'; import { dbLogger } from '../util/logger'; +import { loadConfig, resolveConnections } from '..'; +import { printLine, printError, printInfo } from '../util/io'; import OperationResult from '../domain/operation/OperationResult'; -/** - * Migration command handler. - */ class MigrateRollback extends Command { static description = 'Rollback migrations up to the last run batch.'; /** - * Success handler for each connection. + * Success handler. */ onSuccess = async (result: OperationResult) => { const log = dbLogger(result.connectionId); @@ -31,7 +28,7 @@ class MigrateRollback extends Command { return; } - // Completed migrations. + // List of migrations rolled back. for (const item of list) { await printLine(cyan(` - ${item}`)); } @@ -40,7 +37,7 @@ class MigrateRollback extends Command { }; /** - * Failure handler for each connection. + * Failure handler. */ onFailed = async (result: OperationResult) => { await printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); diff --git a/src/commands/prune.ts b/src/commands/prune.ts index 7b6976c2..764c9bb8 100644 --- a/src/commands/prune.ts +++ b/src/commands/prune.ts @@ -1,23 +1,23 @@ -import { Command } from '@oclif/command'; import { bold, red } from 'chalk'; +import { Command } from '@oclif/command'; -import { printLine, printError } from '../util/io'; - -import OperationResult from '../domain/operation/OperationResult'; import { prune } from '../api'; +import { printLine, printError } from '../util/io'; import { loadConfig, resolveConnections } from '..'; +import OperationResult from '../domain/operation/OperationResult'; class Prune extends Command { static description = 'Drop all the synchronized db objects except the ones created via migrations.'; + /** - * Success handler for each connection. + * Success handler. */ onSuccess = async (result: OperationResult) => { await printLine(bold(` ▸ ${result.connectionId} - Successful`) + ` (${result.timeElapsed}s)`); }; /** - * Failure handler for each connection. + * Failure handler. */ onFailed = async (result: OperationResult) => { await printLine(bold(red(` ▸ ${result.connectionId} - Failed`))); diff --git a/src/commands/synchronize.ts b/src/commands/synchronize.ts index bbed8536..f45b38e6 100644 --- a/src/commands/synchronize.ts +++ b/src/commands/synchronize.ts @@ -1,16 +1,13 @@ import { bold, cyan, red, green } from 'chalk'; import { Command, flags } from '@oclif/command'; +import { synchronize } from '../api'; import { getElapsedTime } from '../util/ts'; import { log, dbLogger } from '../util/logger'; import { loadConfig, resolveConnections } from '../config'; import { printError, printLine, printInfo } from '../util/io'; -import { synchronize } from '../api'; import OperationResult from '../domain/operation/OperationResult'; -/** - * Synchronize command handler. - */ class Synchronize extends Command { static description = 'Synchronize database'; @@ -24,16 +21,22 @@ class Synchronize extends Command { 'skip-migration': flags.boolean({ description: 'Skip running migrations' }) }; + /** + * Started event handler. + */ onStarted = async (result: OperationResult) => { await printLine(bold(` ▸ ${result.connectionId}`)); await printInfo(' [✓] Synchronization - started'); }; + /** + * Prune success handler. + */ onPruneSuccess = (result: OperationResult) => printLine(green(' [✓] Synchronization - pruned') + ` (${result.timeElapsed}s)`); /** - * Success handler for migration run during sync process. + * Migration success handler. */ onMigrationSuccess = async (result: OperationResult) => { const logDb = dbLogger(result.connectionId); @@ -57,7 +60,7 @@ class Synchronize extends Command { }; /** - * Failure handler for migration during sync process. + * Migration failure handler. */ onMigrationFailed = async (result: OperationResult) => { await printLine(red(` [✖] Migrations - failed (${result.timeElapsed}s)\n`)); @@ -66,13 +69,13 @@ class Synchronize extends Command { }; /** - * Success handler for each connection. + * Success handler for the whole process - after all completed. */ onSuccess = (result: OperationResult) => printLine(green(' [✓] Synchronization - completed') + ` (${result.timeElapsed}s)\n`); /** - * Failure handler for each connection. + * Failure handler for the whole process - if the process failed. */ onFailed = async (result: OperationResult) => { await printLine(red(` [✖] Synchronization - failed (${result.timeElapsed}s)\n`)); diff --git a/src/domain/MigrationCommandContext.ts b/src/domain/MigrationContext.ts similarity index 69% rename from src/domain/MigrationCommandContext.ts rename to src/domain/MigrationContext.ts index 38cab9c1..76df2e32 100644 --- a/src/domain/MigrationCommandContext.ts +++ b/src/domain/MigrationContext.ts @@ -3,9 +3,9 @@ import * as Knex from 'knex'; import OperationParams from './operation/OperationParams'; import OperationContext from './operation/OperationContext'; -interface MigrationCommandContext extends OperationContext { +interface MigrationContext extends OperationContext { params: OperationParams; knexMigrationConfig: Knex.MigratorConfig; } -export default MigrationCommandContext; +export default MigrationContext; diff --git a/src/domain/SyncContext.ts b/src/domain/SynchronizeContext.ts similarity index 76% rename from src/domain/SyncContext.ts rename to src/domain/SynchronizeContext.ts index 4016448b..2d45f531 100644 --- a/src/domain/SyncContext.ts +++ b/src/domain/SynchronizeContext.ts @@ -6,9 +6,9 @@ import OperationContext from './operation/OperationContext'; /** * Synchronize context for a database connection. */ -interface SyncContext extends OperationContext { +interface SynchronizeContext extends OperationContext { params: SynchronizeParams; migrateFunc: (trx: Knex.Transaction) => Promise; } -export default SyncContext; +export default SynchronizeContext; diff --git a/src/domain/migration/MigrationContext.ts b/src/domain/migration/MigrationContext.ts deleted file mode 100644 index fa1ed5ff..00000000 --- a/src/domain/migration/MigrationContext.ts +++ /dev/null @@ -1,13 +0,0 @@ -import MigrationRunner from './MigrationRunner'; - -/** - * A contract for migration source context. - */ -interface MigrationContext { - connectionId: string; - keys(): string[]; - get(name: string): MigrationRunner; - bind(connectionId: string): MigrationContext; -} - -export default MigrationContext; diff --git a/src/init.ts b/src/init.ts index d51d7e64..e46be604 100644 --- a/src/init.ts +++ b/src/init.ts @@ -4,10 +4,10 @@ import * as path from 'path'; import { log } from './util/logger'; import { validate, isCLI } from './config'; import Configuration from './domain/Configuration'; -import * as migratorService from './service/migrator'; -import MigrationContext from './domain/migration/MigrationContext'; +import * as migratorService from './migration/service/migrator'; import KnexMigrationSource from './migration/KnexMigrationSource'; -import SqlMigrationContext from './migration/SqlMigrationContext'; +import MigrationSourceContext from './migration/domain/MigrationSourceContext'; +import SqlMigrationSourceContext from './migration/source-types/SqlMigrationSourceContext'; export interface PrepareOptions { loadMigrations?: boolean; @@ -49,12 +49,12 @@ export async function prepare(config: Configuration, options: PrepareOptions): P * * @param {Configuration} config * @param {PrepareOptions} options - * @returns {(Promise)} + * @returns {(Promise)} */ async function resolveMigrationContext( config: Configuration, options: PrepareOptions -): Promise { +): Promise { if (options.loadMigrations !== true) { return null; } @@ -70,7 +70,7 @@ async function resolveMigrationContext( log('Available migration sources:\n%O', src); - return new SqlMigrationContext(src); + return new SqlMigrationSourceContext(src); default: // TODO: We'll need to support different types of migrations eg both sql & js diff --git a/src/migration/KnexMigrationSource.ts b/src/migration/KnexMigrationSource.ts index 93346c94..40e389ff 100644 --- a/src/migration/KnexMigrationSource.ts +++ b/src/migration/KnexMigrationSource.ts @@ -1,11 +1,14 @@ import { dbLogger } from '../util/logger'; -import MigrationContext from '../domain/migration/MigrationContext'; +import MigrationSourceContext from './domain/MigrationSourceContext'; /** - * MigrationSource class for the Knex Migration API. + * Migration source class for the Knex Migration API. + * An instance of this class is passed to the knex's migration config as `migrationSource`. + * + * Reference: http://knexjs.org/#Migrations-API */ class KnexMigrationSource { - private migrationContext: MigrationContext; + private migrationContext: MigrationSourceContext; private log: debug.Debugger; /** @@ -13,7 +16,7 @@ class KnexMigrationSource { * * @param {MigrationEntry[]} migrationLists */ - constructor(migrationContext: MigrationContext) { + constructor(migrationContext: MigrationSourceContext) { this.log = dbLogger(migrationContext.connectionId); this.migrationContext = migrationContext; } diff --git a/src/domain/migration/MigrationRunner.ts b/src/migration/domain/MigrationRunner.ts similarity index 100% rename from src/domain/migration/MigrationRunner.ts rename to src/migration/domain/MigrationRunner.ts diff --git a/src/migration/domain/MigrationSourceContext.ts b/src/migration/domain/MigrationSourceContext.ts new file mode 100644 index 00000000..75e8a425 --- /dev/null +++ b/src/migration/domain/MigrationSourceContext.ts @@ -0,0 +1,14 @@ +import MigrationRunner from './MigrationRunner'; + +/** + * A contract for migration source context. + * All types of migration source types (eg: sql, javascript) implement this contract. + */ +interface MigrationSourceContext { + connectionId: string; + keys(): string[]; + get(name: string): MigrationRunner; + bind(connectionId: string): MigrationSourceContext; +} + +export default MigrationSourceContext; diff --git a/src/domain/migration/SqlMigrationEntry.ts b/src/migration/domain/SqlMigrationEntry.ts similarity index 76% rename from src/domain/migration/SqlMigrationEntry.ts rename to src/migration/domain/SqlMigrationEntry.ts index 0332fae9..4d8f5b0b 100644 --- a/src/domain/migration/SqlMigrationEntry.ts +++ b/src/migration/domain/SqlMigrationEntry.ts @@ -1,4 +1,4 @@ -import SqlCode from '../SqlCode'; +import SqlCode from '../../domain/SqlCode'; interface SqlMigrationEntry { name: string; diff --git a/src/migration/service/knexMigrator.ts b/src/migration/service/knexMigrator.ts new file mode 100644 index 00000000..3fc59c44 --- /dev/null +++ b/src/migration/service/knexMigrator.ts @@ -0,0 +1,58 @@ +import * as Knex from 'knex'; + +import { dbLogger } from '../../util/logger'; +import { executeOperation } from '../../service/execution'; +import MigrationContext from '../../domain/MigrationContext'; +import OperationResult from '../../domain/operation/OperationResult'; + +export enum KnexMigrationAPI { + MIGRATE_LIST = 'migrate.list', + MIGRATE_LATEST = 'migrate.latest', + MIGRATE_ROLLBACK = 'migrate.rollback' +} + +/** + * A map of Knex's migration API functions. + */ +const migrationApiMap = { + // Run up to the latest migrations. + [KnexMigrationAPI.MIGRATE_LATEST]: (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => + trx.migrate.latest(config), + + // Rollback migrations. + [KnexMigrationAPI.MIGRATE_ROLLBACK]: (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => + trx.migrate.rollback(config), + + // List migrations. + [KnexMigrationAPI.MIGRATE_LIST]: (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => + trx.migrate.list(config) +}; + +/** + * Invoke Knex's migration API for given function. + * + * @param {Knex.Transaction} trx + * @param {KnexMigrationAPI} funcName + * @param {MigrationContext} context + * @returns {Promise} + */ +export async function invokeMigrationApi( + trx: Knex.Transaction, + funcName: KnexMigrationAPI, + context: MigrationContext +): Promise { + return executeOperation(context, async () => { + const func = migrationApiMap[funcName]; + + const dbLog = dbLogger(context.connectionId); + + dbLog(`BEGIN: ${funcName}`); + + const data = await func(trx, context.knexMigrationConfig); + + dbLog(`END: ${funcName}`); + dbLog('Result:\n%O', data); + + return data; + }); +} diff --git a/src/service/migrator.ts b/src/migration/service/migrator.ts similarity index 91% rename from src/service/migrator.ts rename to src/migration/service/migrator.ts index 1bed8b3d..4f57e4ae 100644 --- a/src/service/migrator.ts +++ b/src/migration/service/migrator.ts @@ -1,8 +1,8 @@ import * as path from 'path'; -import { glob, exists } from '../util/fs'; -import { resolveFile } from './sqlRunner'; -import SqlMigrationEntry from '../domain/migration/SqlMigrationEntry'; +import { glob, exists } from '../../util/fs'; +import { resolveFile } from '../../service/sqlRunner'; +import SqlMigrationEntry from '../domain/SqlMigrationEntry'; const FILE_PATTERN = /(.+)\.(up|down)\.sql$/; diff --git a/src/migration/SqlMigrationContext.ts b/src/migration/source-types/SqlMigrationSourceContext.ts similarity index 78% rename from src/migration/SqlMigrationContext.ts rename to src/migration/source-types/SqlMigrationSourceContext.ts index 8b895cea..2a1eaa2b 100644 --- a/src/migration/SqlMigrationContext.ts +++ b/src/migration/source-types/SqlMigrationSourceContext.ts @@ -1,14 +1,14 @@ import * as Knex from 'knex'; -import MigrationRunner from '../domain/migration/MigrationRunner'; -import { dbLogger, log as logger } from '../util/logger'; -import MigrationContext from '../domain/migration/MigrationContext'; -import SqlMigrationEntry from '../domain/migration/SqlMigrationEntry'; +import MigrationRunner from '../domain/MigrationRunner'; +import { dbLogger, log as logger } from '../../util/logger'; +import SqlMigrationEntry from '../domain/SqlMigrationEntry'; +import MigrationSourceContext from '../domain/MigrationSourceContext'; /** * SQL source migration context for KnexMigrationSource. */ -class SqlMigrationContext implements MigrationContext { +class SqlMigrationSourceContext implements MigrationSourceContext { private list: SqlMigrationEntry[]; private log: debug.Debugger; public connectionId: string; @@ -30,9 +30,9 @@ class SqlMigrationContext implements MigrationContext { * Bind connectionId to the context. * * @param {string} connectionId - * @returns {MigrationContext} this + * @returns {MigrationSourceContext} this */ - bind(connectionId: string): MigrationContext { + bind(connectionId: string): MigrationSourceContext { this.connectionId = connectionId; this.log = dbLogger(connectionId); @@ -90,4 +90,4 @@ class SqlMigrationContext implements MigrationContext { } } -export default SqlMigrationContext; +export default SqlMigrationSourceContext; diff --git a/src/service/configInjection.ts b/src/service/configInjection.ts index f45f976c..866b81de 100644 --- a/src/service/configInjection.ts +++ b/src/service/configInjection.ts @@ -4,7 +4,7 @@ import * as Knex from 'knex'; import Mapping from '../domain/Mapping'; import { dbLogger } from '../util/logger'; -import SyncContext from '../domain/SyncContext'; +import SynchronizeContext from '../domain/SynchronizeContext'; import { expandEnvVarsInMap } from '../util/env'; import KeyValuePair from '../domain/KeyValuePair'; import { INJECTED_CONFIG_TABLE } from '../constants'; @@ -54,10 +54,10 @@ export function convertToKeyValuePairs(vars: Mapping): KeyValuePair[] { * Setup the table in the database with the injected config. * * @param {Knex.Transaction} trx - * @param {SyncContext} context + * @param {SynchronizeContext} context * @returns {Promise} */ -export async function setup(trx: Knex.Transaction, context: SyncContext): Promise { +export async function setup(trx: Knex.Transaction, context: SynchronizeContext): Promise { const log = dbLogger(context.connectionId); const { injectedConfig } = context.config; @@ -92,10 +92,10 @@ export async function setup(trx: Knex.Transaction, context: SyncContext): Promis * Drop the injected config table. * * @param {Knex.Transaction} trx - * @param {SyncContext} context + * @param {SynchronizeContext} context * @returns {Promise} */ -export async function cleanup(trx: Knex.Transaction, context: SyncContext): Promise { +export async function cleanup(trx: Knex.Transaction, context: SynchronizeContext): Promise { const log = dbLogger(context.connectionId); await trx.schema.dropTableIfExists(INJECTED_CONFIG_TABLE); diff --git a/src/service/execution.ts b/src/service/execution.ts index 8ba5eefb..76274927 100644 --- a/src/service/execution.ts +++ b/src/service/execution.ts @@ -1,9 +1,9 @@ +import { getElapsedTime } from '../util/ts'; import { log, dbLogger } from '../util/logger'; import Configuration from '../domain/Configuration'; import { Promiser, runSequentially } from '../util/promise'; -import OperationContext from '../domain/operation/OperationContext'; import OperationResult from '../domain/operation/OperationResult'; -import { getElapsedTime } from '../util/ts'; +import OperationContext from '../domain/operation/OperationContext'; /** * Execute a list of processes according to the configuration. diff --git a/src/service/knexMigrator.ts b/src/service/knexMigrator.ts deleted file mode 100644 index 22545915..00000000 --- a/src/service/knexMigrator.ts +++ /dev/null @@ -1,43 +0,0 @@ -import * as Knex from 'knex'; - -import { dbLogger } from '../util/logger'; -import MigrationCommandContext from '../domain/MigrationCommandContext'; -import OperationResult from '../domain/operation/OperationResult'; -import { executeOperation } from './execution'; - -/** - * A map of Knex's migration API functions. - */ -export const migrationApiMap = { - 'migrate.latest': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.latest(config), - 'migrate.rollback': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.rollback(config), - 'migrate.list': (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => trx.migrate.list(config) -}; - -/** - * Invoke Knex's migration API for given function. - * - * @param {Knex.Transaction} trx - * @param {MigrationCommandContext} context - * @param {((trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => Promise)} func - * @returns {Promise} - */ -export async function invokeMigrationApi( - trx: Knex.Transaction, - context: MigrationCommandContext, - func: (trx: Knex | Knex.Transaction, config: Knex.MigratorConfig) => Promise -): Promise { - return executeOperation(context, async () => { - const { knexMigrationConfig } = context; - const funcName = func.name || 'func'; - - const dbLog = dbLogger(context.connectionId); - dbLog(`BEGIN: ${funcName}`); - const data = await func(trx, knexMigrationConfig); - - dbLog(`END: ${funcName}`); - dbLog('Result:\n%O', data); - - return data; - }); -} diff --git a/src/service/sync.ts b/src/service/sync.ts index bb173b8a..5f9abefb 100644 --- a/src/service/sync.ts +++ b/src/service/sync.ts @@ -3,7 +3,7 @@ import * as Knex from 'knex'; import * as sqlRunner from './sqlRunner'; import { dbLogger } from '../util/logger'; import { getElapsedTime } from '../util/ts'; -import SyncContext from '../domain/SyncContext'; +import SynchronizeContext from '../domain/SynchronizeContext'; import * as configInjection from './configInjection'; import OperationResult from '../domain/operation/OperationResult'; import OperationContext from '../domain/operation/OperationContext'; @@ -13,10 +13,10 @@ import { executeOperation } from './execution'; * Migrate SQL on a database. * * @param {Knex.Transaction} trx - * @param {SyncContext} context + * @param {SynchronizeContext} context * @returns {Promise} */ -async function setup(trx: Knex.Transaction, context: SyncContext): Promise { +async function setup(trx: Knex.Transaction, context: SynchronizeContext): Promise { const { connectionId } = context; const { basePath, hooks, sql } = context.config; const log = dbLogger(connectionId); @@ -81,13 +81,13 @@ async function teardown(trx: Knex.Transaction, context: OperationContext): Promi } /** - * Synchronize on a single database connection. + * Run synchronize on the given database connection (transaction). * * @param {Knex.Transaction} trx - * @param {SyncContext} context + * @param {SynchronizeContext} context * @returns {Promise} */ -export async function synchronizeDatabase(trx: Knex.Transaction, context: SyncContext): Promise { +export async function runSynchronize(trx: Knex.Transaction, context: SynchronizeContext): Promise { return executeOperation(context, async options => { const { connectionId, migrateFunc } = context; const { timeStart } = options; @@ -127,12 +127,12 @@ export async function synchronizeDatabase(trx: Knex.Transaction, context: SyncCo } /** - * Prune on a single database connection. + * Rune prune operation (drop all synchronized objects) on the given database connection (transaction). * * @param {Knex.Transaction} trx * @param {OperationContext} context * @returns {Promise} */ -export async function pruneDatabase(trx: Knex.Transaction, context: OperationContext): Promise { +export async function runPrune(trx: Knex.Transaction, context: OperationContext): Promise { return executeOperation(context, () => teardown(trx, context)); } diff --git a/test/migration/KnexMigrationSource.test.ts b/test/migration/KnexMigrationSource.test.ts index 836b1b42..02433462 100644 --- a/test/migration/KnexMigrationSource.test.ts +++ b/test/migration/KnexMigrationSource.test.ts @@ -1,12 +1,12 @@ import { expect } from 'chai'; import { describe, it } from 'mocha'; -import MigrationContext from '../../src/domain/migration/MigrationContext'; import KnexMigrationSource from '../../src/migration/KnexMigrationSource'; +import MigrationSourceContext from '../../src/migration/domain/MigrationSourceContext'; -describe('UTIL: KnexMigrationSource', () => { +describe('MIGRATION: KnexMigrationSource', () => { const getInstance = () => { - const migrationContext: MigrationContext = new (class { + const migrationContext: MigrationSourceContext = new (class { connectionId = 'testdb1'; bind(connectionId: string) { diff --git a/test/migration/SqlMigrationContext.test.ts b/test/migration/SqlMigrationContext.test.ts index a5ee09f1..5333be6c 100644 --- a/test/migration/SqlMigrationContext.test.ts +++ b/test/migration/SqlMigrationContext.test.ts @@ -1,11 +1,11 @@ import { expect } from 'chai'; import { describe, it } from 'mocha'; -import SqlMigrationEntry from '../../src/domain/migration/SqlMigrationEntry'; -import SqlMigrationContext from '../../src/migration/SqlMigrationContext'; +import SqlMigrationEntry from '../../src/migration/domain/SqlMigrationEntry'; +import SqlMigrationSourceContext from '../../src/migration/source-types/SqlMigrationSourceContext'; -describe('UTIL: SqlMigrationContext', () => { - const getInstance = (list: SqlMigrationEntry[]) => new SqlMigrationContext(list); +describe('MIGRATION: SqlMigrationSourceContext', () => { + const getInstance = (list: SqlMigrationEntry[]) => new SqlMigrationSourceContext(list); describe('keys', () => { it('should return an empty list if migrations are empty.', () => { diff --git a/test/service/migrator.test.ts b/test/migration/migrator.test.ts similarity index 97% rename from test/service/migrator.test.ts rename to test/migration/migrator.test.ts index ce18a302..546ac17e 100644 --- a/test/service/migrator.test.ts +++ b/test/migration/migrator.test.ts @@ -3,9 +3,9 @@ import { describe, it } from 'mocha'; import { expect } from 'chai'; import { write, mkdtemp } from '../../src/util/fs'; -import * as migratorService from '../../src/service/migrator'; +import * as migratorService from '../../src/migration/service/migrator'; -describe('SERVICE: migrator', () => { +describe('MIGRATION: migrator', () => { describe('getSqlMigrationNames', async () => { it('should return the list of valid migrations under the directory.', async () => { const migrationPath = await mkdtemp();