From 69b366a4ae39202d0ea3fd3fc9e73fb77785431f Mon Sep 17 00:00:00 2001 From: Harminder virk Date: Sat, 3 Aug 2019 12:44:49 +0530 Subject: [PATCH] feat: implement connection class to manage and monitor pool resources --- .env | 6 ++ docker-compose.yml | 24 +++---- index.ts | 144 ++++++++++++++++++++++++++++++++++++++++ src/Connection/index.ts | 101 ++++++++++++++++++++++++++++ test-helpers/index.ts | 66 ++++++++++++++++++ test/connection.spec.ts | 94 ++++++++++++++++++++++++++ 6 files changed, 421 insertions(+), 14 deletions(-) create mode 100644 .env create mode 100644 index.ts create mode 100644 src/Connection/index.ts create mode 100644 test-helpers/index.ts create mode 100644 test/connection.spec.ts diff --git a/.env b/.env new file mode 100644 index 00000000..22665531 --- /dev/null +++ b/.env @@ -0,0 +1,6 @@ +DB=mysql +MYSQL_HOST=0.0.0.0 +MYSQL_PORT=3306 +DB_NAME=lucid +MYSQL_USER=virk +MYSQL_PASSWORD=password diff --git a/docker-compose.yml b/docker-compose.yml index 4af99a19..df823a6e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,18 +1,14 @@ -version: '2' +version: '3.3' services: - mysql: + db: image: mysql:5.7 - expose: - - '3306' + restart: always + environment: + MYSQL_DATABASE: 'lucid' + MYSQL_USER: 'virk' + MYSQL_PASSWORD: 'password' + MYSQL_ROOT_PASSWORD: 'password' ports: - '3306:3306' - restart: 'always' - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: '' - MYSQL_ROOT_PASSWORD: 'root' - MYSQL_USER: 'travis' - MYSQL_PASSWORD: '' - MYSQL_ROOT_HOST: '%' - MYSQL_DATABASE: 'testing_lucid' - volumes: - - ./mysqldata:/var/lib/mysql + expose: + - '3306' diff --git a/index.ts b/index.ts new file mode 100644 index 00000000..bf99a8e9 --- /dev/null +++ b/index.ts @@ -0,0 +1,144 @@ +// import * as Knex from 'knex' +// // import { join } from 'path' +// // import { DatabaseQueryBuilderContract } from '@ioc:Adonis/Addons/Database' + +// const knex = Knex({ +// client: 'pg', +// connection: { +// host: '0.0.0.0', +// user: 'virk', +// password: '', +// database: 'directory-service', +// }, +// pool: { +// min: 0, +// max: 5, +// idleTimeoutMillis: 30000, +// }, +// useNullAsDefault: true, +// }) + +// // let i = 0; +// knex['_context'].client.pool.on('destroySuccess', _eventId => { +// // i++ +// console.log( +// knex['_context'].client.pool.numUsed(), +// knex['_context'].client.pool.numFree(), +// knex['_context'].client.pool.numPendingAcquires(), +// ) + +// // if (i === 3) { +// // knex['_context'].client.pool.destroy() +// // } +// }); + +// knex['_context'].client.pool.on('poolDestroySuccess', _resource => { +// console.log('poolDestroySuccess>>>') +// }); + +// // setInterval(() => { +// // console.log('ping') +// // }, 1000) + +// // type User = { +// // id: number, +// // } + +// // console.log(knex.raw(['10']).toQuery()) + +// // knex.schema.createTable('users', (table) => { +// // table.increments('id') +// // table.string('username') +// // table.integer('age') +// // table.timestamps() +// // }).then(() => console.log('created')) + +// // knex.table('users').insert([ +// // { username: 'virk', age: 29 }, { username: 'nikk', age: 28 }, { username: 'prasan', age: 29 }, +// // ]).then(console.log) + +// Promise.all([ +// knex +// .select('*') +// .from('users') +// .debug(true) +// .then((result) => { +// console.log(result) +// }), +// knex +// .select('*') +// .from('users') +// .debug(true) +// .then((result) => { +// console.log(result) +// }), +// knex +// .select('*') +// .from('users') +// .debug(true) +// .then((result) => { +// console.log(result) +// }), +// knex +// .select('*') +// .from('users') +// .debug(true) +// .then((result) => { +// console.log(result) +// }), +// knex +// .select('*') +// .from('users') +// .debug(true) +// .then((result) => { +// console.log(result) +// }), +// ]).then(() => { +// }) + +// // knex.transaction().then((trx) => { +// // }) + +// // console.log(query.toSQL()) + +// // type FilteredKeys = { [P in keyof T]: T[P] extends Function ? never : P }[keyof T] + +// // type GetRefs = T['refs'] extends object ? { +// // [P in keyof T['refs']]: InstanceType[P] +// // } : { +// // [P in FilteredKeys>]: InstanceType[P] +// // } + +// // class BaseModel { +// // public static refs: unknown + +// // public static query ( +// // this: T, +// // ): DatabaseQueryBuilderContract, InstanceType> { +// // return {} as DatabaseQueryBuilderContract, InstanceType> +// // } +// // } + +// // class User extends BaseModel { +// // public username: string +// // public age: number + +// // public castToInt (): number { +// // return 22 +// // } +// // } + +// // class Post extends BaseModel { +// // } + +// // const foo: DatabaseQueryBuilderContract<{ username: string, age: number }> +// // const b = foo +// // .select('*') +// // .select({ +// // 'a': 'username', +// // 'age': 'age', +// // }).first() + +// // async function foo () { +// // const a = User.query().select(['username', 'age']).first() +// // } diff --git a/src/Connection/index.ts b/src/Connection/index.ts new file mode 100644 index 00000000..26639728 --- /dev/null +++ b/src/Connection/index.ts @@ -0,0 +1,101 @@ +/* +* @adonisjs/lucid +* +* (c) Harminder Virk +* +* For the full copyright and license information, please view the LICENSE +* file that was distributed with this source code. +*/ + +/// + +import { Pool } from 'tarn' +import * as knex from 'knex' +import { EventEmitter } from 'events' +import { ConnectionConfigContract, ConnectionContract } from '@ioc:Adonis/Addons/Database' + +/** + * Connection class manages a given database connection. Internally it uses + * knex to build the database connection with appropriate database + * driver. + */ +export class Connection extends EventEmitter implements ConnectionContract { + /** + * Reference to knex. The instance is created once the `open` + * method is invoked + */ + public client?: knex + + /** + * List of events emitted by this class + */ + public readonly EVENTS: ['open', 'close', 'close:error'] + + constructor (public name: string, public config: ConnectionConfigContract) { + super() + } + + /** + * Does cleanup by removing knex reference and removing + * all listeners + */ + private _monitorPoolResources () { + this.pool!.on('destroySuccess', () => { + /** + * Force close when `numUsed` and `numFree` both are zero. This happens + * when `min` resources inside the pool are set to `0`. + */ + if (this.pool!.numFree() === 0 && this.pool!.numUsed() === 0) { + this.close() + } + }) + + /** + * Pool has destroyed and hence we must cleanup resources + * as well. + */ + this.pool!.on('poolDestroySuccess', () => { + this.client = undefined + this.emit('close') + this.removeAllListeners() + }) + } + + /** + * Returns the pool instance for the given connection + */ + public get pool (): null | Pool { + return this.client ? this.client['_context'].client.pool : null + } + + /** + * Opens the connection by creating knex instance + */ + public open () { + try { + this.client = knex(this.config) + this._monitorPoolResources() + this.emit('open') + } catch (error) { + this.emit('error', error) + throw error + } + } + + /** + * Closes DB connection by destroying knex instance. The `connection` + * object must be free for garbage collection. + * + * In case of error this method will emit `close:error` event followed + * by the `close` event. + */ + public async close (): Promise { + if (this.client) { + try { + await this.client!.destroy() + } catch (error) { + this.emit('close:error', error) + } + } + } +} diff --git a/test-helpers/index.ts b/test-helpers/index.ts new file mode 100644 index 00000000..5c735606 --- /dev/null +++ b/test-helpers/index.ts @@ -0,0 +1,66 @@ +/* +* @adonisjs/lucid +* +* (c) Harminder Virk +* +* For the full copyright and license information, please view the LICENSE +* file that was distributed with this source code. +*/ + +/// + +import { join } from 'path' +import * as dotenv from 'dotenv' +import { Filesystem } from '@poppinss/dev-utils' +import { ConnectionConfigContract } from '@ioc:Adonis/Addons/Database' + +export const fs = new Filesystem(join(__dirname, 'tmp')) +dotenv.config() + +/** + * Returns config based upon DB set in environment variables + */ +export function getConfig (): ConnectionConfigContract { + switch (process.env.DB) { + case 'sqlite': + return { + client: 'sqlite', + connection: { + filename: join(fs.basePath, 'db.sqlite'), + }, + useNullAsDefault: true, + } + case 'mysql': + return { + client: 'mysql', + connection: { + host: process.env.MYSQL_HOST as string, + port: Number(process.env.MYSQL_PORT), + database: process.env.DB_NAME as string, + user: process.env.MYSQL_USER as string, + password: process.env.MYSQL_PASSWORD as string, + }, + useNullAsDefault: true, + } + default: + throw new Error(`Missing test config for ${process.env.DB} connection`) + } +} + +/** + * Does base setup by creating databases + */ +export async function setup () { + if (process.env.DB === 'sqlite') { + await fs.ensureRoot() + } +} + +/** + * Does cleanup removes database + */ +export async function cleanup () { + if (process.env.DB === 'sqlite') { + await fs.cleanup() + } +} diff --git a/test/connection.spec.ts b/test/connection.spec.ts new file mode 100644 index 00000000..76d91079 --- /dev/null +++ b/test/connection.spec.ts @@ -0,0 +1,94 @@ +/* +* @adonisjs/lucid +* +* (c) Harminder Virk +* +* For the full copyright and license information, please view the LICENSE +* file that was distributed with this source code. +*/ + +/// + +import * as test from 'japa' +import { MysqlConfigContract } from '@ioc:Adonis/Addons/Database' + +import { getConfig } from '../test-helpers' +import { Connection } from '../src/Connection' + +test.group('Connection', () => { + test('do not instantiate knex unless open is called', (assert) => { + const connection = new Connection('primary', getConfig()) + assert.isUndefined(connection.client) + }) + + test('instantiate knex when open is invoked', async (assert, done) => { + const connection = new Connection('primary', getConfig()) + connection.on('open', () => { + assert.isDefined(connection.client) + assert.equal(connection.pool!.numUsed(), 0) + done() + }) + + connection.open() + }) + + test('on close destroy knex', async (assert) => { + const connection = new Connection('primary', getConfig()) + connection.open() + await connection.close() + assert.isUndefined(connection.client) + }) + + test('destroy connection when pool min resources are zero and connection is idle', async (assert, done) => { + const connection = new Connection('primary', Object.assign(getConfig(), { + pool: { + min: 0, + idleTimeoutMillis: 10, + }, + })) + + connection.open() + await connection.client!.raw('select 1+1 as result') + + connection.on('close', () => { + assert.isUndefined(connection.client) + done() + }) + }) + + test('on close emit close event', async (assert, done) => { + const connection = new Connection('primary', getConfig()) + connection.open() + + connection.on('close', () => { + assert.isUndefined(connection.client) + done() + }) + + await connection.close() + }) + + test('raise error when unable to make connection', (assert) => { + const connection = new Connection('primary', Object.assign({}, getConfig(), { client: null })) + + const fn = () => connection.open() + assert.throw(fn, /knex: Required configuration option/) + }) +}) + +if (process.env.DB === 'mysql') { + test.group('Connection | mysql', () => { + test('pass user config to mysql driver', async (assert) => { + const config = getConfig() as MysqlConfigContract + config.connection.charset = 'utf-8' + config.connection.typeCast = false + + const connection = new Connection('primary', config) + connection.open() + + assert.equal(connection.client!['_context'].client.constructor.name, 'Client_MySQL') + assert.equal(connection.client!['_context'].client.config.connection.charset, 'utf-8') + assert.equal(connection.client!['_context'].client.config.connection.typeCast, false) + }) + }) +}