Skip to content

Commit

Permalink
Support for Redis Sentinel (#501)
Browse files Browse the repository at this point in the history
* Replacing redis client to ioredis that support Sentinel; replace in-memory to ioredis-mock
* A lot of test fixes
  • Loading branch information
XVincentX authored Nov 15, 2017
1 parent ff7c212 commit 80dd9b4
Show file tree
Hide file tree
Showing 48 changed files with 1,198 additions and 994 deletions.
13 changes: 13 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,18 @@ jobs:
- run: npm install
- run: npm test

"node-8-real-redis":
docker:
- image: circleci/node:8
- image: redis:alpine
working_directory: ~/repo
environment:
- EG_DB_EMULATE: 0
steps:
- checkout
- run: npm install
- run: npm test

deploy:
docker:
- image: circleci/node:8
Expand All @@ -46,6 +58,7 @@ workflows:
jobs:
- node-6
- node-8
- node-8-real-redis
- deploy:
filters:
tags:
Expand Down
88 changes: 42 additions & 46 deletions lib/db.js
Original file line number Diff line number Diff line change
@@ -1,57 +1,53 @@
const logger = require('./logger').db;
const redisCommands = require('redis-commands');
require('util.promisify/shim')(); // NOTE: shim for native node 8.0 uril.promisify
const util = require('util');
const config = require('./config');
const fs = require('fs');
let db;
const redisOptions = config.systemConfig.db && config.systemConfig.db.redis;

module.exports = function () {
if (db) {
return db;
}
const config = require('./config');
const redisOptions = config.systemConfig.db && config.systemConfig.db.redis;

// special mode, will emulate all redis commands.
// designed for demo and test scenarious to avoid having real Redis instance
const emulate = process.argv[2] === 'emulate' || redisOptions.emulate;

const redis = require(emulate ? 'fakeredis' : 'redis');
promisify(redis.RedisClient.prototype, redisCommands.list);
promisify(redis.Multi.prototype, ['exec', 'execAtomic']);

function promisify (obj, methods) {
methods.forEach((method) => {
if (obj[method]) {
obj[method + 'Async'] = util.promisify(obj[method]);
}
});
}
// special mode, will emulate all redis commands.
// designed for demo and test scenarious to avoid having real Redis instance
let emulate = process.argv[2] === 'emulate' || redisOptions.emulate;

// TLS for redis, allowing for TLS options to be specified as file paths.
if (redisOptions.tls) {
if (redisOptions.tls.keyFile) {
redisOptions.tls.key = fs.readFileSync(redisOptions.tls.keyFile);
}
if (process.env.EG_DB_EMULATE) {
emulate = !!parseInt(process.env.EG_DB_EMULATE);
}

if (redisOptions.tls.certFile) {
redisOptions.tls.cert = fs.readFileSync(redisOptions.tls.certFile);
}
if (redisOptions.tls) {
if (redisOptions.tls.keyFile) {
redisOptions.tls.key = fs.readFileSync(redisOptions.tls.keyFile);
};

if (redisOptions.tls.caFile) {
redisOptions.tls.ca = fs.readFileSync(redisOptions.tls.caFile);
}
if (redisOptions.tls.certFile) {
redisOptions.tls.cert = fs.readFileSync(redisOptions.tls.certFile);
}

db = redis.createClient(redisOptions);
if (redisOptions.tls.caFile) {
redisOptions.tls.ca = fs.readFileSync(redisOptions.tls.caFile);
}
}

const Redis = require(emulate ? 'ioredis-mock' : 'ioredis');
const db = new Redis(redisOptions);

// TODO: ALERT: this is temporary fix before transformers will land into ioredis-mock
db.prepareHMSET = function (hashKey, obj) {
const result = [];
let pos = 0;
for (const key in obj) {
if (obj.hasOwnProperty(key)) {
result[pos] = key;
result[pos + 1] = obj[key];
}
pos += 2;
}
return [hashKey].concat(result);
};

db.on('ready', function () {
logger.debug('Redis is ready');
});
db.on('ready', function () {
logger.debug('Redis is ready');
});

db.on('error', function (err) {
logger.error('Error in Redis: ', err);
});
db.on('error', function (err) {
logger.error('Error in Redis: ', err);
});

return db;
};
module.exports = db;
14 changes: 6 additions & 8 deletions lib/services/authorization-codes/authorization-code.dao.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
'use strict';

const db = require('../../db')();
const db = require('../../db');
const config = require('../../config');

const dao = {};
Expand All @@ -10,13 +8,13 @@ const authCodeNamespace = 'auth-code';
dao.save = function (code) {
// key for the code hash table
const redisCodeKey = config.systemConfig.db.redis.namespace.concat('-', authCodeNamespace).concat(':', code.id);
return db.hmsetAsync(redisCodeKey, code);
return db.hmset.apply(db, db.prepareHMSET(redisCodeKey, code));
};

dao.find = function (criteria) {
return db.hgetallAsync(config.systemConfig.db.redis.namespace.concat('-', authCodeNamespace).concat(':', criteria.id))
return db.hgetall(config.systemConfig.db.redis.namespace.concat('-', authCodeNamespace).concat(':', criteria.id))
.then((code) => {
if (!code) {
if (!code || !code.expiresAt) {
return null;
}
code.expiresAt = parseInt(code.expiresAt);
Expand All @@ -31,11 +29,11 @@ dao.find = function (criteria) {
};

dao.get = function (id) {
return db.hgetallAsync(config.systemConfig.db.redis.namespace.concat('-', authCodeNamespace).concat(':', id));
return db.hgetall(config.systemConfig.db.redis.namespace.concat('-', authCodeNamespace).concat(':', id));
};

dao.remove = function (id) {
return db.delAsync(config.systemConfig.db.redis.namespace.concat('-', authCodeNamespace).concat(':', id));
return db.del(config.systemConfig.db.redis.namespace.concat('-', authCodeNamespace).concat(':', id));
};

module.exports = dao;
41 changes: 22 additions & 19 deletions lib/services/consumers/application.dao.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
'use strict';

const db = require('../../db')();
const db = require('../../db');
const config = require('../../config');

const dao = {};
Expand All @@ -19,13 +17,14 @@ const userAppsHashKey = (value) => `${config.systemConfig.db.redis.namespace}-${
const appNameSetKey = (value) => `${config.systemConfig.db.redis.namespace}-${appnameNamespace}:${value}`;

dao.insert = function (app) {
const addApp = () => db
.multi()
.hmset(appHashKey(app.id), app)
const addApp = () => {
const pipeline = db.multi();
return pipeline.hmset.apply(pipeline, db.prepareHMSET(appHashKey(app.id), app))
.sadd(userAppsHashKey(app.userId), app.id)
.sadd(appNameSetKey(app.name), app.id)
.execAsync()
.exec()
.then(res => res.every(val => val));
};

return dao.find(app.name).then(appId => {
if (appId) {
Expand All @@ -47,7 +46,7 @@ dao.update = function (id, props) {
const hashKey = appHashKey(id);

return db
.hmsetAsync(hashKey, props)
.hmset.apply(db, db.prepareHMSET(hashKey, props))
.then(function (res) {
return !!res;
});
Expand All @@ -56,11 +55,11 @@ dao.update = function (id, props) {
dao.findAll = function (query) {
const key = appHashKey('');
const startFrom = query.start || 0;
return db.scanAsync(startFrom, 'MATCH', key + '*', 'COUNT', '100').then(resp => {
return db.scan(startFrom, 'MATCH', key + '*', 'COUNT', '100').then(resp => {
const nextKey = resp[0];
const appKeys = resp[1];
if (!appKeys || appKeys.length === 0) return Promise.resolve({ apps: [], nextKey: 0 });
const promises = appKeys.map(key => db.hgetallAsync(key));
const promises = appKeys.map(key => db.hgetall(key));
return Promise.all(promises).then(apps => {
return {
apps,
Expand All @@ -71,7 +70,7 @@ dao.findAll = function (query) {
};

dao.find = function (appName) {
return db.smembersAsync(appNameSetKey(appName))
return db.smembers(appNameSetKey(appName))
.then(function (Ids) {
if (Ids && Ids.length !== 0) {
if (Ids.length === 1) {
Expand All @@ -86,7 +85,7 @@ dao.find = function (appName) {
};

dao.get = function (id) {
return db.hgetallAsync(appHashKey(id))
return db.hgetall(appHashKey(id))
.then(function (app) {
if (!app || !Object.keys(app).length) {
return false;
Expand All @@ -103,15 +102,15 @@ dao.getAll = function (userId) {
};

dao.getAllAppIdsByUser = function (userId) {
return db.smembersAsync(userAppsHashKey(userId));
return db.smembers(userAppsHashKey(userId));
};

dao.activate = function (id) {
return db.hmsetAsync(appHashKey(id), ['isActive', 'true', 'updatedAt', String(new Date())]);
return db.hmset(appHashKey(id), 'isActive', 'true', 'updatedAt', String(new Date()));
};

dao.deactivate = function (id) {
return db.hmsetAsync(appHashKey(id), ['isActive', 'false', 'updatedAt', String(new Date())]);
return db.hmset(appHashKey(id), 'isActive', 'false', 'updatedAt', String(new Date()));
};

dao.deactivateAll = function (userId) {
Expand All @@ -128,15 +127,19 @@ dao.remove = function ({ name, id, userId }) {
.del(appHashKey(id))
.srem(userAppsHashKey(userId), id)
.srem(appNameSetKey(name), id)
.execAsync()
.exec()
.then(responses => responses.every(res => res));
};

dao.removeAll = function (userId) {
return this.getAllAppIdsByUser(userId)
.then(appIds => Promise.all(appIds.map(appId => this.get(appId))))
.then(apps => Promise.all(apps.map(app => this.remove(app, userId))))
.then(responses => responses.every(res => res));
.then(appIds => {
const removeAppPromises = appIds.map(appId => {
return this.get(appId).then((app) => this.remove(app));
});
return Promise.all(removeAppPromises)
.then(responses => responses.every(res => res));
});
};

module.exports = dao;
15 changes: 12 additions & 3 deletions lib/services/consumers/application.service.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,12 @@ const uuidv4 = require('uuid/v4');
const s = {};

s.insert = function (_app, userId) {
const app = validateAndCreateApp(_app, userId);

let app;
try {
app = validateAndCreateApp(_app, userId);
} catch (err) {
return Promise.reject(err);
}
return applicationDao.insert(app)
.then(function (success) {
if (!success) {
Expand Down Expand Up @@ -67,7 +71,12 @@ s.getAll = function (userId) {

s.remove = function (id) {
return this.get(id) // make sure app exists
.then(applicationDao.remove)
.then(app => {
if (!app) {
return Promise.reject(new Error('app not found, failed to remove'));
}
return applicationDao.remove(app);
})
.then(function (removed) {
return removed ? true : Promise.reject(new Error('failed to remove app')); // TODO: replace with server error
});
Expand Down
28 changes: 12 additions & 16 deletions lib/services/consumers/user.dao.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
'use strict';

const db = require('../../db')();
const db = require('../../db');
const config = require('../../config');

const dao = {};
Expand All @@ -13,17 +11,15 @@ dao.insert = function (user) {

// name for the user's username set
const redisUsernameSetKey = config.systemConfig.db.redis.namespace.concat('-', usernameNamespace).concat(':', user.username);

return db
.multi()
.hmset(redisUserKey, user)
const pipeline = db.multi();
return pipeline.hmset.apply(pipeline, db.prepareHMSET(redisUserKey, user))
.sadd(redisUsernameSetKey, user.id)
.execAsync()
.exec()
.then(res => res.every(val => val));
};

dao.getUserById = function (userId) {
return db.hgetallAsync(config.systemConfig.db.redis.namespace.concat('-', userNamespace).concat(':', userId))
return db.hgetall(config.systemConfig.db.redis.namespace.concat('-', userNamespace).concat(':', userId))
.then(function (user) {
if (!user || !Object.keys(user).length) {
return false;
Expand All @@ -35,11 +31,11 @@ dao.getUserById = function (userId) {
dao.findAll = function (query) {
const startFrom = query.start || 0;
const key = config.systemConfig.db.redis.namespace.concat('-', userNamespace).concat(':');
return db.scanAsync(startFrom, 'MATCH', key + '*', 'COUNT', '100').then(resp => {
return db.scan(startFrom, 'MATCH', key + '*', 'COUNT', '100').then(resp => {
const nextKey = resp[0];
const userKeys = resp[1];
if (!userKeys || userKeys.length === 0) return Promise.resolve({ users: [], nextKey: 0 });
const promises = userKeys.map(key => db.hgetallAsync(key));
const promises = userKeys.map(key => db.hgetall(key));
return Promise.all(promises).then(users => {
return {
users,
Expand All @@ -50,7 +46,7 @@ dao.findAll = function (query) {
};

dao.find = function (username) {
return db.smembersAsync(config.systemConfig.db.redis.namespace.concat('-', usernameNamespace).concat(':', username))
return db.smembers(config.systemConfig.db.redis.namespace.concat('-', usernameNamespace).concat(':', username))
.then(function (Ids) {
if (Ids && Ids.length !== 0) {
return Ids[0];
Expand All @@ -62,16 +58,16 @@ dao.update = function (userId, props) {
// key for the user in redis
const redisUserKey = config.systemConfig.db.redis.namespace.concat('-', userNamespace).concat(':', userId);
return db
.hmsetAsync(redisUserKey, props)
.hmset.apply(db, db.prepareHMSET(redisUserKey, props))
.then(res => !!res);
};

dao.activate = function (id) {
return db.hmsetAsync(config.systemConfig.db.redis.namespace.concat('-', userNamespace).concat(':', id), ['isActive', 'true', 'updatedAt', String(new Date())]);
return db.hmset(config.systemConfig.db.redis.namespace.concat('-', userNamespace).concat(':', id), 'isActive', 'true', 'updatedAt', String(new Date()));
};

dao.deactivate = function (id) {
return db.hmsetAsync(config.systemConfig.db.redis.namespace.concat('-', userNamespace).concat(':', id), ['isActive', 'false', 'updatedAt', String(new Date())]);
return db.hmset(config.systemConfig.db.redis.namespace.concat('-', userNamespace).concat(':', id), 'isActive', 'false', 'updatedAt', String(new Date()));
};

dao.remove = function (userId) {
Expand All @@ -84,7 +80,7 @@ dao.remove = function (userId) {
.multi()
.del(config.systemConfig.db.redis.namespace.concat('-', userNamespace).concat(':', userId))
.srem(config.systemConfig.db.redis.namespace.concat('-', usernameNamespace).concat(':', user.username), userId)
.execAsync()
.exec()
.then(replies => replies.every(res => res));
});
};
Expand Down
Loading

0 comments on commit 80dd9b4

Please sign in to comment.