Merge pull request ExpressGateway#890 from ravikp7/lazy-load-policies

feat: lazy load and hot reload for policies

lib/config/config.js

@@ -1,4 +1,5 @@
 const fs = require('fs');
+const chalk = require('chalk');
 const util = require('util');
 const readFile = util.promisify(fs.readFile);
 const writeFile = util.promisify(fs.writeFile);
@@ -67,7 +68,7 @@ class Config {
       const name = path.basename(module, '.json');
       this.models[name] = require(module);
       schemas.register('model', name, this.models[name]);
-      log.verbose(`Registered schema for ${name} model.`);
+      log.verbose(`Registered schema for ${chalk.green(name)} model.`);
     });
   }
 

lib/gateway/index.js

@@ -1,21 +1,17 @@
 const express = require('express');
+const chalk = require('chalk');
 const log = require('../logger').gateway;
 const servers = require('./server');
 const pipelines = require('./pipelines');
 const eventBus = require('../eventBus');
 const policies = require('../policies');
 const conditions = require('../conditions');
 const passport = require('passport');
+const pluginsLoader = require('../plugins');
 
 module.exports = function ({ plugins, config } = {}) {
   const appPromises = [];
   const apps = {};
-  if (plugins && plugins.policies && plugins.policies.length) {
-    plugins.policies.forEach(p => {
-      log.debug('registering policy', p.name);
-      policies.register(p);
-    });
-  }
   config = config || require('../config');
   const { httpServer, httpsServer } = bootstrap({ plugins, config });
 
@@ -49,11 +45,19 @@ module.exports = function ({ plugins, config } = {}) {
     });
 };
 
-function bootstrap({ plugins, config } = {}) {
-  const app = express();
-  app.set('x-powered-by', false);
+const bootstrapPolicies = ({ app, plugins, config } = {}) => {
+  if (plugins && plugins.policies && plugins.policies.length) {
+    plugins.policies.forEach(policy => {
+      if (!policies[policy.name]) {
+        log.verbose(`registering policy ${chalk.green(policy.name)} from ${plugins.name} plugin`);
+        policies.register(policy);
+      } else log.verbose(`policy ${chalk.magenta(policy.name)} from ${plugins.name} is already loaded`);
+    });
+  }
+
+  // Load policies present in config
+  policies.load(config.gatewayConfig.policies);
 
-  let rootRouter;
   // Load all routes from policies
   // TODO: after all complext policies will go to plugin this code can be removed
   // NOTE: plugins have mechanism to provide custom routes
@@ -72,27 +76,33 @@ function bootstrap({ plugins, config } = {}) {
   const conditionEngine = conditions.init();
   if (plugins && plugins.conditions && plugins.conditions.length) {
     plugins.conditions.forEach(cond => {
-      log.debug('registering condition', cond.name);
+      log.debug(`registering condition ${cond.name}`);
       conditionEngine.register(cond);
     });
   }
+};
+
+function bootstrap({ plugins, config } = {}) {
+  let rootRouter;
+  const app = express();
+  app.set('x-powered-by', false);
   app.use(passport.initialize());
+  bootstrapPolicies({ app, plugins, config });
   rootRouter = pipelines.bootstrap({ app: express.Router(), config });
-  app.use((req, res, next) => {
-    // rootRouter will process all requests;
-    // after hot swap old instance will continue to serve previous requests
-    // new instance will be serving new requests
-    // once all old requests are served old instance is target for GC
-    rootRouter(req, res, next);
-  });
+  app.use((req, res, next) => rootRouter(req, res, next));
 
   eventBus.on('hot-reload', (hotReloadContext) => {
+    const oldConfig = config;
+    const oldPlugins = plugins;
     const oldRootRouter = rootRouter;
     try {
-      rootRouter = pipelines.bootstrap({ app: express.Router(), config: hotReloadContext.config });
-      log.info('hot-reload router completed');
+      const newConfig = hotReloadContext.config;
+      bootstrapPolicies({ app, plugins: pluginsLoader.load(newConfig), config: newConfig });
+      rootRouter = pipelines.bootstrap({ app: express.Router(), config: newConfig });
+      log.info('hot-reload config completed');
     } catch (err) {
-      log.error('Could not hot-reload gateway.config.yml. Configuration is invalid.', err);
+      log.error(`Could not hot-reload gateway.config.yml. Configuration is invalid. ${err}`);
+      bootstrapPolicies({ app, plugins: oldPlugins, config: oldConfig });
       rootRouter = oldRootRouter;
     }
   });

lib/policies/basic-auth/basic-auth.js

@@ -1,44 +1,5 @@
-'use strict';
-
+require('./registerStrategy')();
 const passport = require('passport');
-const BasicStrategy = require('passport-http').BasicStrategy;
-const services = require('../../services/index');
-const authService = services.auth;
-
-passport.use(new BasicStrategy({ passReqToCallback: true }, authenticateBasic));
-
-function authenticateBasic (req, clientId, clientSecret, done) {
-  let credentialType, endpointScopes, requestedScopes;
-
-  if (req.egContext && req.egContext.apiEndpoint && req.egContext.apiEndpoint.scopes) {
-    endpointScopes = req.egContext.apiEndpoint.scopes && req.egContext.apiEndpoint.scopes.map(s => s.scope || s);
-    credentialType = 'basic-auth';
-  } else {
-    credentialType = 'oauth2';
-    if (req.query && req.query.scope) {
-      requestedScopes = req.query.scope.split(' ');
-    } else if (req.body && req.body.scope) {
-      requestedScopes = req.body.scope.split(' ');
-    }
-  }
-  return authService.authenticateCredential(clientId, clientSecret, credentialType)
-    .then(consumer => {
-      if (!consumer) {
-        return done(null, false);
-      }
-      return authService.authorizeCredential(consumer.id, credentialType, endpointScopes || requestedScopes)
-        .then(authorized => {
-          if (!authorized) {
-            return done(null, false);
-          }
-
-          consumer.authorizedScopes = endpointScopes;
-
-          return done(null, consumer);
-        });
-    })
-    .catch(err => done(err));
-}
 
 module.exports = function (actionParams) {
   return function (req, res, next) {

lib/policies/basic-auth/registerStrategy.js

@@ -0,0 +1,41 @@
+const passport = require('passport');
+const BasicStrategy = require('passport-http').BasicStrategy;
+const services = require('../../services/index');
+const authService = services.auth;
+
+module.exports = function registerBasicStrategy() {
+  passport.use(new BasicStrategy({ passReqToCallback: true }, authenticateBasic));
+
+  function authenticateBasic(req, clientId, clientSecret, done) {
+    let credentialType, endpointScopes, requestedScopes;
+
+    if (req.egContext && req.egContext.apiEndpoint && req.egContext.apiEndpoint.scopes) {
+      endpointScopes = req.egContext.apiEndpoint.scopes && req.egContext.apiEndpoint.scopes.map(s => s.scope || s);
+      credentialType = 'basic-auth';
+    } else {
+      credentialType = 'oauth2';
+      if (req.query && req.query.scope) {
+        requestedScopes = req.query.scope.split(' ');
+      } else if (req.body && req.body.scope) {
+        requestedScopes = req.body.scope.split(' ');
+      }
+    }
+    return authService.authenticateCredential(clientId, clientSecret, credentialType)
+      .then(consumer => {
+        if (!consumer) {
+          return done(null, false);
+        }
+        return authService.authorizeCredential(consumer.id, credentialType, endpointScopes || requestedScopes)
+          .then(authorized => {
+            if (!authorized) {
+              return done(null, false);
+            }
+
+            consumer.authorizedScopes = endpointScopes;
+
+            return done(null, consumer);
+          });
+      })
+      .catch(err => done(err));
+  }
+};

lib/policies/index.js

@@ -53,10 +53,17 @@ const policyNames = fs
   .readdirSync(path.resolve(__dirname))
   .filter(dir => fs.lstatSync(path.resolve(__dirname, dir)).isDirectory());
 
-policyNames.forEach((name) => {
-  const policy = require(path.resolve(__dirname, name));
-  policies[name] = Object.assign(policy, { name });
-  register(policy);
-});
+const load = (policiesToBeLoaded) => {
+  const corePolicies = policiesToBeLoaded.filter(name => policyNames.includes(name));
+
+  corePolicies.forEach(name => {
+    if (!policies[name]) {
+      logger.debug(`registering policy ${chalk.green(name)}`);
+      const policy = require(path.resolve(__dirname, name));
+      policies[name] = Object.assign(policy, { name });
+      register(policy);
+    }
+  });
+};
 
-module.exports = { register, resolve, policies };
+module.exports = { register, resolve, policies, load };

lib/policies/oauth2/index.js

@@ -1,3 +1,8 @@
+const schemas = require('../../schemas');
+const jwtSchema = require('../jwt').schema;
+
+schemas.register('policy', 'jwt', jwtSchema);
+
 module.exports = {
   policy: require('./oauth2'),
   routes: require('./oauth2-routes'),

lib/policies/oauth2/oauth2.js

@@ -1,4 +1,3 @@
-'use strict';
 const passport = require('passport');
 const LocalStrategy = require('passport-local').Strategy;
 const ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy;
@@ -9,6 +8,7 @@ const jwtPolicy = require('../jwt/jwt');
 const services = require('../../services/index');
 const authService = services.auth;
 
+require('../basic-auth/registerStrategy')();
 passport.use(new LocalStrategy({ passReqToCallback: true }, authenticateLocal));
 passport.use(new ClientPasswordStrategy({ passReqToCallback: true }, authenticateBasic));
 passport.use(new BearerStrategy({ passReqToCallback: true }, authenticateToken));
@@ -24,7 +24,7 @@ passport.deserializeUser((id, done) => {
     .catch(err => done(err));
 });
 
-function authenticateToken (req, accessToken, done) {
+function authenticateToken(req, accessToken, done) {
   let endpointScopes;
   if (req.egContext.apiEndpoint && req.egContext.apiEndpoint.scopes) {
     endpointScopes = req.egContext.apiEndpoint.scopes;
@@ -66,7 +66,7 @@ function authenticateToken (req, accessToken, done) {
     });
 }
 
-function authenticateBasic (req, clientId, clientSecret, done) {
+function authenticateBasic(req, clientId, clientSecret, done) {
   let requestedScopes;
 
   if (req.query.scope) {
@@ -95,7 +95,7 @@ function authenticateBasic (req, clientId, clientSecret, done) {
     .catch(done);
 }
 
-function authenticateLocal (req, clientId, clientSecret, done) {
+function authenticateLocal(req, clientId, clientSecret, done) {
   const credentialType = 'basic-auth';
 
   return authService.authenticateCredential(clientId, clientSecret, credentialType)

package.json

@@ -120,6 +120,7 @@
     "eslint-plugin-node": "^8.0.1",
     "eslint-plugin-promise": "^4.0.1",
     "eslint-plugin-standard": "^4.0.0",
+    "find-free-port": "2.0.0",
     "husky": "^1.3.1",
     "istanbul": "0.4.5",
     "lint-staged": "8.1.5",

test/common/cli.helper.js

@@ -6,7 +6,7 @@ const _cpr = util.promisify(require('cpr'));
 
 const modulePath = path.resolve(__dirname, '..', '..', 'bin', 'index.js');
 
-module.exports.bootstrapFolder = function (options) {
+module.exports.bootstrapFolder = function () {
   return dir()
     .then(tempDir => Promise.all([
       tempDir,
@@ -30,7 +30,7 @@ module.exports.runCLICommand = function ({ adminPort, adminUrl, configDirectoryP
   cliExecOptions.env.EG_ADMIN_URL = adminUrl || `http://localhost:${adminPort}`;
   const command = ['node', modulePath].concat(cliArgs).join(' ');
   return new Promise((resolve, reject) => {
-    exec(command, cliExecOptions, (err, stdout, stderr) => {
+    exec(command, cliExecOptions, (err, stdout) => {
       if (err) {
         reject(err);
         return;

test/common/gateway.helper.js

@@ -53,11 +53,12 @@ module.exports.startGatewayInstance = function ({ dirInfo, gatewayConfig, backen
         const gatewayProcess = fork(modulePath, [], {
           cwd: dirInfo.basePath,
           env: childEnv,
-          stdio: ['pipe', 'pipe', 'pipe', 'ipc']
+          stdio: 'pipe'
         });
 
         gatewayProcess.on('error', reject);
-        gatewayProcess.stdout.on('data', () => {
+
+        gatewayProcess.stdout.once('data', () => {
           request
             .get(`http://localhost:${gatewayPort}/not-found`)
             .ok(res => true)

test/common/server-helper.js

@@ -1,4 +1,4 @@
-const net = require('net');
+const fp = require('find-free-port');
 const express = require('express');
 
 const generateBackendServer = port => {
@@ -19,32 +19,7 @@ const generateBackendServer = port => {
 };
 
 const findOpenPortNumbers = function (count = 1) {
-  let completeCount = 0;
-  const ports = [];
-  return new Promise((resolve, reject) => {
-    for (let i = 0; i < count; i++) {
-      const server = net.createServer();
-
-      server.listen(0);
-
-      server.on('listening', () => {
-        ports.push(server.address().port);
-
-        server.once('close', () => {
-          completeCount++;
-
-          if (completeCount === count) {
-            resolve(ports);
-          }
-        });
-        server.close();
-      });
-
-      server.on('error', (err) => {
-        reject(err);
-      });
-    }
-  });
+  return fp(3000, 3100, '127.0.0.1', count);
 };
 
 module.exports = {

test/e2e/hot-reload.test.js

@@ -148,5 +148,27 @@ describe('hot-reload', () => {
         fs.writeFileSync(testGatewayConfigPath, '{er:t4');
       });
     });
+
+    describe('adds the required policies in when required gateway.config.yml', function () {
+      it('will respond with a 401 - basic-auth policy', function (done) {
+        this.timeout(TEST_TIMEOUT);
+        watcher.once('change', (evt) => {
+          setTimeout(() => {
+            request
+              .get(`http://localhost:${originalGatewayPort}`)
+              .end((err, res) => {
+                should(err).not.be.undefined();
+                should(res.clientError).not.be.undefined();
+                should(res.statusCode).be.eql(401);
+                done();
+              });
+          }, GATEWAY_STARTUP_WAIT_TIME);
+        });
+
+        testGatewayConfigData.policies.push('basic-auth');
+        testGatewayConfigData.pipelines.adminAPI.policies.unshift({ 'basic-auth': {} });
+        fs.writeFileSync(testGatewayConfigPath, yaml.dump(testGatewayConfigData));
+      });
+    });
   });
 });