bin: allow testing subsystems

this PR adds the functionality to run `citgm-all` on specific tags in
the lookup by passing in the `includeTags` or `excludeTags` option.
fixes #72

README.md

@@ -107,6 +107,8 @@ Options:
   -j, --parallel <number>     Run tests in parallel
   -J, --autoParallel          Run tests in parallel (automatically detect core count)
   --tmpDir <path>             Directory to test modules in
+  --includeTags tag1 tag2     Only test modules from the lookup that contain a matching tag field
+  --excludeTags tag1 tag2     Specify which tags to skip from the lookup (takes priority over includeTags)
 ```
 
 When using a JSON config file, the properties need to be the same as the
@@ -133,6 +135,7 @@ For syntax, see [lookup.json](./lib/lookup.json), the available attributes are:
 "envVar"                     Pass an environment variable before running
 "install": ["--param1", "--param2"] - Array of extra command line parameters passed to 'npm install'
 "maintainers": ["user1", "user2"] - List of module maintainers to be contacted with issues
+"tags": ["tag1", "tag2"]     Specify which tags apply to the module
 ```
 
 If you want to pass options to npm, eg `--registry`, you can usually define an

bin/citgm-all.js

@@ -5,6 +5,7 @@ const os = require('os');
 const _ = require('lodash');
 const async = require('async');
 
+const checkTags = require('../lib/check-tags');
 const citgm = require('../lib/citgm');
 const commonArgs = require('../lib/common-args');
 const getLookup = require('../lib/lookup').get;
@@ -29,6 +30,14 @@ const yargs = commonArgs(require('yargs'))
     alias: 'J',
     type: 'boolean',
     description: 'Auto detect number of cores to use to run tests in parallel'
+  })
+  .option('includeTags', {
+    type: 'array',
+    description: 'Define which tags from the lookup to run'
+  })
+  .option('excludeTags', {
+    type: 'array',
+    description: 'Define which tags from the lookup to skip'
   });
 
 const app = yargs.argv;
@@ -55,9 +64,20 @@ const options = {
   level: app.verbose,
   npmLevel: app.npmLoglevel,
   timeoutLength: app.timeout,
-  tmpDir: app.tmpDir
+  tmpDir: app.tmpDir,
+  includeTags: app.includeTags || [],
+  excludeTags: app.excludeTags || []
 };
 
+if (options.includeTags.length){
+  log.info('includeTags', 'Only running tests matching these tags: '
+      + app.includeTags);
+}
+if (options.excludeTags.length){
+  log.info('excludeTags', 'Not running tests matching these tags: '
+     + app.excludeTags);
+}
+
 const lookup = getLookup(options);
 if (!lookup) {
   log.error('the json file cannot be found or there is an error in the file!');
@@ -97,6 +117,10 @@ function runCitgm (mod, name, next) {
   const runner = citgm.Tester(name, options);
   let bailed = false;
 
+  if (checkTags(options, mod, name, log)) {
+    return next(); // Skip this module
+  }
+
   function cleanup() {
     bailed = true;
     runner.cleanup();

lib/check-tags.js

@@ -0,0 +1,45 @@
+'use strict';
+const _ = require('lodash');
+
+function checkTags(options, mod, name, log) {
+  // Returns true if the module should be skipped.
+
+  if ((options.excludeTags.length && !options.includeTags.length && !mod.tags)
+    || (!options.includeTags.length && !options.excludeTags.length)) {
+    return false; // No checks to run.
+  } else if (options.includeTags.length && !options.excludeTags.length
+      && !mod.tags) {
+    return true; // No tags for this module.
+  }
+
+  if (typeof mod.tags === 'string') {
+    mod.tags = [mod.tags]; // If mod.tags is a single string, convert to array.
+  }
+
+  let excludeTagMatches = _.intersection(options.excludeTags, mod.tags);
+
+  if (excludeTagMatches.length) {
+    log.info(name,
+        `skipped as these excludeTags matched: ${excludeTagMatches}`);
+    return true; // We matched an excludeTag.
+  }
+
+  let includeTagMatches = _.intersection(options.includeTags, mod.tags);
+
+  if (includeTagMatches.length) {
+    log.info(name,
+        `will run as these includeTags matched: ${includeTagMatches}`);
+    return false; // We matched an includeTag.
+  }
+
+  if (options.includeTags.length) {
+    log.info(`${name} skipped as no includeTags matched`);
+    return true; // We did not match an includeTag.
+  } else {
+    log.info(`${name} will run as no excludeTags matched`);
+    return false; // We did not match an excludeTag.
+  }
+
+}
+
+module.exports = checkTags;

lib/lookup.js

@@ -101,6 +101,9 @@ function resolve(context, next) {
             ' lookup-install', rep.install);
         context.module.install = rep.install;
       }
+      if (rep.tags) {
+        context.module.tags = rep.tags;
+      }
       context.module.flaky = context.options.failFlaky ?
           false : isMatch(rep.flaky);
       context.module.expectFail = context.options.expectFail ?

man/citgm-all.1

@@ -70,6 +70,12 @@ Run tests in parallel (automatically detect core count)
 .TP
 .BR \-\-tmpDir " " \fI<path>\fR
 Directory to test modules in
+.TP
+.BR \-\-includeTags " " \fI<tag1 tag2>\fR
+Only test modules from the lookup that contain a matching tag field
+.TP
+.BR \-\-excludeTags " " \fI<tag1 tag2>\fR
+Specify which tags to skip from the lookup (takes priority over includeTags)
 
 .SH SEE ALSO
 citgm

test/bin/test-citgm-all.js

@@ -116,6 +116,42 @@ test('citgm-all: flaky-fail ignoring flakyness', function (t) {
   });
 });
 
+test('citgm-all: includeTags', function (t) {
+  t.plan(1);
+  const proc = spawn(citgmAllPath, ['--includeTags', 'tag1', '-l',
+    'test/fixtures/custom-lookup-tags.json']);
+  proc.on('error', function(err) {
+    t.error(err);
+  });
+  proc.on('close', function (code) {
+    t.equals(code, 0, 'citgm-all should only run omg-i-pass');
+  });
+});
+
+test('citgm-all: excludeTags', function (t) {
+  t.plan(1);
+  const proc = spawn(citgmAllPath, ['--excludeTags', 'tag2', '-l',
+    'test/fixtures/custom-lookup-tags.json']);
+  proc.on('error', function(err) {
+    t.error(err);
+  });
+  proc.on('close', function (code) {
+    t.equals(code, 0, 'citgm-all should not run omg-i-fail');
+  });
+});
+
+test('citgm-all: includeTags multiple', function (t) {
+  t.plan(1);
+  const proc = spawn(citgmAllPath, ['--includeTags', 'tag1 noTag1 NoTag2', '-l',
+    'test/fixtures/custom-lookup-tags.json']);
+  proc.on('error', function(err) {
+    t.error(err);
+  });
+  proc.on('close', function (code) {
+    t.equals(code, 0, 'citgm-all should only run omg-i-pass');
+  });
+});
+
 test('citgm-all: skip /w rootcheck /w tap to fs /w junit to fs /w append',
 function (t) {
   t.plan(1);

test/fixtures/custom-lookup-tags.json

@@ -0,0 +1,13 @@
+{
+  "omg-i-pass": {
+    "npm": true,
+    "tags": "tag1"
+  },
+  "omg-i-fail": {
+    "npm": true,
+    "tags": ["tag2", "tagNotUsed"]
+  },
+  "omg-i-pass-too": {
+    "npm": true
+  }
+}