bin: allow testing subsystems

this PR adds the functionality to run `citgm-all` on specific tags in
the lookup by passing in the `includeTags` or `excludeTags` option.
fixes #72

README.md

@@ -107,6 +107,8 @@ Options:
   -j, --parallel <number>     Run tests in parallel
   -J, --autoParallel          Run tests in parallel (automatically detect core count)
   --tmpDir <path>             Directory to test modules in
+  --includeTags tag1 tag2     Only test modules from the lookup that contain a matching tag field
+  --excludeTags tag1 tag2     Specify which tags to skip from the lookup (takes priority over includeTags)
 ```
 
 When using a JSON config file, the properties need to be the same as the
@@ -133,6 +135,7 @@ For syntax, see [lookup.json](./lib/lookup.json), the available attributes are:
 "envVar"                     Pass an environment variable before running
 "install": ["--param1", "--param2"] - Array of extra command line parameters passed to 'npm install'
 "maintainers": ["user1", "user2"] - List of module maintainers to be contacted with issues
+"tags": ["tag1", "tag2"]     Specify which tags apply to the module
 ```
 
 If you want to pass options to npm, eg `--registry`, you can usually define an

bin/citgm-all.js

@@ -5,6 +5,7 @@ const os = require('os');
 const _ = require('lodash');
 const async = require('async');
 
+const checkTags = require('../lib/check-tags');
 const citgm = require('../lib/citgm');
 const commonArgs = require('../lib/common-args');
 const getLookup = require('../lib/lookup').get;
@@ -29,6 +30,14 @@ const yargs = commonArgs(require('yargs'))
     alias: 'J',
     type: 'boolean',
     description: 'Auto detect number of cores to use to run tests in parallel'
+  })
+  .option('includeTags', {
+    type: 'array',
+    description: 'Define which tags from the lookup to run'
+  })
+  .option('excludeTags', {
+    type: 'array',
+    description: 'Define which tags from the lookup to skip'
   });
 
 const app = yargs.argv;
@@ -47,6 +56,15 @@ if (!app.su) {
   log.warn('root', 'Running as root! Use caution!');
 }
 
+if (app.includeTags){
+  log.info('includeTags', 'Only running tests matching these tags: '
+      + app.includeTags);
+}
+if (app.excludeTags){
+  log.info('excludeTags', 'Not running tests matching these tags: '
+     + app.excludeTags);
+}
+
 const options = {
   lookup: app.lookup,
   nodedir: app.nodedir,
@@ -55,7 +73,9 @@ const options = {
   level: app.verbose,
   npmLevel: app.npmLoglevel,
   timeoutLength: app.timeout,
-  tmpDir: app.tmpDir
+  tmpDir: app.tmpDir,
+  includeTags: app.includeTags,
+  excludeTags: app.excludeTags
 };
 
 const lookup = getLookup(options);
@@ -97,6 +117,10 @@ function runCitgm (mod, name, next) {
   const runner = citgm.Tester(name, options);
   let bailed = false;
 
+  if (checkTags(app, mod, name, log)) {
+    return next(); // Skip this module
+  }
+
   function cleanup() {
     bailed = true;
     runner.cleanup();

lib/check-tags.js

@@ -0,0 +1,41 @@
+'use strict';
+const _ = require('lodash');
+
+function checkTags(app, mod, name, log) {
+  // Returns true if the module should be skipped
+
+  if ((app.excludeTags && !mod.tags) ||
+      (!app.includeTags && !app.excludeTags)) {
+    return false; // No checks to run.
+  } else if (app.includeTags && !mod.tags) {
+    return true; // No tags for this module
+  }
+
+  if (typeof mod.tags === 'string') {
+    mod.tags = [mod.tags]; // If mod.tags is a single field
+  }
+
+  let excludeTagMatches = _.intersection(app.excludeTags, mod.tags);
+
+  if (excludeTagMatches.length) {
+    log.info(name,
+        `skipped as these excludeTags matched: ${excludeTagMatches}`);
+    return true; // We matched an excludeTag.
+  } else if (app.excludeTags) {
+    log.info(`${name} will run as no excludeTags matched`);
+    return false; // We did not match an excludeTag.
+  }
+
+  let includeTagMatches = _.intersection(app.includeTags, mod.tags);
+
+  if (includeTagMatches.length) {
+    log.info(name,
+        `will run as these includeTags matched: ${includeTagMatches}`);
+    return false; // We matched an includeTag.
+  } else {
+    log.info(name, 'skipped as no includeTags matched');
+    return true; // We did not match an includeTag.
+  }
+}
+
+module.exports = checkTags;

lib/lookup.js

@@ -101,6 +101,9 @@ function resolve(context, next) {
             ' lookup-install', rep.install);
         context.module.install = rep.install;
       }
+      if (rep.tags) {
+        context.module.tags = rep.tags;
+      }
       context.module.flaky = context.options.failFlaky ?
           false : isMatch(rep.flaky);
       context.module.expectFail = context.options.expectFail ?

lib/lookup.json

@@ -1,7 +1,8 @@
 {
   "lodash": {
     "expectFail": "fips",
-    "maintainers": "jdalton"
+    "maintainers": "jdalton",
+    "tags": "test"
   },
   "underscore": {
     "flaky": ["aix", "s390"],

man/citgm-all.1

@@ -70,6 +70,12 @@ Run tests in parallel (automatically detect core count)
 .TP
 .BR \-\-tmpDir " " \fI<path>\fR
 Directory to test modules in
+.TP
+.BR \-\-includeTags " " \fI<tag1 tag2>\fR
+Only test modules from the lookup that contain a matching tag field
+.TP
+.BR \-\-excludeTags " " \fI<tag1 tag2>\fR
+Specify which tags to skip from the lookup (takes priority over includeTags)
 
 .SH SEE ALSO
 citgm

test/bin/test-citgm-all.js

@@ -116,6 +116,42 @@ test('citgm-all: flaky-fail ignoring flakyness', function (t) {
   });
 });
 
+test('citgm-all: includeTags', function (t) {
+  t.plan(1);
+  const proc = spawn(citgmAllPath, ['--includeTags', 'tag1', '-l',
+    'test/fixtures/custom-lookup-tags.json']);
+  proc.on('error', function(err) {
+    t.error(err);
+  });
+  proc.on('close', function (code) {
+    t.equals(code, 0, 'citgm-all should only run omg-i-pass');
+  });
+});
+
+test('citgm-all: includeTags multiple', function (t) {
+  t.plan(1);
+  const proc = spawn(citgmAllPath, ['--includeTags', 'tag1 noTag1 NoTag2', '-l',
+    'test/fixtures/custom-lookup-tags.json']);
+  proc.on('error', function(err) {
+    t.error(err);
+  });
+  proc.on('close', function (code) {
+    t.equals(code, 0, 'citgm-all should only run omg-i-pass');
+  });
+});
+
+test('citgm-all: excludeTags', function (t) {
+  t.plan(1);
+  const proc = spawn(citgmAllPath, ['--excludeTags', 'tag2', '-l',
+    'test/fixtures/custom-lookup-tags.json']);
+  proc.on('error', function(err) {
+    t.error(err);
+  });
+  proc.on('close', function (code) {
+    t.equals(code, 0, 'citgm-all should not run omg-i-fail');
+  });
+});
+
 test('citgm-all: skip /w rootcheck /w tap to fs /w junit to fs /w append',
 function (t) {
   t.plan(1);

test/fixtures/custom-lookup-tags.json

@@ -0,0 +1,13 @@
+{
+  "omg-i-pass": {
+    "npm": true,
+    "tags": "tag1"
+  },
+  "omg-i-fail": {
+    "npm": true,
+    "tags": ["tag2", "tagNotUsed"]
+  },
+  "omg-i-pass-too": {
+    "npm": true
+  }
+}

test/test-check-tags.js

@@ -0,0 +1,128 @@
+'use strict';
+
+const test = require('tap').test;
+const rewire = require('rewire');
+
+const checkTags = rewire('../lib/check-tags');
+const log = require('../lib/out')({});
+
+test('test includeTags and matching tag', function (t) {
+  const app = {
+    includeTags: ['test']
+  };
+  const mod = {
+    tags: 'test'
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test', log);
+  t.false(result, 'should return false');
+});
+
+test('test includeTags and same excludeTags', function (t) {
+  const app = {
+    includeTags: ['test'],
+    excludeTags: ['test']
+  };
+  const mod = {
+    tags: 'test'
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test', log);
+  t.true(result, 'excludeTags should take priority over includeTags');
+});
+
+test('test several includeTags and matching tag', function (t) {
+  const app = {
+    includeTags: ['test', 'this', 'should', 'still', 'pass']
+  };
+  const mod = {
+    tags: 'test'
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test', log);
+  t.false(result, 'should return false');
+});
+
+test('test several includeTags and multiple tags', function (t) {
+  const app = {
+    includeTags: ['test', 'this', 'should', 'still', 'pass']
+  };
+  const mod = {
+    tags: ['test', 'this', 'is']
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test', log);
+  t.false(result, 'should return false');
+});
+
+
+test('test includeTags and multiple tags', function (t) {
+  const app = {
+    includeTags: ['test']
+  };
+  const mod = {
+    tags: ['test', 'noTest']
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test', log);
+  t.false(result, 'should return false');
+});
+
+
+test('test includeTags and no matching tag', function (t) {
+  const app = {
+    includeTags: ['test']
+  };
+  const mod = {
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test', log);
+  t.true(result, 'should return true');
+});
+
+test('test includeTags and different tag', function (t) {
+  const app = {
+    includeTags: ['test']
+  };
+  const mod = {
+    tags: 'noMatch'
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test', log);
+  t.true(result, 'should return true');
+});
+
+test('test excludeTags and matching tag', function (t) {
+  const app = {
+    excludeTags: ['test']
+  };
+  const mod = {
+    tags: 'test'
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test', log);
+  t.true(result, 'should return true');
+});
+
+test('test excludeTags and no matching tag', function (t) {
+  const app = {
+    excludeTags: ['test']
+  };
+  const mod = {
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test', log);
+  t.false(result, 'should return false');
+});
+
+test('test excludeTags and different tag', function (t) {
+  const app = {
+    excludeTags: ['test']
+  };
+  const mod = {
+    tags: 'noMatch'
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test', log);
+  t.false(result, 'should return false');
+});