bin: allow testing subsystems

this PR adds the functionality to run `citgm-all` on specific tags in
the lookup by passing in the `withTags` or `excludeTags` option.
fixes nodejs#72

README.md

@@ -107,6 +107,8 @@ Options:
   -j, --parallel <number>     Run tests in parallel
   -J, --autoParallel          Run tests in parallel (automatically detect core count)
   --tmpDir <path>             Directory to test modules in
+  --withTags <tag1,tag2>      Only test modules from the lookup that contain a matching tag field
+  --excludeTags <tag1,tag2>   Specify which tags to skip from the lookup (takes priority over withTags)
 ```
 
 When using a JSON config file, the properties need to be the same as the
@@ -133,6 +135,7 @@ For syntax, see [lookup.json](./lib/lookup.json), the available attributes are:
 "sha": "<git-commit-sha>"    Test against a specific commit
 "envVar"                     Pass an environment variable before running
 "install": ["--param1", "--param2"] - Array of extra command line parameters passed to 'npm install'
+"tags": ["tag1", "tag2"]     Specify which tags apply to the module
 ```
 
 If you want to pass options to npm, eg `--registry`, you can usually define an

bin/citgm-all.js

@@ -5,6 +5,7 @@ const os = require('os');
 const _ = require('lodash');
 const async = require('async');
 
+const checkTags = require('../lib/check-tags');
 const citgm = require('../lib/citgm');
 const commonArgs = require('../lib/common-args');
 const getLookup = require('../lib/lookup').get;
@@ -29,6 +30,14 @@ const yargs = commonArgs(require('yargs'))
     alias: 'J',
     type: 'boolean',
     description: 'Auto detect number of cores to use to run tests in parallel'
+  })
+  .option('withTags', {
+    type: 'string',
+    description: 'Define which tags from the lookup to run'
+  })
+  .option('excludeTags', {
+    type: 'string',
+    description: 'Define which tags from the lookup to skip'
   });
 
 const app = yargs.argv;
@@ -47,6 +56,17 @@ if (!app.su) {
   log.warn('root', 'Running as root! Use caution!');
 }
 
+if (app.withTags){
+  app.withTags = app.withTags.split(',');
+  log.info('withTags', 'Only running tests matching these tags: '
+      + app.withTags);
+}
+if (app.excludeTags){
+  app.excludeTags = app.excludeTags.split(',');
+  log.info('excludeTags', 'Not running tests matching these tags: '
+     + app.excludeTags);
+}
+
 const options = {
   lookup: app.lookup,
   nodedir: app.nodedir,
@@ -55,7 +75,9 @@ const options = {
   level: app.verbose,
   npmLevel: app.npmLoglevel,
   timeoutLength: app.timeout,
-  tmpDir: app.tmpDir
+  tmpDir: app.tmpDir,
+  withTags: app.withTags,
+  excludeTags: app.excludeTags
 };
 
 const lookup = getLookup(options);
@@ -97,6 +119,10 @@ function runCitgm (mod, name, next) {
   const runner = citgm.Tester(name, options);
   let bailed = false;
 
+  if (checkTags(app, mod, name)) {
+    return next(); // Skip this module
+  }
+
   function cleanup() {
     bailed = true;
     runner.cleanup();

lib/check-tags.js

@@ -0,0 +1,43 @@
+'use strict';
+const log = require('./out')({});
+
+function checkTags(app, mod, name) {
+  // Returns true if the module should be skipped
+
+  if ((app.excludeTags && !mod.tags) || (!app.withTags && !app.excludeTags)) {
+    return false; // No checks to run.
+  } else if (app.withTags && !mod.tags) {
+    return true; // No tags for this module
+  }
+
+  if (typeof mod.tags === 'string') {
+    mod.tags = mod.tags.split(','); // If mod.tags is a single field
+  }
+
+  function checkMatches (A, B) {
+    return A.filter((tag) => B.indexOf(tag) !== -1);
+  }
+
+  let excludeTagMatches = app.excludeTags ?
+      checkMatches(mod.tags, app.excludeTags) : [];
+  if (excludeTagMatches.length) {
+    log.info(name,
+        `skipped as these excludeTags matched: ${excludeTagMatches}`);
+    return true; // We matched an excludeTag.
+  } else if (app.excludeTags) {
+    log.info(`${name} will run as no excludeTags matched`);
+    return false; // We did not match an excludeTag.
+  }
+
+  let withTagMatches = app.withTags ?
+    checkMatches(mod.tags, app.withTags) : [];
+  if (withTagMatches.length) {
+    log.info(name, `will run as these withTags matched: ${withTagMatches}`);
+    return false; // We matched a withTag.
+  } else {
+    log.info(name, 'skipped as no withTags matched');
+    return true; // We didn't match any withTags.
+  }
+}
+
+module.exports = checkTags;

lib/lookup.js

@@ -106,6 +106,9 @@ function resolve(context, next) {
             rep.script);
         context.module.script = rep.script;
       }
+      if (rep.tags) {
+        context.module.tags = rep.tags;
+      }
       context.module.flaky = context.options.failFlaky ?
           false : isMatch(rep.flaky);
       context.module.expectFail = context.options.expectFail ?

lib/lookup.json

@@ -1,6 +1,7 @@
 {
   "lodash": {
-    "expectFail": "fips"
+    "expectFail": "fips",
+    "tags": ["tag1", "tag2"]
   },
   "underscore": {
     "master": true,
@@ -15,7 +16,8 @@
   "commander": {
     "prefix": "v",
     "repo": "https://github.com/tj/commander.js",
-    "skip": "win32"
+    "skip": "win32",
+    "tags": ["tag1", "tag2"]
   },
   "express": {
     "flaky": "ppc",

man/citgm-all.1

@@ -70,6 +70,12 @@ Run tests in parallel (automatically detect core count)
 .TP
 .BR \-\-tmpDir " " \fI<path>\fR
 Directory to test modules in
+.TP
+.BR \-\-withTags " " \fI<tag1,tag2>\fR
+Only test modules from the lookup that contain a matching tag field
+.TP
+.BR \-\-excludeTags " " \fI<tag1,tag2>\fR
+Specify which tags to skip from the lookup (takes priority over withTags)
 
 .SH SEE ALSO
 citgm

test/bin/test-citgm-all.js

@@ -128,6 +128,42 @@ test('citgm-all: flaky-fail ignoring flakyness', function (t) {
   });
 });
 
+test('citgm-all: withTags', function (t) {
+  t.plan(1);
+  const proc = spawn(citgmAllPath, ['--withTags', 'tag1', '-l',
+    'test/fixtures/custom-lookup-tags.json']);
+  proc.on('error', function(err) {
+    t.error(err);
+  });
+  proc.on('close', function (code) {
+    t.equals(code, 0, 'citgm-all should only run omg-i-pass');
+  });
+});
+
+test('citgm-all: withTags multiple', function (t) {
+  t.plan(1);
+  const proc = spawn(citgmAllPath, ['--withTags', 'tag1,noTag1,NoTag2', '-l',
+    'test/fixtures/custom-lookup-tags.json']);
+  proc.on('error', function(err) {
+    t.error(err);
+  });
+  proc.on('close', function (code) {
+    t.equals(code, 0, 'citgm-all should only run omg-i-pass');
+  });
+});
+
+test('citgm-all: excludeTags', function (t) {
+  t.plan(1);
+  const proc = spawn(citgmAllPath, ['--excludeTags', 'tag2', '-l',
+    'test/fixtures/custom-lookup-tags.json']);
+  proc.on('error', function(err) {
+    t.error(err);
+  });
+  proc.on('close', function (code) {
+    t.equals(code, 0, 'citgm-all should not run omg-i-fail');
+  });
+});
+
 test('citgm-all: skip /w rootcheck /w tap to fs /w junit to fs /w append',
 function (t) {
   t.plan(1);

test/fixtures/custom-lookup-tags.json

@@ -0,0 +1,13 @@
+{
+  "omg-i-pass": {
+    "npm": true,
+    "tags": "tag1"
+  },
+  "omg-i-fail": {
+    "npm": true,
+    "tags": ["tag2", "tagNotUsed"]
+  },
+  "omg-i-pass-too": {
+    "npm": true
+  }
+}

test/test-check-tags.js

@@ -0,0 +1,127 @@
+'use strict';
+
+const test = require('tap').test;
+const rewire = require('rewire');
+
+const checkTags = rewire('../lib/check-tags');
+
+test('test withTags and matching tag', function (t) {
+  const app = {
+    withTags: 'test'
+  };
+  const mod = {
+    tags: 'test'
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test');
+  t.false(result, 'should return false');
+});
+
+test('test withTags and same excludeTags', function (t) {
+  const app = {
+    withTags: 'test',
+    excludeTags: 'test'
+  };
+  const mod = {
+    tags: 'test'
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test');
+  t.true(result, 'excludeTags should take priority over withTags');
+});
+
+test('test several withTags and matching tag', function (t) {
+  const app = {
+    withTags: 'test,this,should,still,pass'
+  };
+  const mod = {
+    tags: 'test'
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test');
+  t.false(result, 'should return false');
+});
+
+test('test several withTags and multiple tags', function (t) {
+  const app = {
+    withTags: 'test,this,should,still,pass'
+  };
+  const mod = {
+    tags: ['test', 'this', 'is']
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test');
+  t.false(result, 'should return false');
+});
+
+
+test('test withTags and multiple tags', function (t) {
+  const app = {
+    withTags: 'test'
+  };
+  const mod = {
+    tags: ['test', 'noTest']
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test');
+  t.false(result, 'should return false');
+});
+
+
+test('test withTags and no matching tag', function (t) {
+  const app = {
+    withTags: 'test'
+  };
+  const mod = {
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test');
+  t.true(result, 'should return true');
+});
+
+test('test withTags and different tag', function (t) {
+  const app = {
+    withTags: 'test'
+  };
+  const mod = {
+    tags: 'noMatch'
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test');
+  t.true(result, 'should return true');
+});
+
+test('test excludeTags and matching tag', function (t) {
+  const app = {
+    excludeTags: 'test'
+  };
+  const mod = {
+    tags: 'test'
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test');
+  t.true(result, 'should return true');
+});
+
+test('test excludeTags and no matching tag', function (t) {
+  const app = {
+    excludeTags: 'test'
+  };
+  const mod = {
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test');
+  t.false(result, 'should return false');
+});
+
+test('test excludeTags and different tag', function (t) {
+  const app = {
+    excludeTags: 'test'
+  };
+  const mod = {
+    tags: 'noMatch'
+  };
+  t.plan(1);
+  const result = checkTags(app, mod, 'test');
+  t.false(result, 'should return false');
+});