test helper improvements

- enables `RawResult` (the result of `invokeMocha()` or `invokeMochaAsync()`) to check the exit code via `to have code` assertion
- add passed/failing/pending assertions for `RawRunResult` (the result of `runMocha()` or `runMochaAsync()`)

- expose `getSummary()`, which can be used with a `RawResult` (when not failing)
- reorganize the module a bit
- create `runMochaAsync()` and `runMochaJSONAsync()` which are like `runMocha()` and `runMochaJSON()` except return `Promise`s
- better trapping of JSON parse errors
- better default handling of `STDERR` output in subprocesses (print it instead of suppress it!)
- do not let the `DEBUG` env variable reach subprocesses _unless it was explicitly supplied_
- add an easily copy-paste-able `command` prop to summary
- add some missing docstrings

Ref: #4198

test/assertions.js

@@ -118,6 +118,24 @@ exports.mixinMochaAssertions = function(expect) {
         });
       }
     )
+    .addAssertion(
+      '<RawRunResult> [not] to have failed [test] count <number>',
+      function(expect, result, count) {
+        expect(result.failing, '[not] to be', count);
+      }
+    )
+    .addAssertion(
+      '<RawRunResult> [not] to have passed [test] count <number>',
+      function(expect, result, count) {
+        expect(result.passing, '[not] to be', count);
+      }
+    )
+    .addAssertion(
+      '<RawRunResult> [not] to have pending [test] count <number>',
+      function(expect, result, count) {
+        expect(result.pending, '[not] to be', count);
+      }
+    )
     .addAssertion('<JSONRunResult> [not] to have test count <number>', function(
       expect,
       result,
@@ -315,7 +333,7 @@ exports.mixinMochaAssertions = function(expect) {
       }
     )
     .addAssertion(
-      '<RawRunResult|JSONRunResult> to have [exit] code <number>',
+      '<RawResult|RawRunResult|JSONRunResult> to have [exit] code <number>',
       function(expect, result, code) {
         expect(result.code, 'to be', code);
       }

test/integration/helpers.js

@@ -4,13 +4,69 @@ var format = require('util').format;
 var spawn = require('cross-spawn').spawn;
 var path = require('path');
 var Base = require('../../lib/reporters/base');
-
+var debug = require('debug')('mocha:tests:integration:helpers');
 var DEFAULT_FIXTURE = resolveFixturePath('__default__');
 var MOCHA_EXECUTABLE = require.resolve('../../bin/mocha');
 var _MOCHA_EXECUTABLE = require.resolve('../../bin/_mocha');
 
 module.exports = {
   DEFAULT_FIXTURE: DEFAULT_FIXTURE,
+
+  /**
+   * regular expression used for splitting lines based on new line / dot symbol.
+   */
+  splitRegExp: new RegExp('[\\n' + Base.symbols.dot + ']+'),
+
+  /**
+   * Invokes the mocha binary. Accepts an array of additional command line args
+   * to pass. The callback is invoked with the exit code and output. Optional
+   * current working directory as final parameter.
+   *
+   * By default, `STDERR` is ignored. Pass `{stdio: 'pipe'}` as `opts` if you
+   * want it.
+   *
+   * In most cases runMocha should be used instead.
+   *
+   * Example response:
+   * {
+   *   code:    1,
+   *   output:  '...'
+   * }
+   *
+   * @param {Array<string>} args - Extra args to mocha executable
+   * @param {Function} done - Callback
+   * @param {Object} [opts] - Options for `spawn()`
+   */
+  invokeMocha: invokeMocha,
+
+  invokeMochaAsync: invokeMochaAsync,
+
+  invokeNode: invokeNode,
+
+  getSummary: getSummary,
+
+  /**
+   * Resolves the path to a fixture to the full path.
+   */
+  resolveFixturePath: resolveFixturePath,
+
+  toJSONRunResult: toJSONRunResult,
+
+  /**
+   * Given a regexp-like string, escape it so it can be used with the `RegExp` constructor
+   * @param {string} str - string to be escaped
+   * @returns {string} Escaped string
+   */
+  escapeRegExp: function escapeRegExp(str) {
+    return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
+  },
+
+  runMocha: runMocha,
+  runMochaJSON: runMochaJSON,
+  runMochaAsync: runMochaAsync,
+  runMochaJSONAsync: runMochaJSONAsync
+};
+
   /**
    * Invokes the mocha binary for the given fixture with color output disabled.
    * Accepts an array of additional command line args to pass. The callback is
@@ -35,7 +91,7 @@ module.exports = {
    * @param {Object} [opts] - Options for `spawn()`
    * @returns {ChildProcess} Mocha process
    */
-  runMocha: function(fixturePath, args, fn, opts) {
+function runMocha(fixturePath, args, fn, opts) {
     if (typeof args === 'function') {
       opts = fn;
       fn = args;
@@ -48,7 +104,7 @@ module.exports = {
     args = args || [];
 
     return invokeSubMocha(
-      args.concat(['-C', path]),
+    args.concat(path),
       function(err, res) {
         if (err) {
           return fn(err);
@@ -58,7 +114,7 @@ module.exports = {
       },
       opts
     );
-  },
+}
 
   /**
    * Invokes the mocha binary for the given fixture using the JSON reporter,
@@ -72,7 +128,7 @@ module.exports = {
    * @param {Object} [opts] - Opts for `spawn()`
    * @returns {*} Parsed object
    */
-  runMochaJSON: function(fixturePath, args, fn, opts) {
+function runMochaJSON(fixturePath, args, fn, opts) {
     if (typeof args === 'function') {
       opts = fn;
       fn = args;
@@ -113,55 +169,55 @@ module.exports = {
       },
       opts
     );
-  },
-
-  /**
-   * regular expression used for splitting lines based on new line / dot symbol.
-   */
-  splitRegExp: new RegExp('[\\n' + Base.symbols.dot + ']+'),
+}
+ * Like {@link runMocha}, but returns a `Promise`.
 
   /**
-   * Invokes the mocha binary. Accepts an array of additional command line args
-   * to pass. The callback is invoked with the exit code and output. Optional
-   * current working directory as final parameter.
    *
-   * By default, `STDERR` is ignored. Pass `{stdio: 'pipe'}` as `opts` if you
-   * want it.
-   *
-   * In most cases runMocha should be used instead.
+ * If you need more granular control, try {@link invokeMochaAsync} instead.
    *
-   * Example response:
-   * {
-   *   code:    1,
-   *   output:  '...'
-   * }
-   *
-   * @param {Array<string>} args - Extra args to mocha executable
-   * @param {Function} done - Callback
-   * @param {Object} [opts] - Options for `spawn()`
+ * @param {string} fixturePath - Path to (or name of, or basename of) fixture file
+ * @param {Options} [args] - Command-line arguments to the `mocha` executable
+ * @param {Object} [opts] - Options for `child_process.spawn`.
+ * @returns {Promise<Summary>}
    */
-  invokeMocha: invokeMocha,
-
-  invokeMochaAsync: invokeMochaAsync,
-
-  invokeNode: invokeNode,
-
-  /**
-   * Resolves the path to a fixture to the full path.
-   */
-  resolveFixturePath: resolveFixturePath,
-
-  toJSONRunResult: toJSONRunResult,
+function runMochaAsync(fixturePath, args, opts) {
+  return new Promise(function(resolve, reject) {
+    runMocha(
+      fixturePath,
+      args,
+      function(err, result) {
+        if (err) {
+          return reject(err);
+        }
+        resolve(result);
+      },
+      opts
+    );
+  });
+}
 
   /**
-   * Given a regexp-like string, escape it so it can be used with the `RegExp` constructor
-   * @param {string} str - string to be escaped
-   * @returns {string} Escaped string
+ * Like {@link runMochaJSON}, but returns a `Promise`.
+ * @param {string} fixturePath - Path to (or name of, or basename of) fixture file
+ * @param {Options} [args] - Command-line args
+ * @param {Object} [opts] - Options for `child_process.spawn`
    */
-  escapeRegExp: function escapeRegExp(str) {
-    return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
+function runMochaJSONAsync(fixturePath, args, opts) {
+  return new Promise(function(resolve, reject) {
+    runMochaJSON(
+      fixturePath,
+      args,
+      function(err, result) {
+        if (err) {
+          return reject(err);
+        }
+        resolve(result);
+      },
+      opts
+    );
+  });
   }
-};
 
 /**
  * Coerce output as returned by _spawnMochaWithListeners using JSON reporter into a JSONRunResult as
@@ -171,9 +227,15 @@ module.exports = {
  */
 function toJSONRunResult(result) {
   var code = result.code;
+  try {
   result = JSON.parse(result.output);
   result.code = code;
   return result;
+  } catch (err) {
+    throw new Error(
+      `Couldn't parse JSON: ${err.message}\n\nOriginal result output: ${result.output}`
+    );
+  }
 }
 
 /**
@@ -267,16 +329,24 @@ function invokeSubMocha(args, fn, opts) {
  */
 function _spawnMochaWithListeners(args, fn, opts) {
   var output = '';
+  opts = opts || {};
   if (opts === 'pipe') {
-    opts = {stdio: 'pipe'};
+    opts = {stdio: ['inherit', 'pipe', 'pipe']};
   }
+  var env = Object.assign({}, process.env);
+  // prevent DEBUG from borking STDERR when piping, unless explicitly set via `opts`
+  delete env.DEBUG;
+
   opts = Object.assign(
     {
       cwd: process.cwd(),
-      stdio: ['ignore', 'pipe', 'ignore']
+      stdio: ['inherit', 'pipe', 'inherit'],
+      env: env
     },
-    opts || {}
+    opts
   );
+
+  debug('spawning: %s', [process.execPath].concat(args).join(' '));
   var mocha = spawn(process.execPath, args, opts);
   var listener = function(data) {
     output += data;
@@ -292,7 +362,8 @@ function _spawnMochaWithListeners(args, fn, opts) {
     fn(null, {
       output: output,
       code: code,
-      args: args
+      args: args,
+      command: args.join(' ')
     });
   });
 
@@ -306,6 +377,11 @@ function resolveFixturePath(fixture) {
   return path.join('test', 'integration', 'fixtures', fixture);
 }
 
+/**
+ * Parses some `mocha` reporter output and returns a summary based on the "epilogue"
+ * @param {string} res - Typically output of STDOUT from the 'spec' reporter
+ * @returns {Summary}
+ */
 function getSummary(res) {
   return ['passing', 'pending', 'failing'].reduce(function(summary, type) {
     var pattern, match;
@@ -317,3 +393,11 @@ function getSummary(res) {
     return summary;
   }, res);
 }
+
+/**
+ * A summary of a `mocha` run
+ * @typedef {Object} Summary
+ * @property {number} passing - Number of passing tests
+ * @property {number} pending - Number of pending tests
+ * @property {number} failing - Number of failing tests
+ */