Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Report skipped tests upon beforeAll hook failure #4221

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 39 additions & 5 deletions docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ Mocha is a feature-rich JavaScript test framework running on [Node.js][] and in
- [test coverage reporting](#wallabyjs)
- [string diff support](#diffs)
- [javascript API for running tests](#more-information)
- proper exit status for CI support etc
- [auto-detects and disables coloring for non-ttys](#reporters)
- [async test timeout support](#delayed-root-suite)
- [test retry support](#retry-tests)
Expand All @@ -36,7 +35,6 @@ Mocha is a feature-rich JavaScript test framework running on [Node.js][] and in
- [auto-exit to prevent "hanging" with an active loop](#-exit)
- [easily meta-generate suites](#markdown) & [test-cases](#list)
- [config file support](#-config-path)
- clickable suite titles to filter test execution
- [node debugger support](#-inspect-inspect-brk-inspect)
- [node native ES modules support](#nodejs-native-esm-support)
- [detects multiple calls to `done()`](#detects-multiple-calls-to-done)
Expand Down Expand Up @@ -449,9 +447,43 @@ setTimeout(function() {
}, 5000);
```

### Failing Hooks

Upon a failing `before` hook all tests in the current suite and also its nested suites will be skipped. Skipped tests are included in the test results and marked as **skipped**. A skipped test is not considered a failed test.

```js
describe('outer', function() {
before(function() {
throw new Error('Exception in before hook');
});

it('should skip this outer test', function() {
// will be skipped and listed as 'skipped'
});

after(function() {
// will be executed
});

describe('inner', function() {
before(function() {
// will be skipped
});

it('should skip this inner test', function() {
// will be skipped and listed as 'skipped'
});

after(function() {
// will be skipped
});
});
});
```

## Pending Tests

"Pending"--as in "someone should write these test cases eventually"--test-cases are simply those _without_ a callback:
"Pending" - as in "someone should write these test cases eventually" - test-cases are simply those _without_ a callback:

```js
describe('Array', function() {
Expand All @@ -462,7 +494,9 @@ describe('Array', function() {
});
```

Pending tests will be included in the test results, and marked as pending. A pending test is not considered a failed test.
By appending `.skip()`, you may also tell Mocha to ignore a test: [inclusive tests](#inclusive-tests).

Pending tests will be included in the test results, and marked as **pending**. A pending test is not considered a failed test.

## Exclusive Tests

Expand Down Expand Up @@ -1641,7 +1675,7 @@ mocha.setup({
### Browser-specific Option(s)

Browser Mocha supports many, but not all [cli options](#command-line-usage).
To use a [cli option](#command-line-usage) that contains a "-", please convert the option to camel-case, (eg. `check-leaks` to `checkLeaks`).
To use a [cli option](#command-line-usage) that contains a "-", please convert the option to camel-case, (e.g. `check-leaks` to `checkLeaks`).

#### Options that differ slightly from [cli options](#command-line-usage):

Expand Down
13 changes: 10 additions & 3 deletions lib/reporters/base.js
Original file line number Diff line number Diff line change
Expand Up @@ -343,18 +343,25 @@ Base.prototype.epilogue = function() {
// passes
fmt =
color('bright pass', ' ') +
color('green', ' %d passing') +
color('green', ' %d / %d passing') +
color('light', ' (%s)');

Base.consoleLog(fmt, stats.passes || 0, milliseconds(stats.duration));
Base.consoleLog(fmt, stats.passes, stats.tests, milliseconds(stats.duration));

// pending
if (stats.pending) {
fmt = color('pending', ' ') + color('pending', ' %d pending');
fmt = color('pending', ' %d pending');

Base.consoleLog(fmt, stats.pending);
}

// skipped
if (stats.skipped) {
fmt = color('fail', ' %d skipped');

Base.consoleLog(fmt, stats.skipped);
}

// failures
if (stats.failures) {
fmt = color('fail', ' %d failing');
Expand Down
15 changes: 11 additions & 4 deletions lib/reporters/json.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,11 @@
var Base = require('./base');
var constants = require('../runner').constants;
var EVENT_TEST_PASS = constants.EVENT_TEST_PASS;
var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING;
var EVENT_TEST_SKIPPED = constants.EVENT_TEST_SKIPPED;
var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL;
var EVENT_TEST_END = constants.EVENT_TEST_END;
var EVENT_RUN_END = constants.EVENT_RUN_END;
var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING;

/**
* Expose `JSON`.
Expand All @@ -35,9 +36,10 @@ function JSONReporter(runner, options) {

var self = this;
var tests = [];
var passes = [];
var pending = [];
var skipped = [];
var failures = [];
var passes = [];

runner.on(EVENT_TEST_END, function(test) {
tests.push(test);
Expand All @@ -55,13 +57,18 @@ function JSONReporter(runner, options) {
pending.push(test);
});

runner.on(EVENT_TEST_SKIPPED, function(test) {
skipped.push(test);
});

runner.once(EVENT_RUN_END, function() {
var obj = {
stats: self.stats,
tests: tests.map(clean),
passes: passes.map(clean),
pending: pending.map(clean),
failures: failures.map(clean),
passes: passes.map(clean)
skipped: skipped.map(clean),
failures: failures.map(clean)
};

runner.testResults = obj;
Expand Down
6 changes: 6 additions & 0 deletions lib/reporters/spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ var EVENT_SUITE_END = constants.EVENT_SUITE_END;
var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL;
var EVENT_TEST_PASS = constants.EVENT_TEST_PASS;
var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING;
var EVENT_TEST_SKIPPED = constants.EVENT_TEST_SKIPPED;
var inherits = require('../utils').inherits;
var color = Base.color;

Expand Down Expand Up @@ -88,6 +89,11 @@ function Spec(runner, options) {
Base.consoleLog(indent() + color('fail', ' %d) %s'), ++n, test.title);
});

runner.on(EVENT_TEST_SKIPPED, function(test) {
var fmt = indent() + color('fail', ' - %s');
Base.consoleLog(fmt, test.title);
});

runner.once(EVENT_RUN_END, self.epilogue.bind(self));
}

Expand Down
20 changes: 18 additions & 2 deletions lib/runnable.js
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ function Runnable(title, fn) {
this._retries = -1;
this._currentRetry = 0;
this.pending = false;
this.skipped = false;
}

/**
Expand Down Expand Up @@ -166,6 +167,17 @@ Runnable.prototype.isPassed = function() {
return !this.isPending() && this.state === constants.STATE_PASSED;
};

/**
* Return `true` if this Runnable has been skipped.
* @return {boolean}
* @private
*/
Runnable.prototype.isSkipped = function() {
return (
!this.pending && (this.skipped || (this.parent && this.parent.isSkipped()))
);
};

/**
* Set or get number of retries.
*
Expand Down Expand Up @@ -270,7 +282,7 @@ Runnable.prototype.run = function(fn) {
var finished;
var emitted;

if (this.isPending()) return fn();
if (this.isSkipped() || this.isPending()) return fn();

// Sometimes the ctx exists, but it is not runnable
if (ctx && ctx.runnable) {
Expand Down Expand Up @@ -458,7 +470,11 @@ var constants = utils.defineConstants(
/**
* Value of `state` prop when a `Runnable` has been skipped by user
*/
STATE_PENDING: 'pending'
STATE_PENDING: 'pending',
/**
* Value of `state` prop when a `Runnable` has been skipped by failing hook
*/
STATE_SKIPPED: 'skipped'
}
);

Expand Down
Loading