Skip to content

Commit

Permalink
Support error logging before jest retry (#12201)
Browse files Browse the repository at this point in the history
Co-authored-by: Zachary Bogard <zbogard@microsoft.com>
  • Loading branch information
zackasaurus and zbogard authored Apr 23, 2022
1 parent a28db24 commit 811228d
Show file tree
Hide file tree
Showing 18 changed files with 151 additions and 12 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

- `[babel-jest]` Export `createTransformer` function ([#12399](https://github.com/facebook/jest/pull/12399))
- `[expect]` Expose `AsymmetricMatchers`, `MatcherFunction` and `MatcherFunctionWithState` interfaces ([#12363](https://github.com/facebook/jest/pull/12363), [#12376](https://github.com/facebook/jest/pull/12376))
- `[jest-circus]` Support error logging before retry ([#12201](https://github.com/facebook/jest/pull/12201))
- `[jest-circus, jest-jasmine2]` Allowed classes and functions as `describe` and `it`/`test` names ([#12484](https://github.com/facebook/jest/pull/12484))
- `[jest-cli, jest-config]` [**BREAKING**] Remove `testURL` config, use `testEnvironmentOptions.url` instead ([#10797](https://github.com/facebook/jest/pull/10797))
- `[jest-cli, jest-core]` Add `--shard` parameter for distributed parallel test execution ([#12546](https://github.com/facebook/jest/pull/12546))
Expand Down
13 changes: 11 additions & 2 deletions docs/JestObjectAPI.md
Original file line number Diff line number Diff line change
Expand Up @@ -831,9 +831,9 @@ Example:
jest.setTimeout(1000); // 1 second
```

### `jest.retryTimes()`
### `jest.retryTimes(numRetries, options)`

Runs failed tests n-times until they pass or until the max number of retries is exhausted. This only works with the default [jest-circus](https://github.com/facebook/jest/tree/main/packages/jest-circus) runner! This must live at the top-level of a test file or in a describe block. Retries _will not_ work if `jest.retryTimes()` is called in a `beforeEach` or a `test` block.
Runs failed tests n-times until they pass or until the max number of retries is exhausted. `options` are optional. This only works with the default [jest-circus](https://github.com/facebook/jest/tree/main/packages/jest-circus) runner! This must live at the top-level of a test file or in a describe block. Retries _will not_ work if `jest.retryTimes()` is called in a `beforeEach` or a `test` block.

Example in a test:

Expand All @@ -844,4 +844,13 @@ test('will fail', () => {
});
```

If `logErrorsBeforeRetry` is enabled, Jest will log the error(s) that caused the test to fail to the console, providing visibility on why a retry occurred.

```js
jest.retryTimes(3, {logErrorsBeforeRetry: true});
test('will fail', () => {
expect(true).toBe(false);
});
```

Returns the `jest` object for chaining.
39 changes: 39 additions & 0 deletions e2e/__tests__/__snapshots__/testRetries.test.ts.snap
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`Test Retries logs error(s) before retry 1`] = `
"LOGGING RETRY ERRORS retryTimes set
RETRY 1
expect(received).toBeFalsy()
Received: true
14 | expect(true).toBeTruthy();
15 | } else {
> 16 | expect(true).toBeFalsy();
| ^
17 | }
18 | });
19 |
at Object.toBeFalsy (__tests__/logErrorsBeforeRetries.test.js:16:18)
RETRY 2
expect(received).toBeFalsy()
Received: true
14 | expect(true).toBeTruthy();
15 | } else {
> 16 | expect(true).toBeFalsy();
| ^
17 | }
18 | });
19 |
at Object.toBeFalsy (__tests__/logErrorsBeforeRetries.test.js:16:18)
PASS __tests__/logErrorsBeforeRetries.test.js
✓ retryTimes set"
`;
11 changes: 11 additions & 0 deletions e2e/__tests__/testRetries.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import * as path from 'path';
import * as fs from 'graceful-fs';
import {skipSuiteOnJasmine} from '@jest/test-utils';
import {extractSummary} from '../Utils';
import runJest from '../runJest';

skipSuiteOnJasmine();
Expand All @@ -19,6 +20,7 @@ describe('Test Retries', () => {
'e2e/test-retries/',
outputFileName,
);
const logErrorsBeforeRetryErrorMessage = 'LOGGING RETRY ERRORS';

afterAll(() => {
fs.unlinkSync(outputFilePath);
Expand All @@ -29,6 +31,15 @@ describe('Test Retries', () => {

expect(result.exitCode).toEqual(0);
expect(result.failed).toBe(false);
expect(result.stderr).not.toContain(logErrorsBeforeRetryErrorMessage);
});

it('logs error(s) before retry', () => {
const result = runJest('test-retries', ['logErrorsBeforeRetries.test.js']);
expect(result.exitCode).toEqual(0);
expect(result.failed).toBe(false);
expect(result.stderr).toContain(logErrorsBeforeRetryErrorMessage);
expect(extractSummary(result.stderr).rest).toMatchSnapshot();
});

it('reporter shows more than 1 invocation if test is retried', () => {
Expand Down
18 changes: 18 additions & 0 deletions e2e/test-retries/__tests__/logErrorsBeforeRetries.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
'use strict';

let i = 0;
jest.retryTimes(3, {logErrorsBeforeRetry: true});
it('retryTimes set', () => {
i++;
if (i === 3) {
expect(true).toBeTruthy();
} else {
expect(true).toBeFalsy();
}
});
8 changes: 7 additions & 1 deletion packages/jest-circus/src/eventHandler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import {
injectGlobalErrorHandlers,
restoreGlobalErrorHandlers,
} from './globalErrorHandlers';
import {TEST_TIMEOUT_SYMBOL} from './types';
import {LOG_ERRORS_BEFORE_RETRY, TEST_TIMEOUT_SYMBOL} from './types';
import {
addErrorToEachTestUnderDescribe,
describeBlockHasTests,
Expand Down Expand Up @@ -205,6 +205,12 @@ const eventHandler: Circus.EventHandler = (event, state) => {
break;
}
case 'test_retry': {
const logErrorsBeforeRetry: boolean =
// eslint-disable-next-line no-restricted-globals
global[LOG_ERRORS_BEFORE_RETRY] || false;
if (logErrorsBeforeRetry) {
event.test.retryReasons.push(...event.test.errors);
}
event.test.errors = [];
break;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ export const runAndTransformResultsToJestFormat = async ({
invocations: testResult.invocations,
location: testResult.location,
numPassingAsserts: 0,
retryReasons: testResult.retryReasons,
status,
title: testResult.testPath[testResult.testPath.length - 1],
};
Expand Down
2 changes: 2 additions & 0 deletions packages/jest-circus/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@ export const STATE_SYM = Symbol('JEST_STATE_SYMBOL');
export const RETRY_TIMES = Symbol.for('RETRY_TIMES');
// To pass this value from Runtime object to state we need to use global[sym]
export const TEST_TIMEOUT_SYMBOL = Symbol.for('TEST_TIMEOUT_SYMBOL');
export const LOG_ERRORS_BEFORE_RETRY = Symbol.for('LOG_ERRORS_BEFORE_RETRY');

declare global {
namespace NodeJS {
interface Global {
[STATE_SYM]: Circus.State;
[RETRY_TIMES]: string;
[TEST_TIMEOUT_SYMBOL]: number;
[LOG_ERRORS_BEFORE_RETRY]: boolean;
}
}
}
3 changes: 3 additions & 0 deletions packages/jest-circus/src/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ export const makeTest = (
mode,
name: convertDescriptorToString(name),
parent,
retryReasons: [],
seenDone: false,
startedAt: null,
status: null,
Expand Down Expand Up @@ -354,6 +355,7 @@ export const makeSingleTestResult = (
errorsDetailed,
invocations: test.invocations,
location,
retryReasons: test.retryReasons.map(_getError).map(getErrorStack),
status,
testPath: Array.from(testPath),
};
Expand Down Expand Up @@ -475,6 +477,7 @@ export const parseSingleTestResult = (
invocations: testResult.invocations,
location: testResult.location,
numPassingAsserts: 0,
retryReasons: Array.from(testResult.retryReasons),
status,
title: testResult.testPath[testResult.testPath.length - 1],
};
Expand Down
10 changes: 9 additions & 1 deletion packages/jest-environment/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -232,10 +232,18 @@ export interface Jest {
* Runs failed tests n-times until they pass or until the max number of
* retries is exhausted.
*
* If `logErrorsBeforeRetry` is enabled, Jest will log the error(s) that caused
* the test to fail to the console, providing visibility on why a retry occurred.
* retries is exhausted.
*
* @remarks
* Only available with `jest-circus` runner.
*/
retryTimes(numRetries: number): Jest;
retryTimes(
numRetries: number,
options?: {logErrorsBeforeRetry?: boolean},
): Jest;

/**
* Exhausts tasks queued by `setImmediate()`.
*
Expand Down
10 changes: 5 additions & 5 deletions packages/jest-message-util/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ const STACK_PATH_REGEXP = /\s*at.*\(?(\:\d*\:\d*|native)\)?/;
const EXEC_ERROR_MESSAGE = 'Test suite failed to run';
const NOT_EMPTY_LINE_REGEXP = /^(?!$)/gm;

const indentAllLines = (lines: string, indent: string) =>
lines.replace(NOT_EMPTY_LINE_REGEXP, indent);
export const indentAllLines = (lines: string): string =>
lines.replace(NOT_EMPTY_LINE_REGEXP, MESSAGE_INDENT);

const trim = (string: string) => (string || '').trim();

Expand All @@ -86,7 +86,7 @@ const getRenderedCallsite = (
{highlightCode: true},
);

renderedCallsite = indentAllLines(renderedCallsite, MESSAGE_INDENT);
renderedCallsite = indentAllLines(renderedCallsite);

renderedCallsite = `\n${renderedCallsite}\n`;
return renderedCallsite;
Expand Down Expand Up @@ -157,7 +157,7 @@ export const formatExecError = (

message = checkForCommonEnvironmentErrors(message);

message = indentAllLines(message, MESSAGE_INDENT);
message = indentAllLines(message);

stack =
stack && !options.noStackTrace
Expand Down Expand Up @@ -360,7 +360,7 @@ export const formatResultsErrors = (
formatStackTrace(stack, config, options, testPath),
)}\n`;

message = indentAllLines(message, MESSAGE_INDENT);
message = indentAllLines(message);

const title = `${chalk.bold.red(
TITLE_INDENT +
Expand Down
34 changes: 33 additions & 1 deletion packages/jest-reporters/src/DefaultReporter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ import type {
TestResult,
} from '@jest/test-result';
import type {Config} from '@jest/types';
import {
formatStackTrace,
indentAllLines,
separateMessageFromStack,
} from 'jest-message-util';
import {clearLine, isInteractive} from 'jest-util';
import BaseReporter from './BaseReporter';
import Status from './Status';
Expand Down Expand Up @@ -181,10 +186,37 @@ export default class DefaultReporter extends BaseReporter {
}

printTestFileHeader(
_testPath: string,
testPath: string,
config: Config.ProjectConfig,
result: TestResult,
): void {
// log retry errors if any exist
result.testResults.forEach(testResult => {
const testRetryReasons = testResult.retryReasons;
if (testRetryReasons && testRetryReasons.length > 0) {
this.log(
`${chalk.reset.inverse.bold.yellow(
' LOGGING RETRY ERRORS ',
)} ${chalk.bold(testResult.fullName)}`,
);
testRetryReasons.forEach((retryReasons, index) => {
let {message, stack} = separateMessageFromStack(retryReasons);
stack = this._globalConfig.noStackTrace
? ''
: chalk.dim(
formatStackTrace(stack, config, this._globalConfig, testPath),
);

message = indentAllLines(message);

this.log(
`${chalk.reset.inverse.bold.blueBright(` RETRY ${index + 1} `)}\n`,
);
this.log(`${message}\n${stack}\n`);
});
}
});

this.log(getResultHeader(result, this._globalConfig, config));
if (result.console) {
this.log(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ const testResult = {
updated: 0,
},
testFilePath: '/foo',
testResults: [],
};

let stdout;
Expand Down
6 changes: 5 additions & 1 deletion packages/jest-runtime/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ type ResolveOptions = Parameters<typeof require.resolve>[1] & {

const testTimeoutSymbol = Symbol.for('TEST_TIMEOUT_SYMBOL');
const retryTimesSymbol = Symbol.for('RETRY_TIMES');
const logErrorsBeforeRetrySymbol = Symbol.for('LOG_ERRORS_BEFORE_RETRY');

const NODE_MODULES = `${path.sep}node_modules${path.sep}`;

Expand Down Expand Up @@ -2117,8 +2118,11 @@ export default class Runtime {
return jestObject;
};

const retryTimes = (numTestRetries: number) => {
const retryTimes: Jest['retryTimes'] = (numTestRetries, options) => {
this._environment.global[retryTimesSymbol] = numTestRetries;
this._environment.global[logErrorsBeforeRetrySymbol] =
options?.logErrorsBeforeRetry;

return jestObject;
};

Expand Down
1 change: 1 addition & 0 deletions packages/jest-types/__typetests__/jest.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ expectError(jest.unmock());

// Mock Functions

expectType<typeof jest>(jest.retryTimes(3, {logErrorsBeforeRetry: true}));
expectType<typeof jest>(jest.clearAllMocks());
expectError(jest.clearAllMocks('moduleName'));

Expand Down
2 changes: 2 additions & 0 deletions packages/jest-types/src/Circus.ts
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ export type TestResult = {
invocations: number;
status: TestStatus;
location?: {column: number; line: number} | null;
retryReasons: Array<FormattedError>;
testPath: Array<TestName | BlockName>;
};

Expand Down Expand Up @@ -234,6 +235,7 @@ export type TestEntry = {
type: 'test';
asyncError: Exception; // Used if the test failure contains no usable stack trace
errors: Array<TestError>;
retryReasons: Array<TestError>;
fn: TestFn;
invocations: number;
mode: TestMode;
Expand Down
1 change: 1 addition & 0 deletions packages/jest-types/src/TestResult.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ export type AssertionResult = {
invocations?: number;
location?: Callsite | null;
numPassingAsserts: number;
retryReasons?: Array<string>;
status: Status;
title: string;
};
Expand Down
2 changes: 1 addition & 1 deletion packages/jest-util/src/deepCyclicCopy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export default function deepCyclicCopy<T>(
options: DeepCyclicCopyOptions = {blacklist: EMPTY, keepPrototype: false},
cycles: WeakMap<any, any> = new WeakMap(),
): T {
if (typeof value !== 'object' || value === null) {
if (typeof value !== 'object' || value === null || Buffer.isBuffer(value)) {
return value;
} else if (cycles.has(value)) {
return cycles.get(value);
Expand Down

0 comments on commit 811228d

Please sign in to comment.