From efd8b37d128e6b14742dc2b3f903ecc8d36d0b8e Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Tue, 4 Apr 2023 00:41:13 +0300 Subject: [PATCH] test: migrate test runner message tests to snapshot PR-URL: https://github.com/nodejs/node/pull/47392 Reviewed-By: Geoffrey Booth Reviewed-By: Benjamin Gruenbaum --- .github/CODEOWNERS | 1 - test/common/README.md | 6 + test/common/assertSnapshot.js | 53 +++ test/common/index.js | 83 ++-- test/common/index.mjs | 2 + .../test-runner/output/abort.js} | 2 +- .../test-runner/output/abort.snapshot} | 16 +- .../test-runner/output/abort_suite.js} | 2 +- .../test-runner/output/abort_suite.snapshot} | 8 +- .../test-runner/output/describe_it.js} | 2 +- .../test-runner/output/describe_it.snapshot} | 14 +- .../test-runner/output/describe_nested.js} | 2 +- .../output/describe_nested.snapshot} | 0 .../test-runner/output/dot_reporter.js | 8 + .../test-runner/output/dot_reporter.snapshot} | 0 .../test-runner/output/hooks.js} | 2 +- .../test-runner/output/hooks.snapshot} | 22 +- .../test-runner/output/name_pattern.js} | 2 +- .../test-runner/output/name_pattern.snapshot} | 0 .../output/name_pattern_with_only.js} | 2 +- .../output/name_pattern_with_only.snapshot} | 0 .../test-runner/output/no_refs.js} | 2 +- .../test-runner/output/no_refs.snapshot} | 0 .../test-runner/output/no_tests.js} | 2 +- .../test-runner/output/no_tests.snapshot} | 0 .../test-runner/output/only_tests.js} | 2 +- .../test-runner/output/only_tests.snapshot} | 0 .../test-runner/output/output.js} | 2 +- .../test-runner/output/output.snapshot} | 4 +- .../fixtures/test-runner/output/output_cli.js | 9 + .../test-runner/output/output_cli.snapshot} | 4 +- .../test-runner/output/spec_reporter.js} | 6 +- .../output/spec_reporter.snapshot} | 439 +++++++++--------- .../output/spec_reporter_successful.js} | 2 +- .../output/spec_reporter_successful.snapshot | 9 + .../test-runner/output/unresolved_promise.js} | 2 +- .../output/unresolved_promise.snapshot} | 2 +- test/message/test_runner_output_cli.js | 7 - .../test_runner_output_dot_reporter.js | 6 - .../test_runner_spec_reporter_successful.out | 9 - test/parallel/test-runner-output.mjs | 57 +++ 41 files changed, 460 insertions(+), 331 deletions(-) create mode 100644 test/common/assertSnapshot.js rename test/{message/test_runner_abort.js => fixtures/test-runner/output/abort.js} (98%) rename test/{message/test_runner_abort.out => fixtures/test-runner/output/abort.snapshot} (95%) rename test/{message/test_runner_abort_suite.js => fixtures/test-runner/output/abort_suite.js} (97%) rename test/{message/test_runner_abort_suite.out => fixtures/test-runner/output/abort_suite.snapshot} (95%) rename test/{message/test_runner_describe_it.js => fixtures/test-runner/output/describe_it.js} (99%) rename test/{message/test_runner_describe_it.out => fixtures/test-runner/output/describe_it.snapshot} (98%) rename test/{message/test_runner_describe_nested.js => fixtures/test-runner/output/describe_nested.js} (87%) rename test/{message/test_runner_describe_nested.out => fixtures/test-runner/output/describe_nested.snapshot} (100%) create mode 100644 test/fixtures/test-runner/output/dot_reporter.js rename test/{message/test_runner_output_dot_reporter.out => fixtures/test-runner/output/dot_reporter.snapshot} (100%) rename test/{message/test_runner_hooks.js => fixtures/test-runner/output/hooks.js} (99%) rename test/{message/test_runner_hooks.out => fixtures/test-runner/output/hooks.snapshot} (95%) rename test/{message/test_runner_test_name_pattern.js => fixtures/test-runner/output/name_pattern.js} (97%) rename test/{message/test_runner_test_name_pattern.out => fixtures/test-runner/output/name_pattern.snapshot} (100%) rename test/{message/test_runner_test_name_pattern_with_only.js => fixtures/test-runner/output/name_pattern_with_only.js} (91%) rename test/{message/test_runner_test_name_pattern_with_only.out => fixtures/test-runner/output/name_pattern_with_only.snapshot} (100%) rename test/{message/test_runner_no_refs.js => fixtures/test-runner/output/no_refs.js} (92%) rename test/{message/test_runner_no_refs.out => fixtures/test-runner/output/no_refs.snapshot} (100%) rename test/{message/test_runner_no_tests.js => fixtures/test-runner/output/no_tests.js} (82%) rename test/{message/test_runner_no_tests.out => fixtures/test-runner/output/no_tests.snapshot} (100%) rename test/{message/test_runner_only_tests.js => fixtures/test-runner/output/only_tests.js} (98%) rename test/{message/test_runner_only_tests.out => fixtures/test-runner/output/only_tests.snapshot} (100%) rename test/{message/test_runner_output.js => fixtures/test-runner/output/output.js} (99%) rename test/{message/test_runner_output.out => fixtures/test-runner/output/output.snapshot} (99%) create mode 100644 test/fixtures/test-runner/output/output_cli.js rename test/{message/test_runner_output_cli.out => fixtures/test-runner/output/output_cli.snapshot} (99%) rename test/{message/test_runner_output_spec_reporter.js => fixtures/test-runner/output/spec_reporter.js} (76%) rename test/{message/test_runner_output_spec_reporter.out => fixtures/test-runner/output/spec_reporter.snapshot} (83%) rename test/{message/test_runner_spec_reporter_successful.js => fixtures/test-runner/output/spec_reporter_successful.js} (81%) create mode 100644 test/fixtures/test-runner/output/spec_reporter_successful.snapshot rename test/{message/test_runner_unresolved_promise.js => fixtures/test-runner/output/unresolved_promise.js} (88%) rename test/{message/test_runner_unresolved_promise.out => fixtures/test-runner/output/unresolved_promise.snapshot} (96%) delete mode 100644 test/message/test_runner_output_cli.js delete mode 100644 test/message/test_runner_output_dot_reporter.js delete mode 100644 test/message/test_runner_spec_reporter_successful.out create mode 100644 test/parallel/test-runner-output.mjs diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 464a819844a2c4..29c7cf7c61bb01 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -135,7 +135,6 @@ # Test runner -/test/message/test_runner_* @nodejs/test_runner /test/parallel/test-runner-* @nodejs/test_runner /doc/api/test.md @nodejs/test_runner /lib/test.js @nodejs/test_runner diff --git a/test/common/README.md b/test/common/README.md index 4d26e355d5480f..7688f2b7aa184f 100644 --- a/test/common/README.md +++ b/test/common/README.md @@ -642,6 +642,12 @@ environment variables. If set, `NODE_COMMON_PORT`'s value overrides the `common.PORT` default value of 12346\. +### `NODE_REGENERATE_SNAPSHOTS` + +If set, test snapshots for a the current test are regenerated. +for example `NODE_REGENERATE_SNAPSHOTS=1 out/Release/node test/parallel/test-runner-output.mjs` +will update all the test runner output snapshots. + ### `NODE_SKIP_FLAG_CHECK` If set, command line arguments passed to individual tests are not validated. diff --git a/test/common/assertSnapshot.js b/test/common/assertSnapshot.js new file mode 100644 index 00000000000000..afde9b5d371cd3 --- /dev/null +++ b/test/common/assertSnapshot.js @@ -0,0 +1,53 @@ +'use strict'; +const common = require('.'); +const path = require('node:path'); +const fs = require('node:fs/promises'); +const assert = require('node:assert/strict'); + + +const stackFramesRegexp = /(\s+)((.+?)\s+\()?(?:\(?(.+?):(\d+)(?::(\d+))?)\)?(\s+\{)?(\n|$)/g; +const windowNewlineRegexp = /\r/g; + +function replaceStackTrace(str) { + return str.replace(stackFramesRegexp, '$1*$7\n'); +} + +function replaceWindowsLineEndings(str) { + return str.replace(windowNewlineRegexp, ''); +} + +function transform(...args) { + return (str) => args.reduce((acc, fn) => fn(acc), str); +} + +function getSnapshotPath(filename) { + const { name, dir } = path.parse(filename); + return path.resolve(dir, `${name}.snapshot`); +} + +async function assertSnapshot(actual, filename = process.argv[1]) { + const snapshot = getSnapshotPath(filename); + if (process.env.NODE_REGENERATE_SNAPSHOTS) { + await fs.writeFile(snapshot, actual); + } else { + const expected = await fs.readFile(snapshot, 'utf8'); + assert.strictEqual(actual, replaceWindowsLineEndings(expected)); + } +} + +async function spawnAndAssert(filename, transform = (x) => x) { + // TODO: Add an option to this function to alternatively or additionally compare stderr. + // For now, tests that want to check stderr or both stdout and stderr can use spawnPromisified. + const flags = common.parseTestFlags(filename); + const { stdout } = await common.spawnPromisified(process.execPath, [...flags, filename]); + await assertSnapshot(transform(stdout), filename); +} + +module.exports = { + assertSnapshot, + getSnapshotPath, + replaceStackTrace, + replaceWindowsLineEndings, + spawnAndAssert, + transform, +}; diff --git a/test/common/index.js b/test/common/index.js index 7418b9b8c980ab..92313f33d27c89 100644 --- a/test/common/index.js +++ b/test/common/index.js @@ -60,6 +60,31 @@ const hasOpenSSL3 = hasCrypto && const hasQuic = hasCrypto && !!process.config.variables.openssl_quic; +function parseTestFlags(filename = process.argv[1]) { + // The copyright notice is relatively big and the flags could come afterwards. + const bytesToRead = 1500; + const buffer = Buffer.allocUnsafe(bytesToRead); + const fd = fs.openSync(filename, 'r'); + const bytesRead = fs.readSync(fd, buffer, 0, bytesToRead); + fs.closeSync(fd); + const source = buffer.toString('utf8', 0, bytesRead); + + const flagStart = source.indexOf('// Flags: --') + 10; + + if (flagStart === 9) { + return []; + } + let flagEnd = source.indexOf('\n', flagStart); + // Normalize different EOL. + if (source[flagEnd - 1] === '\r') { + flagEnd--; + } + return source + .substring(flagStart, flagEnd) + .replace(/_/g, '-') + .split(' '); +} + // Check for flags. Skip this for workers (both, the `cluster` module and // `worker_threads`) and child processes. // If the binary was built without-ssl then the crypto flags are @@ -70,44 +95,25 @@ if (process.argv.length === 2 && hasCrypto && require('cluster').isPrimary && fs.existsSync(process.argv[1])) { - // The copyright notice is relatively big and the flags could come afterwards. - const bytesToRead = 1500; - const buffer = Buffer.allocUnsafe(bytesToRead); - const fd = fs.openSync(process.argv[1], 'r'); - const bytesRead = fs.readSync(fd, buffer, 0, bytesToRead); - fs.closeSync(fd); - const source = buffer.toString('utf8', 0, bytesRead); - - const flagStart = source.indexOf('// Flags: --') + 10; - if (flagStart !== 9) { - let flagEnd = source.indexOf('\n', flagStart); - // Normalize different EOL. - if (source[flagEnd - 1] === '\r') { - flagEnd--; - } - const flags = source - .substring(flagStart, flagEnd) - .replace(/_/g, '-') - .split(' '); - const args = process.execArgv.map((arg) => arg.replace(/_/g, '-')); - for (const flag of flags) { - if (!args.includes(flag) && - // If the binary is build without `intl` the inspect option is - // invalid. The test itself should handle this case. - (process.features.inspector || !flag.startsWith('--inspect'))) { - console.log( - 'NOTE: The test started as a child_process using these flags:', - inspect(flags), - 'Use NODE_SKIP_FLAG_CHECK to run the test with the original flags.', - ); - const args = [...flags, ...process.execArgv, ...process.argv.slice(1)]; - const options = { encoding: 'utf8', stdio: 'inherit' }; - const result = spawnSync(process.execPath, args, options); - if (result.signal) { - process.kill(0, result.signal); - } else { - process.exit(result.status); - } + const flags = parseTestFlags(); + const args = process.execArgv.map((arg) => arg.replace(/_/g, '-')); + for (const flag of flags) { + if (!args.includes(flag) && + // If the binary is build without `intl` the inspect option is + // invalid. The test itself should handle this case. + (process.features.inspector || !flag.startsWith('--inspect'))) { + console.log( + 'NOTE: The test started as a child_process using these flags:', + inspect(flags), + 'Use NODE_SKIP_FLAG_CHECK to run the test with the original flags.', + ); + const args = [...flags, ...process.execArgv, ...process.argv.slice(1)]; + const options = { encoding: 'utf8', stdio: 'inherit' }; + const result = spawnSync(process.execPath, args, options); + if (result.signal) { + process.kill(0, result.signal); + } else { + process.exit(result.status); } } } @@ -917,6 +923,7 @@ const common = { mustSucceed, nodeProcessAborted, PIPE, + parseTestFlags, platformTimeout, printSkipMessage, pwdCommand, diff --git a/test/common/index.mjs b/test/common/index.mjs index e77b1b298cbc30..6b4cd00e410d78 100644 --- a/test/common/index.mjs +++ b/test/common/index.mjs @@ -37,6 +37,7 @@ const { getCallSite, mustNotCall, mustNotMutateObjectDeep, + parseTestFlags, printSkipMessage, skip, nodeProcessAborted, @@ -88,6 +89,7 @@ export { getCallSite, mustNotCall, mustNotMutateObjectDeep, + parseTestFlags, printSkipMessage, skip, nodeProcessAborted, diff --git a/test/message/test_runner_abort.js b/test/fixtures/test-runner/output/abort.js similarity index 98% rename from test/message/test_runner_abort.js rename to test/fixtures/test-runner/output/abort.js index bb1e473f89c6eb..0cd9c9b9273503 100644 --- a/test/message/test_runner_abort.js +++ b/test/fixtures/test-runner/output/abort.js @@ -1,6 +1,6 @@ // Flags: --no-warnings 'use strict'; -require('../common'); +require('../../../common'); const test = require('node:test'); test('promise timeout signal', { signal: AbortSignal.timeout(1) }, async (t) => { diff --git a/test/message/test_runner_abort.out b/test/fixtures/test-runner/output/abort.snapshot similarity index 95% rename from test/message/test_runner_abort.out rename to test/fixtures/test-runner/output/abort.snapshot index 91055794b03e27..f756377172da65 100644 --- a/test/message/test_runner_abort.out +++ b/test/fixtures/test-runner/output/abort.snapshot @@ -31,7 +31,7 @@ TAP version 13 # Subtest: not ok 2 not ok 6 - not ok 2 --- - duration_ms: * + duration_ms: ZERO failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' @@ -39,7 +39,7 @@ TAP version 13 # Subtest: not ok 3 not ok 7 - not ok 3 --- - duration_ms: * + duration_ms: ZERO failureType: 'testAborted' error: 'This operation was aborted' code: 20 @@ -59,7 +59,7 @@ TAP version 13 # Subtest: not ok 4 not ok 8 - not ok 4 --- - duration_ms: * + duration_ms: ZERO failureType: 'testAborted' error: 'This operation was aborted' code: 20 @@ -79,7 +79,7 @@ TAP version 13 # Subtest: not ok 5 not ok 9 - not ok 5 --- - duration_ms: * + duration_ms: ZERO failureType: 'testAborted' error: 'This operation was aborted' code: 20 @@ -161,7 +161,7 @@ not ok 2 - promise abort signal # Subtest: not ok 2 not ok 6 - not ok 2 --- - duration_ms: * + duration_ms: ZERO failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' @@ -169,7 +169,7 @@ not ok 2 - promise abort signal # Subtest: not ok 3 not ok 7 - not ok 3 --- - duration_ms: * + duration_ms: ZERO failureType: 'testAborted' error: 'This operation was aborted' code: 20 @@ -189,7 +189,7 @@ not ok 2 - promise abort signal # Subtest: not ok 4 not ok 8 - not ok 4 --- - duration_ms: * + duration_ms: ZERO failureType: 'testAborted' error: 'This operation was aborted' code: 20 @@ -209,7 +209,7 @@ not ok 2 - promise abort signal # Subtest: not ok 5 not ok 9 - not ok 5 --- - duration_ms: * + duration_ms: ZERO failureType: 'testAborted' error: 'This operation was aborted' code: 20 diff --git a/test/message/test_runner_abort_suite.js b/test/fixtures/test-runner/output/abort_suite.js similarity index 97% rename from test/message/test_runner_abort_suite.js rename to test/fixtures/test-runner/output/abort_suite.js index 61415c5cca93f7..419698320a5f7d 100644 --- a/test/message/test_runner_abort_suite.js +++ b/test/fixtures/test-runner/output/abort_suite.js @@ -1,6 +1,6 @@ // Flags: --no-warnings 'use strict'; -require('../common'); +require('../../../common'); const { describe, it } = require('node:test'); describe('describe timeout signal', { signal: AbortSignal.timeout(1) }, (t) => { diff --git a/test/message/test_runner_abort_suite.out b/test/fixtures/test-runner/output/abort_suite.snapshot similarity index 95% rename from test/message/test_runner_abort_suite.out rename to test/fixtures/test-runner/output/abort_suite.snapshot index 00040b760cdfb2..995e8512218bdb 100644 --- a/test/message/test_runner_abort_suite.out +++ b/test/fixtures/test-runner/output/abort_suite.snapshot @@ -31,7 +31,7 @@ TAP version 13 # Subtest: not ok 2 not ok 6 - not ok 2 --- - duration_ms: * + duration_ms: ZERO failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' @@ -39,7 +39,7 @@ TAP version 13 # Subtest: not ok 3 not ok 7 - not ok 3 --- - duration_ms: * + duration_ms: ZERO failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' @@ -47,7 +47,7 @@ TAP version 13 # Subtest: not ok 4 not ok 8 - not ok 4 --- - duration_ms: * + duration_ms: ZERO failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' @@ -55,7 +55,7 @@ TAP version 13 # Subtest: not ok 5 not ok 9 - not ok 5 --- - duration_ms: * + duration_ms: ZERO failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' diff --git a/test/message/test_runner_describe_it.js b/test/fixtures/test-runner/output/describe_it.js similarity index 99% rename from test/message/test_runner_describe_it.js rename to test/fixtures/test-runner/output/describe_it.js index 8d162c48a7305c..c6d3f9c1b72fb3 100644 --- a/test/message/test_runner_describe_it.js +++ b/test/fixtures/test-runner/output/describe_it.js @@ -1,6 +1,6 @@ // Flags: --no-warnings 'use strict'; -require('../common'); +require('../../../common'); const assert = require('node:assert'); const { describe, it, test } = require('node:test'); const util = require('util'); diff --git a/test/message/test_runner_describe_it.out b/test/fixtures/test-runner/output/describe_it.snapshot similarity index 98% rename from test/message/test_runner_describe_it.out rename to test/fixtures/test-runner/output/describe_it.snapshot index 209cf5bb6715e8..e085ff9535ec70 100644 --- a/test/message/test_runner_describe_it.out +++ b/test/fixtures/test-runner/output/describe_it.snapshot @@ -196,10 +196,10 @@ ok 21 - immediate resolve pass * * * + new Promise () * * - * - * + Array.map () ... # Subtest: mixing describe/it and test should work ok 2 - mixing describe/it and test should work @@ -469,10 +469,10 @@ not ok 53 - custom inspect symbol that throws fail * * * + new Promise () * * - * - * + Array.map () ... # Subtest: sync throw fails at second not ok 2 - sync throw fails at second @@ -491,7 +491,7 @@ not ok 53 - custom inspect symbol that throws fail * * * - * + async Promise.all (index 0) ... 1..2 not ok 54 - subtest sync throw fails @@ -506,7 +506,7 @@ not ok 54 - subtest sync throw fails # Subtest: should not run not ok 1 - should not run --- - duration_ms: * + duration_ms: ZERO failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' @@ -535,7 +535,7 @@ not ok 55 - describe sync throw fails # Subtest: should not run not ok 1 - should not run --- - duration_ms: * + duration_ms: ZERO failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' diff --git a/test/message/test_runner_describe_nested.js b/test/fixtures/test-runner/output/describe_nested.js similarity index 87% rename from test/message/test_runner_describe_nested.js rename to test/fixtures/test-runner/output/describe_nested.js index 64819d62f7ebd0..40ea150a018f3d 100644 --- a/test/message/test_runner_describe_nested.js +++ b/test/fixtures/test-runner/output/describe_nested.js @@ -1,6 +1,6 @@ // Flags: --no-warnings 'use strict'; -require('../common'); +require('../../../common'); const { describe, it } = require('node:test'); describe('nested - no tests', () => { diff --git a/test/message/test_runner_describe_nested.out b/test/fixtures/test-runner/output/describe_nested.snapshot similarity index 100% rename from test/message/test_runner_describe_nested.out rename to test/fixtures/test-runner/output/describe_nested.snapshot diff --git a/test/fixtures/test-runner/output/dot_reporter.js b/test/fixtures/test-runner/output/dot_reporter.js new file mode 100644 index 00000000000000..72a8aaa10e491b --- /dev/null +++ b/test/fixtures/test-runner/output/dot_reporter.js @@ -0,0 +1,8 @@ +// Flags: --no-warnings +'use strict'; +require('../../../common'); +const fixtures = require('../../../common/fixtures'); +const spawn = require('node:child_process').spawn; + +spawn(process.execPath, + ['--no-warnings', '--test-reporter', 'dot', fixtures.path('test-runner/output/output.js')], { stdio: 'inherit' }); diff --git a/test/message/test_runner_output_dot_reporter.out b/test/fixtures/test-runner/output/dot_reporter.snapshot similarity index 100% rename from test/message/test_runner_output_dot_reporter.out rename to test/fixtures/test-runner/output/dot_reporter.snapshot diff --git a/test/message/test_runner_hooks.js b/test/fixtures/test-runner/output/hooks.js similarity index 99% rename from test/message/test_runner_hooks.js rename to test/fixtures/test-runner/output/hooks.js index bd5adb762d86fd..16a58c5a17af57 100644 --- a/test/message/test_runner_hooks.js +++ b/test/fixtures/test-runner/output/hooks.js @@ -1,6 +1,6 @@ // Flags: --no-warnings 'use strict'; -const common = require('../common'); +const common = require('../../../common'); const assert = require('assert'); const { test, describe, it, before, after, beforeEach, afterEach } = require('node:test'); diff --git a/test/message/test_runner_hooks.out b/test/fixtures/test-runner/output/hooks.snapshot similarity index 95% rename from test/message/test_runner_hooks.out rename to test/fixtures/test-runner/output/hooks.snapshot index 48280122e1e21e..50e866f3cdff5d 100644 --- a/test/message/test_runner_hooks.out +++ b/test/fixtures/test-runner/output/hooks.snapshot @@ -37,7 +37,7 @@ ok 1 - describe hooks # Subtest: 1 not ok 1 - 1 --- - duration_ms: * + duration_ms: ZERO failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' @@ -45,7 +45,7 @@ ok 1 - describe hooks # Subtest: 2 not ok 2 - 2 --- - duration_ms: * + duration_ms: ZERO failureType: 'cancelledByParent' error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' @@ -113,7 +113,7 @@ not ok 3 - after throws * * * - * + async Promise.all (index 0) * * ... @@ -132,7 +132,7 @@ not ok 3 - after throws * * * - * + async Promise.all (index 0) * * ... @@ -162,7 +162,7 @@ not ok 4 - beforeEach throws * * * - * + async Promise.all (index 0) * ... # Subtest: 2 @@ -181,7 +181,7 @@ not ok 4 - beforeEach throws * * * - * + async Promise.all (index 0) * ... 1..2 @@ -208,10 +208,10 @@ not ok 5 - afterEach throws * * * + new Promise () * * - * - * + Array.map () ... # Subtest: 2 ok 2 - 2 @@ -242,10 +242,10 @@ not ok 6 - afterEach when test fails * * * + new Promise () * * - * - * + Array.map () ... # Subtest: 2 not ok 2 - 2 @@ -263,7 +263,7 @@ not ok 6 - afterEach when test fails * * * - * + async Promise.all (index 0) * ... 1..2 diff --git a/test/message/test_runner_test_name_pattern.js b/test/fixtures/test-runner/output/name_pattern.js similarity index 97% rename from test/message/test_runner_test_name_pattern.js rename to test/fixtures/test-runner/output/name_pattern.js index 738feafe31eced..6e8bca0f4ffb0e 100644 --- a/test/message/test_runner_test_name_pattern.js +++ b/test/fixtures/test-runner/output/name_pattern.js @@ -1,6 +1,6 @@ // Flags: --no-warnings --test-name-pattern=enabled --test-name-pattern=/pattern/i 'use strict'; -const common = require('../common'); +const common = require('../../../common'); const { after, afterEach, diff --git a/test/message/test_runner_test_name_pattern.out b/test/fixtures/test-runner/output/name_pattern.snapshot similarity index 100% rename from test/message/test_runner_test_name_pattern.out rename to test/fixtures/test-runner/output/name_pattern.snapshot diff --git a/test/message/test_runner_test_name_pattern_with_only.js b/test/fixtures/test-runner/output/name_pattern_with_only.js similarity index 91% rename from test/message/test_runner_test_name_pattern_with_only.js rename to test/fixtures/test-runner/output/name_pattern_with_only.js index 0f074594525351..1dc0f362f9be28 100644 --- a/test/message/test_runner_test_name_pattern_with_only.js +++ b/test/fixtures/test-runner/output/name_pattern_with_only.js @@ -1,6 +1,6 @@ // Flags: --no-warnings --test-only --test-name-pattern=enabled 'use strict'; -const common = require('../common'); +const common = require('../../../common'); const { test } = require('node:test'); test('enabled and only', { only: true }, common.mustCall(async (t) => { diff --git a/test/message/test_runner_test_name_pattern_with_only.out b/test/fixtures/test-runner/output/name_pattern_with_only.snapshot similarity index 100% rename from test/message/test_runner_test_name_pattern_with_only.out rename to test/fixtures/test-runner/output/name_pattern_with_only.snapshot diff --git a/test/message/test_runner_no_refs.js b/test/fixtures/test-runner/output/no_refs.js similarity index 92% rename from test/message/test_runner_no_refs.js rename to test/fixtures/test-runner/output/no_refs.js index 8f2815f067af15..7b36e01133da6e 100644 --- a/test/message/test_runner_no_refs.js +++ b/test/fixtures/test-runner/output/no_refs.js @@ -1,6 +1,6 @@ // Flags: --no-warnings 'use strict'; -require('../common'); +require('../../../common'); const test = require('node:test'); // When run alone, the test below does not keep the event loop alive. diff --git a/test/message/test_runner_no_refs.out b/test/fixtures/test-runner/output/no_refs.snapshot similarity index 100% rename from test/message/test_runner_no_refs.out rename to test/fixtures/test-runner/output/no_refs.snapshot diff --git a/test/message/test_runner_no_tests.js b/test/fixtures/test-runner/output/no_tests.js similarity index 82% rename from test/message/test_runner_no_tests.js rename to test/fixtures/test-runner/output/no_tests.js index c4d226c0bd27d6..f9a9506706180e 100644 --- a/test/message/test_runner_no_tests.js +++ b/test/fixtures/test-runner/output/no_tests.js @@ -1,6 +1,6 @@ // Flags: --no-warnings 'use strict'; -require('../common'); +require('../../../common'); const test = require('node:test'); // No TAP output should be generated. diff --git a/test/message/test_runner_no_tests.out b/test/fixtures/test-runner/output/no_tests.snapshot similarity index 100% rename from test/message/test_runner_no_tests.out rename to test/fixtures/test-runner/output/no_tests.snapshot diff --git a/test/message/test_runner_only_tests.js b/test/fixtures/test-runner/output/only_tests.js similarity index 98% rename from test/message/test_runner_only_tests.js rename to test/fixtures/test-runner/output/only_tests.js index 7cce63d2d8f866..97df061f8268d3 100644 --- a/test/message/test_runner_only_tests.js +++ b/test/fixtures/test-runner/output/only_tests.js @@ -1,6 +1,6 @@ // Flags: --no-warnings --test-only 'use strict'; -require('../common'); +require('../../../common'); const { test, describe, it } = require('node:test'); // These tests should be skipped based on the 'only' option. diff --git a/test/message/test_runner_only_tests.out b/test/fixtures/test-runner/output/only_tests.snapshot similarity index 100% rename from test/message/test_runner_only_tests.out rename to test/fixtures/test-runner/output/only_tests.snapshot diff --git a/test/message/test_runner_output.js b/test/fixtures/test-runner/output/output.js similarity index 99% rename from test/message/test_runner_output.js rename to test/fixtures/test-runner/output/output.js index b00e833a212492..47d99d1c8d4984 100644 --- a/test/message/test_runner_output.js +++ b/test/fixtures/test-runner/output/output.js @@ -1,6 +1,6 @@ // Flags: --no-warnings 'use strict'; -require('../common'); +require('../../../common'); const assert = require('node:assert'); const test = require('node:test'); const util = require('util'); diff --git a/test/message/test_runner_output.out b/test/fixtures/test-runner/output/output.snapshot similarity index 99% rename from test/message/test_runner_output.out rename to test/fixtures/test-runner/output/output.snapshot index c19e2058de7298..db19d8ca549a38 100644 --- a/test/message/test_runner_output.out +++ b/test/fixtures/test-runner/output/output.snapshot @@ -122,9 +122,9 @@ not ok 13 - async assertion fail failureType: 'testCodeFailure' error: |- Expected values to be strictly equal: - + true !== false - + code: 'ERR_ASSERTION' name: 'AssertionError' expected: false diff --git a/test/fixtures/test-runner/output/output_cli.js b/test/fixtures/test-runner/output/output_cli.js new file mode 100644 index 00000000000000..3a90d7c1991709 --- /dev/null +++ b/test/fixtures/test-runner/output/output_cli.js @@ -0,0 +1,9 @@ +// Flags: --no-warnings +'use strict'; +require('../../../common'); +const fixtures = require('../../../common/fixtures'); +const spawn = require('node:child_process').spawn; + +spawn(process.execPath, + ['--no-warnings', '--test', '--test-reporter', 'tap', fixtures.path('test-runner/output/output.js')], + { stdio: 'inherit' }); diff --git a/test/message/test_runner_output_cli.out b/test/fixtures/test-runner/output/output_cli.snapshot similarity index 99% rename from test/message/test_runner_output_cli.out rename to test/fixtures/test-runner/output/output_cli.snapshot index ba43eabb1940e2..5aba59661451a9 100644 --- a/test/message/test_runner_output_cli.out +++ b/test/fixtures/test-runner/output/output_cli.snapshot @@ -122,9 +122,9 @@ not ok 13 - async assertion fail failureType: 'testCodeFailure' error: |- Expected values to be strictly equal: - + true !== false - + code: 'ERR_ASSERTION' name: 'AssertionError' expected: false diff --git a/test/message/test_runner_output_spec_reporter.js b/test/fixtures/test-runner/output/spec_reporter.js similarity index 76% rename from test/message/test_runner_output_spec_reporter.js rename to test/fixtures/test-runner/output/spec_reporter.js index 49d8d3f2293da1..6a7c2d655f93b3 100644 --- a/test/message/test_runner_output_spec_reporter.js +++ b/test/fixtures/test-runner/output/spec_reporter.js @@ -1,9 +1,11 @@ // Flags: --no-warnings 'use strict'; -require('../common'); +require('../../../common'); +const fixtures = require('../../../common/fixtures'); const spawn = require('node:child_process').spawn; + const child = spawn(process.execPath, - ['--no-warnings', '--test-reporter', 'spec', 'test/message/test_runner_output.js'], + ['--no-warnings', '--test-reporter', 'spec', fixtures.path('test-runner/output/output.js')], { stdio: 'pipe' }); // eslint-disable-next-line no-control-regex child.stdout.on('data', (d) => process.stdout.write(d.toString().replace(/[^\x00-\x7F]/g, '').replace(/\u001b\[\d+m/g, ''))); diff --git a/test/message/test_runner_output_spec_reporter.out b/test/fixtures/test-runner/output/spec_reporter.snapshot similarity index 83% rename from test/message/test_runner_output_spec_reporter.out rename to test/fixtures/test-runner/output/spec_reporter.snapshot index 8db20b3195e6f9..0a470191638573 100644 --- a/test/message/test_runner_output_spec_reporter.out +++ b/test/fixtures/test-runner/output/spec_reporter.snapshot @@ -2,23 +2,23 @@ sync pass todo with message (*ms) sync fail todo (*ms) Error: thrown from sync fail todo - * - * - * - * - * - * - * + * + * + * + * + * + * + * sync fail todo with message (*ms) Error: thrown from sync fail todo with message - * - * - * - * - * - * - * + * + * + * + * + * + * + * sync skip pass (*ms) # SKIP sync skip pass with message (*ms) # SKIP @@ -26,48 +26,48 @@ this test should pass sync throw fail (*ms) Error: thrown from sync throw fail - * - * - * - * - * - * - * + * + * + * + * + * + * + * async skip pass (*ms) # SKIP async pass (*ms) async throw fail (*ms) Error: thrown from async throw fail - * - * - * - * - * - * - * + * + * + * + * + * + * + * async skip fail (*ms) # SKIP Error: thrown from async throw fail - * - * - * - * - * - * - * + * + * + * + * + * + * + * async assertion fail (*ms) AssertionError [ERR_ASSERTION]: Expected values to be strictly equal: - + true !== false - - * - * - * - * - * - * - * { + + * + * + * + * + * + * + * { generatedMessage: true, code: 'ERR_ASSERTION', actual: true, @@ -78,13 +78,13 @@ resolve pass (*ms) reject fail (*ms) Error: rejected from reject fail - * - * - * - * - * - * - * + * + * + * + * + * + * + * unhandled rejection - passes but warns (*ms) async unhandled rejection - passes but warns (*ms) @@ -94,16 +94,16 @@ subtest sync throw fail +sync throw fail (*ms) Error: thrown from subtest sync throw fail - * - * - * - * - * - * - * - * - * - * + * + * + * + * + * + * + * + * + * + * this subtest should make its parent test fail subtest sync throw fail (*ms) @@ -133,13 +133,13 @@ sync skip option with message (*ms) # SKIP sync skip option is false fail (*ms) Error: this should be executed - * - * - * - * - * - * - * + * + * + * + * + * + * + * (*ms) functionOnly (*ms) @@ -149,9 +149,8 @@ (*ms) # SKIP test with a name and options provided (*ms) # SKIP functionAndOptions (*ms) # SKIP - escaped description \ # * - * - (*ms) + escaped description \ # \#\ +  (*ms) escaped skip message (*ms) # SKIP escaped todo message (*ms) escaped diagnostic (*ms) @@ -159,8 +158,8 @@ callback pass (*ms) callback fail (*ms) Error: callback failure - * - * + * + * sync t is this in test (*ms) async t is this in test (*ms) @@ -170,13 +169,13 @@ callback throw (*ms) Error: thrown from callback throw - * - * - * - * - * - * - * + * + * + * + * + * + * + * callback called twice (*ms) 'callback invoked multiple times' @@ -184,7 +183,7 @@ callback called twice in different ticks (*ms) callback called twice in future tick (*ms) Error [ERR_TEST_FAILURE]: callback invoked multiple times - * { + * { failureType: 'multipleCallbackInvocations', cause: 'callback invoked multiple times', code: 'ERR_TEST_FAILURE' @@ -192,8 +191,8 @@ callback async throw (*ms) Error: thrown from callback async throw - * - * + * + * callback async throw after done (*ms) only is set but not in only mode @@ -204,8 +203,8 @@ 'only' and 'runOnly' require the --test-only command-line option. running subtest 4 (*ms) only is set but not in only mode (*ms) - 'only' and 'runOnly' require the --test-only command-line option. + 'only' and 'runOnly' require the --test-only command-line option. custom inspect symbol fail (*ms) customized @@ -215,29 +214,29 @@ subtest sync throw fails sync throw fails at first (*ms) Error: thrown from subtest sync throw fails at first - * - * - * - * - * - * - * - * - * - * + * + * + * + * + * + * + * + * + * + * sync throw fails at second (*ms) Error: thrown from subtest sync throw fails at second - * - * - * - * - * - * - * - * - * - * + * + * + * + * + * + * + * + * + * + * subtest sync throw fails (*ms) @@ -255,15 +254,15 @@ unfinished test with uncaughtException (*ms) Error: foo - * - * - * + * + * + * unfinished test with unhandledRejection (*ms) Error: bar - * - * - * + * + * + * assertion errors display actual and expected properly (*ms) AssertionError [ERR_ASSERTION]: Expected values to be loosely deep-equal: @@ -279,7 +278,7 @@ bar: 2, c: [Circular *1] } - * { + * { generatedMessage: true, code: 'ERR_ASSERTION', actual: [Object], @@ -310,66 +309,66 @@ sync fail todo (*ms) Error: thrown from sync fail todo - * - * - * - * - * - * - * + * + * + * + * + * + * + * sync fail todo with message (*ms) Error: thrown from sync fail todo with message - * - * - * - * - * - * - * + * + * + * + * + * + * + * sync throw fail (*ms) Error: thrown from sync throw fail - * - * - * - * - * - * - * + * + * + * + * + * + * + * async throw fail (*ms) Error: thrown from async throw fail - * - * - * - * - * - * - * + * + * + * + * + * + * + * async skip fail (*ms) Error: thrown from async throw fail - * - * - * - * - * - * - * + * + * + * + * + * + * + * async assertion fail (*ms) AssertionError [ERR_ASSERTION]: Expected values to be strictly equal: - + true !== false - - * - * - * - * - * - * - * { + + * + * + * + * + * + * + * { generatedMessage: true, code: 'ERR_ASSERTION', actual: true, @@ -379,26 +378,26 @@ reject fail (*ms) Error: rejected from reject fail - * - * - * - * - * - * - * + * + * + * + * + * + * + * +sync throw fail (*ms) Error: thrown from subtest sync throw fail - * - * - * - * - * - * - * - * - * - * + * + * + * + * + * + * + * + * + * + * subtest sync throw fail (*ms) '1 subtest failed' @@ -414,38 +413,38 @@ sync skip option is false fail (*ms) Error: this should be executed - * - * - * - * - * - * - * + * + * + * + * + * + * + * callback fail (*ms) Error: callback failure - * - * + * + * callback also returns a Promise (*ms) 'passed a callback but also returned a Promise' callback throw (*ms) Error: thrown from callback throw - * - * - * - * - * - * - * + * + * + * + * + * + * + * callback called twice (*ms) 'callback invoked multiple times' callback called twice in future tick (*ms) Error [ERR_TEST_FAILURE]: callback invoked multiple times - * + * { failureType: 'multipleCallbackInvocations', cause: 'callback invoked multiple times', code: 'ERR_TEST_FAILURE' @@ -453,8 +452,8 @@ callback async throw (*ms) Error: thrown from callback async throw - * - * + * + * custom inspect symbol fail (*ms) customized @@ -464,53 +463,53 @@ sync throw fails at first (*ms) Error: thrown from subtest sync throw fails at first - * - * - * - * - * - * - * - * - * - * + * + * + * + * + * + * + * + * + * + * sync throw fails at second (*ms) Error: thrown from subtest sync throw fails at second - * - * - * - * - * - * - * - * - * - * + * + * + * + * + * + * + * + * + * + * subtest sync throw fails (*ms) '2 subtests failed' timed out async test (*ms) - 'test timed out after 5ms' + 'test timed out after *ms' timed out callback test (*ms) - 'test timed out after 5ms' + 'test timed out after *ms' rejected thenable (*ms) 'custom error' unfinished test with uncaughtException (*ms) Error: foo - * - * - * + * + * + * unfinished test with unhandledRejection (*ms) Error: bar - * - * - * + * + * + * assertion errors display actual and expected properly (*ms) AssertionError [ERR_ASSERTION]: Expected values to be loosely deep-equal: @@ -526,7 +525,7 @@ bar: 2, c: [Circular *1] } - * { + * { generatedMessage: true, code: 'ERR_ASSERTION', actual: [Object], diff --git a/test/message/test_runner_spec_reporter_successful.js b/test/fixtures/test-runner/output/spec_reporter_successful.js similarity index 81% rename from test/message/test_runner_spec_reporter_successful.js rename to test/fixtures/test-runner/output/spec_reporter_successful.js index e5e21e29e15e3e..fb9800c0491722 100644 --- a/test/message/test_runner_spec_reporter_successful.js +++ b/test/fixtures/test-runner/output/spec_reporter_successful.js @@ -1,6 +1,6 @@ // Flags: --no-warnings --test-reporter=spec 'use strict'; -require('../common'); +require('../../../common'); const { it } = require('node:test'); it('should pass', () => {}); diff --git a/test/fixtures/test-runner/output/spec_reporter_successful.snapshot b/test/fixtures/test-runner/output/spec_reporter_successful.snapshot new file mode 100644 index 00000000000000..a494136f56e97c --- /dev/null +++ b/test/fixtures/test-runner/output/spec_reporter_successful.snapshot @@ -0,0 +1,9 @@ +✔ should pass (*ms) +ℹ tests 1 +ℹ suites 0 +ℹ pass 1 +ℹ fail 0 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms * diff --git a/test/message/test_runner_unresolved_promise.js b/test/fixtures/test-runner/output/unresolved_promise.js similarity index 88% rename from test/message/test_runner_unresolved_promise.js rename to test/fixtures/test-runner/output/unresolved_promise.js index a36652669fc0a9..a43f064d44de8e 100644 --- a/test/message/test_runner_unresolved_promise.js +++ b/test/fixtures/test-runner/output/unresolved_promise.js @@ -1,6 +1,6 @@ // Flags: --no-warnings 'use strict'; -require('../common'); +require('../../../common'); const test = require('node:test'); test('pass'); diff --git a/test/message/test_runner_unresolved_promise.out b/test/fixtures/test-runner/output/unresolved_promise.snapshot similarity index 96% rename from test/message/test_runner_unresolved_promise.out rename to test/fixtures/test-runner/output/unresolved_promise.snapshot index d3fadb6b556356..4b1593c3365798 100644 --- a/test/message/test_runner_unresolved_promise.out +++ b/test/fixtures/test-runner/output/unresolved_promise.snapshot @@ -17,7 +17,7 @@ not ok 2 - never resolving promise # Subtest: fail not ok 3 - fail --- - duration_ms: 0 + duration_ms: ZERO failureType: 'cancelledByParent' error: 'Promise resolution is still pending but the event loop has already resolved' code: 'ERR_TEST_FAILURE' diff --git a/test/message/test_runner_output_cli.js b/test/message/test_runner_output_cli.js deleted file mode 100644 index 5645f1afb1f3a2..00000000000000 --- a/test/message/test_runner_output_cli.js +++ /dev/null @@ -1,7 +0,0 @@ -// Flags: --no-warnings -'use strict'; -require('../common'); -const spawn = require('node:child_process').spawn; -spawn(process.execPath, - ['--no-warnings', '--test', '--test-reporter', 'tap', 'test/message/test_runner_output.js'], - { stdio: 'inherit' }); diff --git a/test/message/test_runner_output_dot_reporter.js b/test/message/test_runner_output_dot_reporter.js deleted file mode 100644 index 8c36b9ba245425..00000000000000 --- a/test/message/test_runner_output_dot_reporter.js +++ /dev/null @@ -1,6 +0,0 @@ -// Flags: --no-warnings -'use strict'; -require('../common'); -const spawn = require('node:child_process').spawn; -spawn(process.execPath, - ['--no-warnings', '--test-reporter', 'dot', 'test/message/test_runner_output.js'], { stdio: 'inherit' }); diff --git a/test/message/test_runner_spec_reporter_successful.out b/test/message/test_runner_spec_reporter_successful.out deleted file mode 100644 index e1576063021fc5..00000000000000 --- a/test/message/test_runner_spec_reporter_successful.out +++ /dev/null @@ -1,9 +0,0 @@ -* should pass *(*ms)* -*tests 1* -*suites 0* -*pass 1* -*fail 0* -*cancelled 0* -*skipped 0* -*todo 0* -*duration_ms * diff --git a/test/parallel/test-runner-output.mjs b/test/parallel/test-runner-output.mjs new file mode 100644 index 00000000000000..769c90c3ac7a9d --- /dev/null +++ b/test/parallel/test-runner-output.mjs @@ -0,0 +1,57 @@ +import * as common from '../common/index.mjs'; +import * as fixtures from '../common/fixtures.mjs'; +import * as snapshot from '../common/assertSnapshot.js'; +import { describe, it } from 'node:test'; + +function replaceTestDuration(str) { + return str + .replaceAll(/duration_ms: 0(\r?\n)/g, 'duration_ms: ZERO$1') + .replaceAll(/duration_ms: [0-9.]+/g, 'duration_ms: *') + .replaceAll(/duration_ms [0-9.]+/g, 'duration_ms *'); +} + +function replaceSpecDuration(str) { + return str + .replaceAll(/\(0(\r?\n)ms\)/g, '(ZEROms)') + .replaceAll(/[0-9.]+ms/g, '*ms') + .replaceAll(/duration_ms [0-9.]+/g, 'duration_ms *'); +} +const defaultTransform = snapshot + .transform(snapshot.replaceWindowsLineEndings, snapshot.replaceStackTrace, replaceTestDuration); + + +const tests = [ + { name: 'test-runner/output/abort.js' }, + { name: 'test-runner/output/abort_suite.js' }, + { name: 'test-runner/output/describe_it.js' }, + { name: 'test-runner/output/describe_nested.js' }, + { name: 'test-runner/output/hooks.js' }, + { name: 'test-runner/output/no_refs.js' }, + { name: 'test-runner/output/no_tests.js' }, + { name: 'test-runner/output/only_tests.js' }, + { name: 'test-runner/output/dot_reporter.js' }, + { + name: 'test-runner/output/spec_reporter_successful.js', + transform: snapshot.transform(snapshot.replaceWindowsLineEndings, snapshot.replaceStackTrace, replaceSpecDuration) + }, + { + name: 'test-runner/output/spec_reporter.js', + transform: snapshot.transform(snapshot.replaceWindowsLineEndings, snapshot.replaceStackTrace, replaceSpecDuration) + }, + { name: 'test-runner/output/output.js' }, + { name: 'test-runner/output/output_cli.js' }, + { name: 'test-runner/output/name_pattern.js' }, + { name: 'test-runner/output/name_pattern_with_only.js' }, + { name: 'test-runner/output/unresolved_promise.js' }, +].map(({ name, transform }) => ({ + name, + fn: common.mustCall(async () => { + await snapshot.spawnAndAssert(fixtures.path(name), transform ?? defaultTransform); + }), +})); + +describe('test runner output', { concurrency: true }, () => { + for (const { name, fn } of tests) { + it(name, fn); + } +});