From d9bca644ff946e5cefe52cc23f358f971c4e68b1 Mon Sep 17 00:00:00 2001 From: galargh Date: Thu, 5 Sep 2024 14:26:25 +0200 Subject: [PATCH] chore: report on errors in before and describe during the test run --- .../fixture-tests/example-test/result.svg | 154 +++--- .../fixture-tests/example-test/result.txt | 6 +- .../fixture-tests/hooks-test/before.ts | 16 + .../fixture-tests/hooks-test/describe.ts | 14 + .../fixture-tests/hooks-test/result.svg | 522 ++++++++++++++---- .../fixture-tests/hooks-test/result.txt | 76 ++- .../src/reporter.ts | 18 +- 7 files changed, 625 insertions(+), 181 deletions(-) diff --git a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/result.svg b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/result.svg index f6699068b1..a15e35857f 100644 --- a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/result.svg +++ b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/result.svg @@ -1,6 +1,6 @@ - Qt Svg Document Generated with Qt @@ -11,14 +11,14 @@ - - + + - + + + + +4) in describe + + + + testing before each - nested - -✔ - neseted foo - -✔ - neseted foo 2 - -+ todo test - -- skipped test -6 passing - (136ms) + (133ms) -2 failing -1 skipped -1 todo -1 cancelled - 1) Foooo - test with cause: test with cause: + -Error: withCause - - at TestContext.<anonymous> (integration-tests/fixture-tests/example-test/test.ts:7:11) - -[cause]: Error: cause - - at TestContext.<anonymous> (integration-tests/fixture-tests/example-test/test.ts:7:43) - 2) Foooo - child - asdasd: asdasd: + -Error: Different arrays - -- Expected - -+ Received - - Array [ - 1, - -- 2, - -+ 3, - 3, - ] - - - at TestContext.<anonymous> (integration-tests/fixture-tests/example-test/test.ts:12:21) - -[cause]: Error: cause - - at TestContext.<anonymous> (integration-tests/fixture-tests/example-test/test.ts:13:16) - 3) in describe - foo: foo: + -Test cancelled by parent error - - This test was cancelled due to an error in its parent suite/it or test/it, or in one of its before/beforeEach + This test was cancelled due to an error in its parent describe or before hook - 4) in describe: 4) in describe: + -Error: before - - at SuiteContext.<anonymous> (integration-tests/fixture-tests/example-test/test.ts:51:11) diff --git a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/result.txt b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/result.txt index 11612ec2e5..d3d2c748aa 100644 --- a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/result.txt +++ b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/result.txt @@ -19,6 +19,8 @@ in describe 3) foo + 4) in describe + testing before each nested ✔ neseted foo @@ -30,7 +32,7 @@ - skipped test -6 passing (136ms) +6 passing (133ms) 2 failing 1 skipped 1 todo @@ -67,7 +69,7 @@ foo: Test cancelled by parent error -  This test was cancelled due to an error in its parent suite/it or test/it, or in one of its before/beforeEach +  This test was cancelled due to an error in its parent describe or before hook 4) in describe: diff --git a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/before.ts b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/before.ts index 7c2bd7fe12..8b2962e8eb 100644 --- a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/before.ts +++ b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/before.ts @@ -14,3 +14,19 @@ describe("before", () => { assert.equal(1, 1); }); }); + +describe("nested before", () => { + describe("level 1", () => { + before(() => { + throw new Error("before hook error"); + }); + + it("should pass", async () => { + assert.equal(1, 1); + }); + + it("should pass, too", async () => { + assert.equal(1, 1); + }); + }); +}); diff --git a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/describe.ts b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/describe.ts index 656bdaa96a..d9f85c3fcc 100644 --- a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/describe.ts +++ b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/describe.ts @@ -12,3 +12,17 @@ describe("describe", () => { throw new Error("describe setup error"); }); + +describe("nested describe", () => { + describe("level 1", () => { + it("should pass", async () => { + assert.equal(1, 1); + }); + + it("should pass, too", async () => { + assert.equal(1, 1); + }); + + throw new Error("describe setup error"); + }); +}); diff --git a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/result.svg b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/result.svg index eb096481bc..1a22a575e0 100644 --- a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/result.svg +++ b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/result.svg @@ -1,6 +1,6 @@ - Qt Svg Document Generated with Qt @@ -11,14 +11,14 @@ - - + + - + + + + +3) after + + + + before each - -4) should pass - -5) should pass, too - before - -6) should pass - -7) should pass, too - + + + +8) before + + + + nested before + level 1 + + + + +9) should pass + + + + + + + +10) should pass, too + + + + + + + +11) level 1 + + + + describe - -9) should pass +12) should pass - -10) should pass, too +13) should pass, too + + + + + + + +14) describe + + + + nested describe + level 1 + + + + +15) should pass + + + + + + + +16) should pass, too + + + + + + + +17) level 1 -2 passing - (143ms) + (144ms) -4 failing -4 cancelled +8 cancelled - 1) after each - should pass: - -Error: after each hook error - - at TestContext.<anonymous> (integration-tests/fixture-tests/hooks-test/after-each.ts:6:11) - 2) after each - should pass, too: - -Error: after each hook error - - at TestContext.<anonymous> (integration-tests/fixture-tests/hooks-test/after-each.ts:6:11) - 3) after: - -Error: after hook error - - at SuiteContext.<anonymous> (integration-tests/fixture-tests/hooks-test/after.ts:6:11) - 4) before each - should pass: - -Error: before each hook error - - at TestContext.<anonymous> (integration-tests/fixture-tests/hooks-test/before-each.ts:6:11) - 5) before each - should pass, too: - -Error: before each hook error - - at TestContext.<anonymous> (integration-tests/fixture-tests/hooks-test/before-each.ts:6:11) - 6) before - should pass: - -Test cancelled by parent error - - This test was cancelled due to an error in its parent describe or before hook - 7) before - should pass, too: - -Test cancelled by parent error - - This test was cancelled due to an error in its parent describe or before hook - 8) before: - -Error: before hook error - - at SuiteContext.<anonymous> (integration-tests/fixture-tests/hooks-test/before.ts:6:11) - 9) describe - should pass: - 9) nested before + level 1 + should pass: + -Test cancelled by parent error - - This test was cancelled due to an error in its parent describe or before hook - 10) describe - 10) nested before + level 1 + should pass, too: + + + + +Test cancelled by parent error + + + + + + + + This test was cancelled due to an error in its parent describe or before hook + + + + 11) nested before + level 1: + + + + +Error: before hook error + + + + + + + + at SuiteContext.<anonymous> (integration-tests/fixture-tests/hooks-test/before.ts:21:13) + + + + 12) describe + should pass: + + + + +Test cancelled by parent error + + + + + + + + This test was cancelled due to an error in its parent describe or before hook + + + + 13) describe + should pass, too: - -Test cancelled by parent error - - This test was cancelled due to an error in its parent describe or before hook - 11) describe: - 14) describe: + -Error: describe setup error - - at SuiteContext.<anonymous> (integration-tests/fixture-tests/hooks-test/describe.ts:13:9) + + + 15) nested describe + level 1 + should pass: + + + + +Test cancelled by parent error + + + + + + + + This test was cancelled due to an error in its parent describe or before hook + + + + 16) nested describe + level 1 + should pass, too: + + + + +Test cancelled by parent error + + + + + + + + This test was cancelled due to an error in its parent describe or before hook + + + + 17) nested describe + level 1: + + + + +Error: describe setup error + + + + + + + + at SuiteContext.<anonymous> (integration-tests/fixture-tests/hooks-test/describe.ts:26:11) + diff --git a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/result.txt b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/result.txt index 2f7c8ca160..3c830746fc 100644 --- a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/result.txt +++ b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/hooks-test/result.txt @@ -6,6 +6,8 @@ ✔ should pass ✔ should pass, too + 3) after + before each 4) should pass 5) should pass, too @@ -14,14 +16,32 @@ 6) should pass 7) should pass, too + 8) before + + nested before + level 1 + 9) should pass + 10) should pass, too + + 11) level 1 + describe - 9) should pass - 10) should pass, too + 12) should pass + 13) should pass, too + 14) describe -2 passing (143ms) + nested describe + level 1 + 15) should pass + 16) should pass, too + + 17) level 1 + + +2 passing (144ms) 4 failing -4 cancelled +8 cancelled 1) after each should pass: @@ -69,20 +89,60 @@ Error: before hook error  at SuiteContext. (integration-tests/fixture-tests/hooks-test/before.ts:6:11) - 9) describe - should pass: + 9) nested before + level 1 + should pass: + + Test cancelled by parent error +  This test was cancelled due to an error in its parent describe or before hook + + 10) nested before + level 1 + should pass, too: + + Test cancelled by parent error +  This test was cancelled due to an error in its parent describe or before hook + + 11) nested before + level 1: + + Error: before hook error +  at SuiteContext. (integration-tests/fixture-tests/hooks-test/before.ts:21:13) + + 12) describe + should pass: Test cancelled by parent error  This test was cancelled due to an error in its parent describe or before hook - 10) describe + 13) describe should pass, too: Test cancelled by parent error  This test was cancelled due to an error in its parent describe or before hook - 11) describe: + 14) describe: Error: describe setup error  at SuiteContext. (integration-tests/fixture-tests/hooks-test/describe.ts:13:9) + 15) nested describe + level 1 + should pass: + + Test cancelled by parent error +  This test was cancelled due to an error in its parent describe or before hook + + 16) nested describe + level 1 + should pass, too: + + Test cancelled by parent error +  This test was cancelled due to an error in its parent describe or before hook + + 17) nested describe + level 1: + + Error: describe setup error +  at SuiteContext. (integration-tests/fixture-tests/hooks-test/describe.ts:26:11) + diff --git a/v-next/hardhat-node-test-reporter/src/reporter.ts b/v-next/hardhat-node-test-reporter/src/reporter.ts index 058f847cb9..347044f20a 100644 --- a/v-next/hardhat-node-test-reporter/src/reporter.ts +++ b/v-next/hardhat-node-test-reporter/src/reporter.ts @@ -103,15 +103,19 @@ export default async function* customReporter( // want to print its failure. if (event.type === "test:fail") { if (!isSubtestFailedError(event.data.details.error)) { + const failure: Failure = { + index: preFormattedFailureReasons.length, + testFail: event.data, + contextStack: stack, + }; + + // We format the failure reason and store it in an array, so that we + // can output it at the end. + preFormattedFailureReasons.push(formatFailureReason(failure)); + await annotatePR(event.data); - preFormattedFailureReasons.push( - formatFailureReason({ - index: preFormattedFailureReasons.length, - testFail: event.data, - contextStack: stack, - }), - ); + yield `\n${formatTestFailure(failure)}\n`; } }