chore: improve the test reporter's messaging

v-next/example-project/hardhat.config.ts

@@ -122,15 +122,15 @@ const config: HardhatUserConfig = {
   ],
   plugins: [
     pluginExample,
-    HardhatMochaTestRunner,
+    // HardhatMochaTestRunner,
     // if testing node plugin, use the following line instead
-    // HardhatNodeTestRunner,
+    HardhatNodeTestRunner,
     viemScketchPlugin,
   ],
   paths: {
-    tests: "test/mocha",
+    // tests: "test/mocha",
     // if testing node plugin, use the following line instead
-    // tests: "test/node",
+    tests: "test/node",
   },
 };
 

v-next/example-project/test/node/other-example-test.js

@@ -3,8 +3,12 @@ import { describe, it } from "node:test";
 
 import hre from "@ignored/hardhat-vnext";
 
-describe("Other example test", () => {
+describe("Other example test", async () => {
+  const taskName = await new Promise((resolve) =>
+    setTimeout(resolve, "example"),
+  );
+
   it("should have the example task", () => {
-    assert.ok(hre.tasks.getTask("example"));
+    assert.ok(hre.tasks.getTask(taskName));
   });
 });

v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/async-describe-test/result.txt

@@ -1,42 +1,33 @@
-
-  1) should pass 1
-
   nested async describe with promise 1
-    2) should pass 2
+    1) should pass 2
 
-  2) nested async describe with promise 1
+  1) nested async describe with promise 1
 
   nested async describe 1
-    3) should pass 3
+    2) should pass 3
 
-  3) nested async describe 1
+  2) nested async describe 1
 
   nested describe 1
-    4) should pass 4
+    3) should pass 4
 
-  4) nested describe 1
+  3) nested describe 1
 
 
-0 passing (128ms)
-1 failing
+0 passing (131ms)
 3 cancelled
 
-  1) should pass 1:
-
-   Error: test could not be started because its parent finished
-       at SuiteContext.<anonymous> (integration-tests/fixture-tests/async-describe-test/test.ts:1:193)
-
-  2) nested async describe with promise 1:
+  1) nested async describe with promise 1:
 
    Error: test could not be started because its parent finished
        at SuiteContext.<anonymous> (integration-tests/fixture-tests/async-describe-test/test.ts:1:244)
 
-  3) nested async describe 1:
+  2) nested async describe 1:
 
    Error: test could not be started because its parent finished
        at SuiteContext.<anonymous> (integration-tests/fixture-tests/async-describe-test/test.ts:1:422)
 
-  4) nested describe 1:
+  3) nested describe 1:
 
    Error: test could not be started because its parent finished
        at SuiteContext.<anonymous> (integration-tests/fixture-tests/async-describe-test/test.ts:1:520)

v-next/hardhat-node-test-reporter/integration-tests/index.ts

@@ -10,7 +10,7 @@ import { run } from "node:test";
 
 import { diff } from "jest-diff";
 
-import reporter from "../src/reporter.js";
+import { hardhatTestReporter } from "../src/reporter.js";
 
 let SHOW_OUTPUT = false;
 const testOnly: string[] = [];
@@ -72,13 +72,14 @@ for (const entry of entries) {
       options = JSON.parse(readFileSync(optionsPath, "utf8"));
     }
 
+    options = { ...options, files: testFiles };
+
+    const reporter = hardhatTestReporter(options);
+
     // We disable github actions annotations, as they are misleading on PRs
     // otherwise.
     process.env.NO_GITHUB_ACTIONS_ANNOTATIONS = "true";
-    const reporterStream = run({
-      ...options,
-      files: testFiles,
-    }).compose(reporter);
+    const reporterStream = run(options).compose(reporter);
 
     for await (const chunk of reporterStream) {
       outputChunks.push(chunk);

v-next/hardhat-node-test-reporter/src/node-test-error-utils.ts

@@ -11,6 +11,23 @@ export function isSubtestFailedError(error: Error): boolean {
   );
 }
 
+/**
+ * Returns true if the error represents that a test/suite was cancelled because
+ * the parent already finished.
+ *
+ * This typically happens if you are running in `only` and the parent is async,
+ * as the parent gets cancelled but continues running an async operation, and
+ * try to create a subtests after it.
+ */
+export function isParentAlreadyFinishedError(error: Error): boolean {
+  return (
+    "code" in error &&
+    "failureType" in error &&
+    error.code === "ERR_TEST_FAILURE" &&
+    error.failureType === "parentAlreadyFinished"
+  );
+}
+
 /**
  * Returns true if the error represents that a test was cancelled because its
  * parent failed.

v-next/hardhat-node-test-reporter/src/reporter.ts

@@ -4,6 +4,7 @@ import type {
   TestEventSource,
   TestReporter,
   TestReporterResult,
+  TestRunOptions,
 } from "./types.js";
 
 import chalk from "chalk";
@@ -22,6 +23,7 @@ import {
 import { annotatePR } from "./github-actions.js";
 import {
   isCancelledByParentError,
+  isParentAlreadyFinishedError,
   isSubtestFailedError,
 } from "./node-test-error-utils.js";
 
@@ -47,11 +49,16 @@ export interface HardhatTestReporterConfig {
  * @param source
  */
 
-const customReporter: TestReporter = hardhatTestReporter();
-
+const customReporter: TestReporter = hardhatTestReporter(getTestRunOptions());
 export default customReporter;
 
+function getTestRunOptions(): TestRunOptions {
+  const only = process.execArgv.includes("--test-only");
+  return { only };
+}
+
 export function hardhatTestReporter(
+  options: TestRunOptions,
   config: HardhatTestReporterConfig = {},
 ): TestReporter {
   return async function* (source: TestEventSource): TestReporterResult {
@@ -102,20 +109,32 @@ export function hardhatTestReporter(
     const preFormattedFailureReasons: string[] = [];
 
     let topLevelFilePassCount = 0;
+    let parentAlreadyFinishedFailureCount = 0;
 
     for await (const event of source) {
       switch (event.type) {
         case "test:diagnostic": {
-          // We need to subtract the number of top-level file passes from the
-          // total number of tests/passes, as we are not printing them.
-          if (
-            event.data.message.startsWith("tests") ||
-            event.data.message.startsWith("pass")
-          ) {
+          // We need to subtract the number of tests/suites that we chose not to
+          // display from the summary
+          if (event.data.message.startsWith("tests")) {
+            const parts = event.data.message.split(" ");
+            const count =
+              parseInt(parts[1], 10) -
+              topLevelFilePassCount -
+              parentAlreadyFinishedFailureCount;
+            event.data.message = `${parts[0]} ${count}`;
+          }
+          if (event.data.message.startsWith("pass")) {
             const parts = event.data.message.split(" ");
             const count = parseInt(parts[1], 10) - topLevelFilePassCount;
             event.data.message = `${parts[0]} ${count}`;
           }
+          if (event.data.message.startsWith("fail")) {
+            const parts = event.data.message.split(" ");
+            const count =
+              parseInt(parts[1], 10) - parentAlreadyFinishedFailureCount;
+            event.data.message = `${parts[0]} ${count}`;
+          }
           diagnostics.push(event.data);
           break;
         }
@@ -138,6 +157,22 @@ export function hardhatTestReporter(
             break;
           }
 
+          // We do not want to display failures caused by the parent already
+          // having finished when running with the --test-only flag because
+          // they are false negatives.
+          // We still display the failed suites, because failures for the nested
+          // tests (cancelled by parent errors) were already displayed.
+          if (
+            options.only === true &&
+            event.data.details.type !== "suite" &&
+            event.type === "test:fail" &&
+            isParentAlreadyFinishedError(event.data.details.error)
+          ) {
+            parentAlreadyFinishedFailureCount++;
+            stack.pop();
+            break;
+          }
+
           if (event.data.details.type === "suite") {
             // If a suite failed for a reason other than a subtest failing, we
             // want to print its failure.

v-next/hardhat-node-test-reporter/src/types.ts

@@ -1,5 +1,8 @@
+import type { run } from "node:test";
 import type { TestEvent } from "node:test/reporters";
 
+export type TestRunOptions = NonNullable<Parameters<typeof run>[0]>;
+
 /**
  * A map from event type to its data type.
  */

v-next/hardhat-node-test-runner/src/task-action.ts

@@ -69,7 +69,9 @@ const testWithHardhat: NewTaskActionFunction<TestActionArguments> = async (
 
     const testOnlyMessage =
       "'only' and 'runOnly' require the --only command-line option.";
-    const customReporter = hardhatTestReporter({ testOnlyMessage });
+    const customReporter = hardhatTestReporter(nodeTestOptions, {
+      testOnlyMessage,
+    });
 
     const reporterStream = run({ files, only })
       .on("test:fail", (event) => {