feat: Align Test Explorer and Mocha Test Hierarchy Behavior (#21)

src/controller.ts

@@ -226,6 +226,7 @@ export class Controller {
           ? sourceMap.originalPositionFor(node.endLine, node.endColumn)
           : start;
       const file = last(this.getContainingItemsForFile(start.uri, { compiledFile: uri }))!.item!;
+      file.error = undefined;
       diagnosticCollection.delete(start.uri);
       newTestsInFile.set(node.name, add(file, node, start, end));
     }

src/runner.ts

@@ -55,6 +55,7 @@ export class TestRunner {
         [],
         request,
         run,
+        config,
       );
       if (run.token.isCancellationRequested) {
         return;
@@ -363,6 +364,7 @@ export class TestRunner {
     baseArgs: ReadonlyArray<string>,
     request: vscode.TestRunRequest,
     run: vscode.TestRun,
+    config: ConfigurationFile,
   ) {
     const reporter = path.resolve(__dirname, 'reporter', 'fullJsonStreamReporter.js');
     const args = [...baseArgs, '--reporter', reporter];
@@ -404,11 +406,15 @@ export class TestRunner {
       }
     }
 
-    // if there's no include, omit --run so that every file is executed
-    // TODO[mocha]: expose an "--include" variant which allows limiting the tests to individual files independent from the loaded config
+    // we specify explicitly our own which files to run
+    // we ignore first all files and then re-add them via --file
+    args.push('--ignore', '**/*.*');
+    const configDir = path.dirname(config.uri.fsPath);
     if (!request.include || !exclude.size) {
-      for (const path of compiledFileTests.value.keys()) {
-        args.push('--run', path);
+      for (const filePath of compiledFileTests.value.keys()) {
+        // use relative paths to reduce number of chars passed on
+        const relativePath = path.relative(configDir, filePath);
+        args.push('--file', relativePath);
       }
     }
 

src/test/integration/overlapping-tests.test.ts

@@ -0,0 +1,164 @@
+/**
+ * Copyright (C) Daniel Kuschny (Danielku15) and contributors.
+ * Copyright (C) Microsoft Corporation. All rights reserved.
+ *
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+
+import { expect } from 'chai';
+import * as vscode from 'vscode';
+import { TestState, captureTestRun, expectTestTree, findTestItem, getController } from '../util';
+
+describe('overlapping tests', () => {
+  it('discovers tests', async () => {
+    const c = await getController();
+
+    await expectTestTree(c, [
+      [
+        'folder',
+        [
+          ['suite01-duplicate.test.js', [['suite01', [['test01']]]]],
+          ['suite03.test.js', [['suite03', [['test01']]]]],
+        ],
+      ],
+      [
+        'suite01.test.js',
+        [
+          ['suite01', [['test01'], ['test02']]],
+          ['suite01.1', [['suite01.1']]],
+        ],
+      ],
+      [
+        'suite02.test.js',
+        [
+          ['suite02', [['test01'], ['test02']]],
+          ['suite02.1', [['test01'], ['test02']]],
+        ],
+      ],
+    ]);
+  });
+
+  it('runs all tests', async () => {
+    const c = await getController();
+    const profiles = c.profiles;
+    expect(profiles).to.have.lengthOf(2);
+
+    const run = await captureTestRun(
+      c,
+      new vscode.TestRunRequest(
+        undefined,
+        undefined,
+        profiles.find((p) => p.kind === vscode.TestRunProfileKind.Run),
+      ),
+    );
+
+    run.expectStates({
+      'suite01.test.js/suite01/test01': ['enqueued', 'started', 'passed'],
+      'suite01.test.js/suite01/test02': ['enqueued', 'started', 'passed'],
+      'suite01.test.js/suite01.1/suite01.1': ['enqueued', 'started', 'passed'],
+      'suite02.test.js/suite02/test01': ['enqueued', 'started', 'passed'],
+      'suite02.test.js/suite02/test02': ['enqueued', 'started', 'passed'],
+      'suite02.test.js/suite02.1/test01': ['enqueued', 'started', 'passed'],
+      'suite02.test.js/suite02.1/test02': ['enqueued', 'started', 'passed'],
+      'folder/suite03.test.js/suite03/test01': ['enqueued', 'started', 'passed'],
+      'folder/suite01-duplicate.test.js/suite01/test01': ['enqueued', 'started', 'passed'],
+    });
+  });
+
+  const testCases: [string, string[], { [id: string]: TestState[] }][] = [
+    [
+      'runs only test of specific file',
+      ['suite01.test.js/suite01/test01'],
+      {
+        'suite01.test.js/suite01/test01': ['enqueued', 'started', 'passed'],
+      },
+    ],
+    [
+      'runs only suite of specific file',
+      ['suite01.test.js/suite01'],
+      {
+        'suite01.test.js/suite01/test01': ['enqueued', 'started', 'passed'],
+        'suite01.test.js/suite01/test02': ['enqueued', 'started', 'passed'],
+      },
+    ],
+    [
+      'runs all suites in specific file',
+      ['suite01.test.js'],
+      {
+        'suite01.test.js/suite01/test01': ['enqueued', 'started', 'passed'],
+        'suite01.test.js/suite01/test02': ['enqueued', 'started', 'passed'],
+        'suite01.test.js/suite01.1/suite01.1': ['enqueued', 'started', 'passed'],
+      },
+    ],
+    [
+      'runs multiple files',
+      ['suite01.test.js', 'suite02.test.js'],
+      {
+        'suite01.test.js/suite01/test01': ['enqueued', 'started', 'passed'],
+        'suite01.test.js/suite01/test02': ['enqueued', 'started', 'passed'],
+        'suite01.test.js/suite01.1/suite01.1': ['enqueued', 'started', 'passed'],
+        'suite02.test.js/suite02/test01': ['enqueued', 'started', 'passed'],
+        'suite02.test.js/suite02/test02': ['enqueued', 'started', 'passed'],
+        'suite02.test.js/suite02.1/test01': ['enqueued', 'started', 'passed'],
+        'suite02.test.js/suite02.1/test02': ['enqueued', 'started', 'passed'],
+      },
+    ],
+    [
+      'runs mixed nodes',
+      [
+        'suite01.test.js/suite01',
+        'suite02.test.js/suite02/test01',
+        'suite02.test.js/suite02.1/test02',
+      ],
+      {
+        'suite01.test.js/suite01/test01': ['enqueued', 'started', 'passed'],
+        'suite01.test.js/suite01/test02': ['enqueued', 'started', 'passed'],
+        'suite02.test.js/suite02/test01': ['enqueued', 'started', 'passed'],
+        'suite02.test.js/suite02.1/test02': ['enqueued', 'started', 'passed'],
+      },
+    ],
+    [
+      'runs all in folder',
+      ['folder'],
+      {
+        'folder/suite03.test.js/suite03/test01': ['enqueued', 'started', 'passed'],
+        'folder/suite01-duplicate.test.js/suite01/test01': ['enqueued', 'started', 'passed'],
+      },
+    ],
+    [
+      'cannot differenciate overlapping suites',
+      ['suite01.test.js/suite01/test02', 'folder/suite01-duplicate.test.js/suite01/test01'],
+      {
+        // these tests are actually expected
+        'suite01.test.js/suite01/test02': ['enqueued', 'started', 'passed'],
+        'folder/suite01-duplicate.test.js/suite01/test01': ['enqueued', 'started', 'passed'],
+
+        // this test we cannot avoid to be executed due to name overlap, it's not really queued but still running
+        'suite01.test.js/suite01/test01': ['started', 'passed'],
+      },
+    ],
+  ];
+
+  for (const [name, include, expected] of testCases) {
+    it(name, async () => {
+      const c = await getController();
+      const profiles = c.profiles;
+      expect(profiles).to.have.lengthOf(2);
+
+      const testItems = include.map((i) => findTestItem(c.ctrl.items, i)!);
+
+      const run = await captureTestRun(
+        c,
+        new vscode.TestRunRequest(
+          testItems,
+          undefined,
+          profiles.find((p) => p.kind === vscode.TestRunProfileKind.Run),
+        ),
+      );
+
+      run.expectStates(expected);
+    });
+  }
+});

src/test/util.ts

@@ -19,6 +19,31 @@ import { getControllersForTestCommand } from '../constants';
 import type { Controller } from '../controller';
 import { IParsedNode, NodeKind } from '../discoverer/types';
 
+export function findTestItem(items: vscode.TestItemCollection, parts: string | string[]) {
+  if (!Array.isArray(parts)) {
+    parts = parts.split('/');
+  }
+
+  if (parts.length == 0) {
+    return undefined;
+  }
+
+  do {
+    const child = items.get(parts.shift()!);
+    if (!child) {
+      return undefined;
+    }
+
+    if (parts.length === 0) {
+      return child;
+    }
+
+    items = child.children;
+  } while (parts.length > 0);
+
+  return undefined;
+}
+
 export function source(...lines: string[]) {
   return lines.join('\n');
 }
@@ -162,7 +187,7 @@ async function rmrf(path: string) {
   }
 }
 
-type TestState = 'enqueued' | 'started' | 'skipped' | 'failed' | 'errored' | 'passed';
+export type TestState = 'enqueued' | 'started' | 'skipped' | 'failed' | 'errored' | 'passed';
 
 export class FakeTestRun implements vscode.TestRun {
   public output: { output: string; location?: vscode.Location; test?: vscode.TestItem }[] = [];

test-workspaces/overlapping-tests/.mocharc.js

@@ -0,0 +1,3 @@
+module.exports = {
+    spec: '**/*.test.js'
+};

test-workspaces/overlapping-tests/folder/suite01-duplicate.test.js

@@ -0,0 +1,6 @@
+describe('suite01', () => {
+    it('test01', () => {
+      console.log('suite01-duplicate/test01');
+    });
+  });
+

test-workspaces/overlapping-tests/folder/suite03.test.js

@@ -0,0 +1,6 @@
+describe('suite03', () => {
+    it('test01', () => {
+      console.log('suite03/test01');
+    });
+  });
+

test-workspaces/overlapping-tests/suite01.test.js

@@ -0,0 +1,14 @@
+describe('suite01', () => {
+  it('test01', () => {
+    console.log('suite01/test01');
+  });
+  it('test02', () => {
+    console.log('suite01/test02');
+  });
+});
+
+describe('suite01.1', () => {
+  it('suite01.1', () => {
+    console.log('suite01/test01');
+  });
+})

test-workspaces/overlapping-tests/suite02.test.js

@@ -0,0 +1,17 @@
+describe('suite02', () => {
+  it('test01', () => {
+    console.log('suite02/test01');
+  });
+  it('test02', () => {
+    console.log('suite02/test02');
+  });
+});
+
+describe('suite02.1', () => {
+  it('test01', () => {
+    console.log('suite02.1/test01');
+  });
+  it('test02', () => {
+    console.log('suite02.1/test02');
+  });
+});