test: refactor tests for easier isolated execution

Refactor tools/test.py to allow nested parallel/sequential/pummel tests.
Refactor tools/test.py to blacklist suites rather than whitelist.
Makefile now allows CI_JS_SUITES and CI_NATIVE_SUITES to be overridden.
Introduce special DEFAULT_JS_SUITES rather than maintaining list of suites.
Delete stale test-http-client-reconnect-bug.js test.

CONTRIBUTING.md

@@ -32,6 +32,7 @@ expect throughout each step of the process.
       * [Commit message guidelines](#commit-message-guidelines)
     * [Step 5: Rebase](#step-5-rebase)
     * [Step 6: Test](#step-6-test)
+    * [Step 6.5: Test Coverage](#step-65-test-coverage)
     * [Step 7: Push](#step-7-push)
     * [Step 8: Opening the Pull Request](#step-8-opening-the-pull-request)
     * [Step 9: Discuss and Update](#step-9-discuss-and-update)
@@ -371,10 +372,16 @@ Bug fixes and features should always come with tests. A
 provided to make the process easier. Looking at other tests to see how they
 should be structured can also help.
 
-The `test` directory within the `nodejs/node` repository is complex and it is
-often not clear where a new test file should go. When in doubt, add new tests
-to the `test/parallel/` directory and the right location will be sorted out
-later.
+Tests should be placed in a directory corresponding to the subsystem that
+they exercise, e.g., `es-module` for ESM loader tests, `child-process` for
+tests related to the `child-process` module. Tests can be further organized
+into the following subtypes:
+
+* `subsystem/parallel`: tests that can be run in parallel with each other.
+  Most new tests should be placed in this location.
+* `subsystem/pummel`: tests that should be run while the system is under
+  heavy load.
+* `subsystem/sequential`: tests that should be run sequentially.
 
 Before submitting your changes in a Pull Request, always run the full Node.js
 test suite. To run the tests (including code linting) on Unix / macOS:
@@ -420,6 +427,31 @@ $ ./node ./test/parallel/test-stream2-transform.js
 Remember to recompile with `make -j4` in between test runs if you change code in
 the `lib` or `src` directories.
 
+#### Step 6.5: Test coverage
+
+A great way to find nooks and crannies of the codebase that could use more
+testing is to use coverage reports. To generate a coverage report simply
+run:
+
+```console
+$ ./configure --coverage
+$ make coverage
+```
+
+Afterwards a detailed HTML coverage report can be found in
+`coverage/index.html`.
+
+It is also possible to generate coverage for a specific subsystem. For instance,
+the following generates coverage for the `child-process` submodule:
+
+```console
+$ ./configure --coverage
+$ CI_JS_SUITES=child-process* CI_NATIVE_SUITES= make coverage
+```
+
+recent coverage reports can be found at
+[coverage.nodejs.org](https://coverage.nodejs.org/).
+
 #### Step 7: Push
 
 Once you are sure your commits are ready to go, with passing tests and linting,

Makefile

@@ -333,8 +333,8 @@ test-all: test-build test/gc/build/Release/binding.node
 test-all-valgrind: test-build
 	$(PYTHON) tools/test.py --mode=debug,release --valgrind
 
-CI_NATIVE_SUITES := addons addons-napi
-CI_JS_SUITES := abort async-hooks doctool inspector known_issues message parallel pseudo-tty sequential
+CI_NATIVE_SUITES ?= addons addons-napi
+CI_JS_SUITES ?= DEFAULT_JS_SUITES
 
 # Build and test addons without building anything else
 test-ci-native: LOGLEVEL := info

test/parallel/test-child-process-bad-stdio.js → test/child-process/parallel/test-child-process-bad-stdio.js

@@ -1,6 +1,6 @@
 'use strict';
 // Flags: --expose_internals
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const cp = require('child_process');
 

test/parallel/test-child-process-buffering.js → test/child-process/parallel/test-child-process-buffering.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 
 function pwd(callback) {

test/parallel/test-child-process-constructor.js → test/child-process/parallel/test-child-process-constructor.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const { ChildProcess } = require('child_process');
 assert.strictEqual(typeof ChildProcess, 'function');

test/parallel/test-child-process-custom-fds.js → test/child-process/parallel/test-child-process-custom-fds.js

@@ -1,6 +1,6 @@
 // Flags: --expose_internals
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const internalCp = require('internal/child_process');
 const oldSpawnSync = internalCp.spawnSync;

test/parallel/test-child-process-cwd.js → test/child-process/parallel/test-child-process-cwd.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 
 let returns = 0;

test/parallel/test-child-process-default-options.js → test/child-process/parallel/test-child-process-default-options.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 
 const spawn = require('child_process').spawn;

test/parallel/test-child-process-detached.js → test/child-process/parallel/test-child-process-detached.js

@@ -20,9 +20,9 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-require('../common');
+require('../../common');
 const assert = require('assert');
-const fixtures = require('../common/fixtures');
+const fixtures = require('../../common/fixtures');
 
 const spawn = require('child_process').spawn;
 const childPath = fixtures.path('parent-process-nonpersistent.js');

test/parallel/test-child-process-disconnect.js → test/child-process/parallel/test-child-process-disconnect.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const fork = require('child_process').fork;
 const net = require('net');

test/parallel/test-child-process-double-pipe.js → test/child-process/parallel/test-child-process-double-pipe.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const os = require('os');
 const util = require('util');

test/parallel/test-child-process-env.js → test/child-process/parallel/test-child-process-env.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 
 const spawn = require('child_process').spawn;

test/parallel/test-child-process-exec-cwd.js → test/child-process/parallel/test-child-process-exec-cwd.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const exec = require('child_process').exec;
 

test/parallel/test-child-process-exec-encoding.js → test/child-process/parallel/test-child-process-exec-encoding.js

@@ -1,5 +1,5 @@
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const cp = require('child_process');
 const stdoutData = 'foo';

test/parallel/test-child-process-exec-env.js → test/child-process/parallel/test-child-process-exec-env.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const exec = require('child_process').exec;
 let success_count = 0;

test/parallel/test-child-process-exec-error.js → test/child-process/parallel/test-child-process-exec-error.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const child_process = require('child_process');
 

test/parallel/test-child-process-exec-kill-throws.js → test/child-process/parallel/test-child-process-exec-kill-throws.js

@@ -1,6 +1,6 @@
 'use strict';
 // Flags: --expose_internals
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const cp = require('child_process');
 

test/parallel/test-child-process-exec-maxBuffer.js → test/child-process/parallel/test-child-process-exec-maxBuffer.js

@@ -1,5 +1,5 @@
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const cp = require('child_process');
 

test/parallel/test-child-process-exec-stdout-stderr-data-string.js → test/child-process/parallel/test-child-process-exec-stdout-stderr-data-string.js

@@ -1,6 +1,6 @@
 'use strict';
 // Refs: https://github.com/nodejs/node/issues/7342
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const exec = require('child_process').exec;
 

test/parallel/test-child-process-exec-timeout.js → test/child-process/parallel/test-child-process-exec-timeout.js

@@ -1,5 +1,5 @@
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const cp = require('child_process');
 

test/parallel/test-child-process-execfile.js → test/child-process/parallel/test-child-process-execfile.js

@@ -1,9 +1,9 @@
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const execFile = require('child_process').execFile;
 const uv = process.binding('uv');
-const fixtures = require('../common/fixtures');
+const fixtures = require('../../common/fixtures');
 
 const fixture = fixtures.path('exit.js');
 

test/parallel/test-child-process-exit-code.js → test/child-process/parallel/test-child-process-exit-code.js

@@ -20,10 +20,10 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const spawn = require('child_process').spawn;
-const fixtures = require('../common/fixtures');
+const fixtures = require('../../common/fixtures');
 
 const exitScript = fixtures.path('exit.js');
 const exitChild = spawn(process.argv[0], [exitScript, 23]);

test/parallel/test-child-process-flush-stdio.js → test/child-process/parallel/test-child-process-flush-stdio.js

@@ -1,5 +1,5 @@
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const cp = require('child_process');
 const assert = require('assert');
 

test/parallel/test-child-process-fork-and-spawn.js → test/child-process/parallel/test-child-process-fork-and-spawn.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const { fork, spawn } = require('child_process');
 

test/parallel/test-child-process-fork-close.js → test/child-process/parallel/test-child-process-fork-close.js

@@ -20,10 +20,10 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const fork = require('child_process').fork;
-const fixtures = require('../common/fixtures');
+const fixtures = require('../../common/fixtures');
 
 const cp = fork(fixtures.path('child-process-message-and-exit.js'));
 

test/parallel/test-child-process-fork-dgram.js → test/child-process/parallel/test-child-process-fork-dgram.js

@@ -27,7 +27,7 @@
  * child process.
  */
 
-const common = require('../common');
+const common = require('../../common');
 if (common.isWindows)
   common.skip('Sending dgram sockets to child processes is not supported');
 

test/parallel/test-child-process-fork-exec-argv.js → test/child-process/parallel/test-child-process-fork-exec-argv.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-require('../common');
+require('../../common');
 const assert = require('assert');
 const child_process = require('child_process');
 const spawn = child_process.spawn;

test/parallel/test-child-process-fork-exec-path.js → test/child-process/parallel/test-child-process-fork-exec-path.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const fs = require('fs');
 const path = require('path');

test/parallel/test-child-process-fork-net.js → test/child-process/parallel/test-child-process-fork-net.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const fork = require('child_process').fork;
 const net = require('net');

test/parallel/test-child-process-fork-net2.js → test/child-process/parallel/test-child-process-fork-net2.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const fork = require('child_process').fork;
 const net = require('net');

test/parallel/test-child-process-fork-no-shell.js → test/child-process/parallel/test-child-process-fork-no-shell.js

@@ -1,6 +1,6 @@
 'use strict';
 // This test verifies that the shell option is not supported by fork().
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const cp = require('child_process');
 const expected = common.isWindows ? '%foo%' : '$foo';

test/parallel/test-child-process-fork-ref.js → test/child-process/parallel/test-child-process-fork-ref.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-require('../common');
+require('../../common');
 const assert = require('assert');
 const fork = require('child_process').fork;
 

test/parallel/test-child-process-fork-ref2.js → test/child-process/parallel/test-child-process-fork-ref2.js

@@ -20,7 +20,7 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const fork = require('child_process').fork;
 
 if (process.argv[2] === 'child') {

test/parallel/test-child-process-fork-regr-gh-2847.js → test/child-process/parallel/test-child-process-fork-regr-gh-2847.js

@@ -2,7 +2,7 @@
 // (asynchronously) to use the closed channel to it's creator caused a segfault.
 'use strict';
 
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 
 const cluster = require('cluster');

test/parallel/test-child-process-fork-stdio-string-variant.js → test/child-process/parallel/test-child-process-fork-stdio-string-variant.js

@@ -1,13 +1,13 @@
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 
 // Ensures that child_process.fork can accept string
 // variant of stdio parameter in options object and
 // throws a TypeError when given an unexpected string
 
 const assert = require('assert');
 const fork = require('child_process').fork;
-const fixtures = require('../common/fixtures');
+const fixtures = require('../../common/fixtures');
 
 const childScript = fixtures.path('child-process-spawn-node');
 const errorRegexp = /^TypeError: Incorrect value of stdio option:/;

test/parallel/test-child-process-fork-stdio.js → test/child-process/parallel/test-child-process-fork-stdio.js

@@ -1,5 +1,5 @@
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const cp = require('child_process');
 const net = require('net');

test/parallel/test-child-process-fork.js → test/child-process/parallel/test-child-process-fork.js

@@ -20,11 +20,11 @@
 // USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 'use strict';
-const common = require('../common');
+const common = require('../../common');
 const assert = require('assert');
 const fork = require('child_process').fork;
 const args = ['foo', 'bar'];
-const fixtures = require('../common/fixtures');
+const fixtures = require('../../common/fixtures');
 
 const n = fork(fixtures.path('child-process-spawn-node.js'), args);