Monorepo: Restore and add test coverage config (#39961)

* Enable and normalize test coverage commands * Revert "Code Coverage: remove automation (#28542)" This reverts commit da5e788. * Revert "Remove remnants of automated code coverage support (#28982)" This reverts commit 8cc3246. * Fix starter plugin test broken by #31417 * Plugin tests are now run in the monorepo rather than wp-content/plugins. * Removed undocumented WP_DEVELOP_DIR variable, documented WORDPRESS_DEVELOP_DIR and WORDPRESS_DIR * Fixed JS tests in some packages where .scripts.test-js had never been added to composer.json. --------- Co-authored-by: Brad Jorsch <brad.jorsch@automattic.com>
Automattic · Oct 31, 2024 · 7fadf4a · 7fadf4a
1 parent 5d2bab4
commit 7fadf4a
Show file tree

Hide file tree

Showing 306 changed files with 2,897 additions and 567 deletions.
diff --git a/.github/actions/tool-setup/action.yml b/.github/actions/tool-setup/action.yml
@@ -3,6 +3,9 @@ description: "Setup needed versions of PHP, Node, and Pnpm."
 inputs:
   php:
     description: "Override the PHP version, or set 'false' to skip setting up PHP."
+  coverage:
+    description: "Set the PHP coverage driver, e.g. 'none' or 'pcov'."
+    default: 'none'
   node:
     description: "Override the Node version, or set 'false' to skip setting up Node and Pnpm."
 runs:
@@ -62,7 +65,7 @@ runs:
         ini-values: error_reporting=E_ALL, display_errors=On, zend.assertions=1
         tools: composer:${{ steps.versions.outputs.composer-version }}
         extensions: mysql, imagick
-        coverage: none
+        coverage: ${{ inputs.coverage }}
 
     - name: Configure composer
       if: steps.versions.outputs.php-version != 'false'

diff --git a/.github/files/generate-ci-matrix.php b/.github/files/generate-ci-matrix.php
@@ -120,6 +120,14 @@
 	'timeout' => 15, // 2021-01-18: Successful runs seem to take ~5 minutes.
 );
 
+// Add Coverage tests.
+$matrix[] = array(
+	'name'    => 'Code coverage',
+	'script'  => 'test-coverage',
+	'wp'      => 'latest',
+	'timeout' => 40, // 2024-10-30: Successful runs seem to take ~30 minutes. We'll need to improve that.
+);
+
 // END matrix definitions.
 // Now, validation.
 

diff --git a/.github/files/process-coverage.sh b/.github/files/process-coverage.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -eo pipefail
+
+EXIT=0
+
+[[ -d coverage ]] && find coverage -type d -empty -delete
+if [[ ! -d coverage ]]; then
+	echo 'No coverage was generated.'
+	exit $EXIT
+fi
+
+echo '::group::Copy coverage into artifacts'
+tar --owner=0 --group=0 --xz -cvvf artifacts/coverage.tar.xz coverage
+echo '::endgroup::'
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -55,7 +55,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          # codecov.io requires a fetch depth > 1.
+          # Test coverage checks require a fetch depth > 1.
           fetch-depth: 2
 
       # For pull requests, list-changed-projects.sh needs the merge base.
@@ -70,6 +70,7 @@ jobs:
         uses: ./.github/actions/tool-setup
         with:
           php: ${{ matrix.php }}
+          coverage: ${{ matrix.script == 'test-coverage' && 'pcov' || 'none' }}
           node: ${{ matrix.node }}
 
       - name: Monorepo install
@@ -79,7 +80,7 @@ jobs:
           echo "::endgroup::"
 
           # If we're going to be making WorDBless use WP "nightlies", remove the relevant package from Composer's cache to get the latest version.
-          if [[ "$WP_BRANCH" == 'trunk' && "$TEST_SCRIPT" == "test-php" ]]; then
+          if [[ "$WP_BRANCH" == 'trunk' && ( "$TEST_SCRIPT" == "test-php" || "$TEST_SCRIPT" == "test-coverage" ) ]]; then
             echo "::group::Clear composer cache for roots/wordpress"
             DIR=$(composer config cache-files-dir)
             rm -rf "$DIR/roots/wordpress" "$DIR/roots/wordpress-no-content"
@@ -119,6 +120,7 @@ jobs:
         run: |
           EXIT=0
           mkdir artifacts
+          [[ "$TEST_SCRIPT" == "test-coverage" ]] && mkdir coverage
           for P in composer.json projects/*/*/composer.json; do
             if [[ "$P" == "composer.json" ]]; then
               DIR="."
@@ -128,12 +130,7 @@ jobs:
               SLUG="${DIR#projects/}"
             fi
 
-            if [[ "${SLUG%%/*}" == "plugins" ]]; then
-              # Plugins run from the WordPress checkout, not the monorepo checkout.
-              if [[ -d "/tmp/wordpress-$WP_BRANCH/src/wp-content/$SLUG" ]]; then
-                DIR="/tmp/wordpress-$WP_BRANCH/src/wp-content/$SLUG"
-              fi
-            elif [[ "$WP_BRANCH" != 'latest' && "$WP_BRANCH" != 'none' && "$FORCE_PACKAGE_TESTS" != "true" ]]; then
+            if [[ "${SLUG%%/*}" != "plugins" && "$WP_BRANCH" != 'latest' && "$WP_BRANCH" != 'none' && "$FORCE_PACKAGE_TESTS" != "true" ]]; then
               echo "Skipping $SLUG, only plugins run for WP_BRANCH = $WP_BRANCH"
               continue
             fi
@@ -160,7 +157,7 @@ jobs:
               echo "::group::Running tests for $SLUG"
 
               # Composer install, if appropriate. Note setup-wordpress-env.sh did it already for plugins.
-              if [[ "${SLUG%%/*}" != "plugins" && "$TEST_SCRIPT" == "test-php" ]]; then
+              if [[ "${SLUG%%/*}" != "plugins" && ( "$TEST_SCRIPT" == "test-php" || "$TEST_SCRIPT" == "test-coverage" ) ]]; then
                 if [[ ! -f "$DIR/composer.lock" ]]; then
                   echo 'No composer.lock, running `composer update`'
                   composer --working-dir="$DIR" update
@@ -187,6 +184,10 @@ jobs:
 
               mkdir -p "artifacts/$SLUG"
               export ARTIFACTS_DIR="$GITHUB_WORKSPACE/artifacts/$SLUG"
+              if [[ "$TEST_SCRIPT" == "test-coverage" ]]; then
+                mkdir -p "coverage/$SLUG"
+                export COVERAGE_DIR="$GITHUB_WORKSPACE/coverage/$SLUG"
+              fi
               FAIL=false
               if ! composer run --timeout=0 --working-dir="$DIR" "$TEST_SCRIPT"; then
                 FAIL=true
@@ -203,6 +204,12 @@ jobs:
           done
           exit $EXIT
 
+      - name: Process coverage results
+        env:
+          CHANGED: ${{ steps.changed.outputs.projects }}
+        if: matrix.script == 'test-coverage'
+        run: .github/files/process-coverage.sh
+
       - name: Check for artifacts
         id: check-artifacts
         # Default for `if` is `success()`, we want this to run always.

diff --git a/composer.lock b/composer.lock
diff --git a/docs/examples/bootstrap.php b/docs/examples/bootstrap.php
@@ -9,9 +9,9 @@
 $_plugin_root = dirname( __DIR__, 2 );
 
 // Locate WordPress or wordpress-develop. We look in several places.
-if ( false !== getenv( 'WP_DEVELOP_DIR' ) ) {
+if ( false !== getenv( 'WORDPRESS_DEVELOP_DIR' ) ) {
 	// Jetpack Monorepo environment variable.
-	$_tests_dir = getenv( 'WP_DEVELOP_DIR' );
+	$_tests_dir = getenv( 'WORDPRESS_DEVELOP_DIR' );
 	if ( file_exists( "$_tests_dir/tests/phpunit/" ) ) {
 		$_tests_dir .= '/tests/phpunit/';
 	}
@@ -38,7 +38,7 @@
 if ( ! isset( $_tests_dir ) || ! file_exists( $_tests_dir . '/includes/bootstrap.php' ) ) {
 	echo 'Failed to automatically locate WordPress or wordpress-develop to run tests.' . PHP_EOL;
 	echo PHP_EOL;
-	echo 'Set the WP_DEVELOP_DIR environment variable to point to a copy of WordPress' . PHP_EOL;
+	echo 'Set the WORDPRESS_DEVELOP_DIR environment variable to point to a copy of WordPress' . PHP_EOL;
 	echo 'or wordpress-develop.' . PHP_EOL;
 	exit( 1 );
 }

diff --git a/docs/monorepo.md b/docs/monorepo.md
@@ -116,6 +116,8 @@ We use `composer.json` to hold metadata about projects. Much of our generic tool
 * `.repositories`: If you include a repository entry referencing monorepo packages, it must have `.options.monorepo` set to true. This allows the build tooling to recognize and remove it.
 * `.scripts.build-development`: If your project has a general build step, this must run the necessary commands. See [Building](#building) for details.
 * `.scripts.build-production`: If your project requires a production-specific build step, this must run the necessary commands. See [Building](#building) for details.
+* `.scripts.test-coverage`: If the package contains any tests, this must run the necessary commands to generate a coverage report. See [Code coverage](#code-coverage) for details.
+  * `.scripts.skip-test-coverage`: Run before `.scripts.test-coverage` in CI. If it exits with code 3, the test run will be skipped.
 * `.scripts.test-e2e`: If the package contains any E2E tests, this must run the necessary commands. See [E2E tests](#e2e-tests) for details.
 * `.scripts.test-js`: If the package contains any JavaScript tests, this must run the necessary commands. See [JavaScript tests](#javascript-tests) for details.
   * `.scripts.skip-test-js`: Run before `.scripts.test-js` in CI. If it exits with code 3, the test run will be skipped.
@@ -183,6 +185,8 @@ The following environment variables are available for all tests:
 
 - `ARTIFACTS_DIR`: If your tests generate any artifacts that might be useful for debugging, you may place them in the directory specified by this variable and they will be uploaded to GitHub after the test run. There's no need to be concerned about collisions with other projects' artifacts, a separate directory is used per project.
 - `MONOREPO_BASE`: Path to the monorepo. Useful if you're using things in `tools/` from plugin tests.
+- `WORDPRESS_DIR`: Path to a copy of WordPress. Other than plugin tests, though, you should probably avoid using this.
+- `WORDPRESS_DEVELOP_DIR`: Path to a checkout of wordpress-develop. Other than plugin tests, though, you should probably avoid using this.
 - `NODE_VERSION`: The version of Node in use, as specified in `.github/versions.sh`.
 - `PHP_VERSION`: The version of PHP in use. Unless otherwise specified below, it will be the same as in `.github/versions.sh`.
 - `TEST_SCRIPT`: The test script being run.
@@ -281,7 +285,7 @@ We currently make use of the following packages in testing; it's encouraged to u
 
 #### PHP tests for plugins
 
-WordPress plugins generally want to run within WordPress. All monorepo plugins are copied into place in a WordPress installation and tests are run from there.
+WordPress plugins may want to run within WordPress. All monorepo plugins are copied into place in a WordPress installation. Environment variable `WORDPRESS_DIR` points to this installation, and `WORDPRESS_DEVELOP_DIR` points a directory with WordPress's `tests/phpunit/`.
 
 Tests will be run against the latest version of WordPress using the variety of supported PHP versions, and against the previous and trunk versions of WordPress using the PHP version in `.github/versions.sh`. The environment variable `WP_BRANCH` will be set to 'latest', 'previous', or 'trunk' accordingly. If you have tests that only need to be run once, run them when `WP_BRANCH` is 'latest'.
 
@@ -299,6 +303,16 @@ JavaScript tests should use `jest`, not `mocha`/`chai`/`sinon`. For React testin
 
 If a project contains end-to-end tests, it must define `.scripts.test-e2e` in `composer.json` to run the tests. If a build step is required before running tests, the necessary commands for that should also be included.
 
+### Code coverage
+
+If a project contains PHP or JavaScript tests, it should also define `.scripts.test-coverage` in `composer.json` to run the tests in a mode that will generate code coverage output. The CI environment will run `pnpm install` and `composer install` beforehand, but if a build step is required before running tests the necessary commands for that should also be included in `.scripts.test-coverage`.
+
+Output should be written to the path specified via the `COVERAGE_DIR` environment variable. Subdirectories of that path may be used as desired.
+
+For PHP tests, you'll probably run PHPUnit as `php -dpcov.directory=. "$(command -v phpunit)" --coverage-clover "$COVERAGE_DIR/php/clover.xml"`.
+
+There's no need to be concerned about collisions with other projects' coverage files, a separate directory is used per project. The coverage files are also automatically copied to `ARTIFACTS_DIR`.
+
 ## Mirror repositories
 
 Most projects in the monorepo should have a mirror repository holding a built version of the project, ready for deployment. Follow these steps to create the mirror repo and configure the monorepo tooling to push to it.

diff --git a/docs/quick-start.md b/docs/quick-start.md
@@ -112,6 +112,6 @@ jetpack watch
 ```
 ### Running Tests
 
-To run PHP and JS tests, you can use the Jetpack CLI: `jetpack test` and then choose the project and type of test you'd like to run.
+To run PHP, JS, and coverage tests, you can use the Jetpack CLI: `jetpack test` and then choose the project and type of test you'd like to run.
 
 That's all!
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/projects/github-actions/test-results-to-slack/changelog/restore-jp_test_coverage b/projects/github-actions/test-results-to-slack/changelog/restore-jp_test_coverage
@@ -0,0 +1,4 @@
+Significance: patch
+Type: added
+
+Enable test coverage.
diff --git a/projects/github-actions/test-results-to-slack/composer.json b/projects/github-actions/test-results-to-slack/composer.json
@@ -11,6 +11,7 @@
 		"build-development": [
 			"pnpm run build"
 		],
+		"test-coverage": "pnpm run test-coverage",
 		"test-js": "pnpm run test"
 	},
 	"repositories": [

diff --git a/projects/github-actions/test-results-to-slack/package.json b/projects/github-actions/test-results-to-slack/package.json
@@ -27,6 +27,7 @@
 	},
 	"scripts": {
 		"build": "ncc build src/index.js -o dist --source-map --license licenses.txt",
-		"test": "jest --config=tests/jest.config.js --verbose --runInBand"
+		"test": "jest --config=tests/jest.config.js --verbose --runInBand",
+		"test-coverage": "pnpm run test --coverage --coverageDirectory=\"$COVERAGE_DIR/js\" --coverageReporters=clover"
 	}
 }
diff --git a/projects/js-packages/ai-client/changelog/restore-jp_test_coverage b/projects/js-packages/ai-client/changelog/restore-jp_test_coverage
@@ -0,0 +1,4 @@
+Significance: patch
+Type: added
+
+Enable test coverage.
diff --git a/projects/js-packages/ai-client/composer.json b/projects/js-packages/ai-client/composer.json
@@ -24,6 +24,9 @@
 			"Composer\\Config::disableProcessTimeout",
 			"pnpm run watch"
 		],
+		"test-coverage": [
+			"pnpm run test-coverage"
+		],
 		"test-js": [
 			"pnpm run test"
 		]

diff --git a/projects/js-packages/ai-client/package.json b/projects/js-packages/ai-client/package.json
@@ -19,6 +19,7 @@
 		"clean": "rm -rf build/",
 		"compile-ts": "tsc --pretty",
 		"test": "NODE_OPTIONS=--experimental-vm-modules jest",
+		"test-coverage": "pnpm run test --coverage --coverageDirectory=\"$COVERAGE_DIR/js\" --coverageReporters=clover",
 		"watch": "tsc --watch --pretty"
 	},
 	"type": "module",

diff --git a/projects/js-packages/analytics/changelog/restore-jp_test_coverage b/projects/js-packages/analytics/changelog/restore-jp_test_coverage
@@ -0,0 +1,4 @@
+Significance: patch
+Type: added
+
+Enable test coverage.
diff --git a/projects/js-packages/analytics/composer.json b/projects/js-packages/analytics/composer.json
@@ -11,6 +11,9 @@
 	"scripts": {
 		"test-js": [
 			"pnpm run test"
+		],
+		"test-coverage": [
+			"pnpm run test-coverage"
 		]
 	},
 	"repositories": [

diff --git a/projects/js-packages/analytics/package.json b/projects/js-packages/analytics/package.json
@@ -24,6 +24,7 @@
 		".": "./index.jsx"
 	},
 	"scripts": {
-		"test": "NODE_OPTIONS=--experimental-vm-modules jest"
+		"test": "NODE_OPTIONS=--experimental-vm-modules jest",
+		"test-coverage": "pnpm run test --coverageDirectory=\"$COVERAGE_DIR/js\" --coverage --coverageReporters=clover"
 	}
 }
diff --git a/projects/js-packages/api/changelog/restore-jp_test_coverage b/projects/js-packages/api/changelog/restore-jp_test_coverage
@@ -0,0 +1,4 @@
+Significance: patch
+Type: added
+
+Enable test coverage.
diff --git a/projects/js-packages/api/composer.json b/projects/js-packages/api/composer.json
@@ -9,6 +9,9 @@
 	"scripts": {
 		"test-js": [
 			"pnpm run test"
+		],
+		"test-coverage": [
+			"pnpm run test-coverage"
 		]
 	},
 	"repositories": [

diff --git a/projects/js-packages/api/package.json b/projects/js-packages/api/package.json
@@ -26,6 +26,7 @@
 		".": "./index.jsx"
 	},
 	"scripts": {
-		"test": "NODE_OPTIONS=--experimental-vm-modules jest"
+		"test": "NODE_OPTIONS=--experimental-vm-modules jest",
+		"test-coverage": "pnpm run test --coverageDirectory=\"$COVERAGE_DIR/js\" --coverage --coverageReporters=clover"
 	}
 }
diff --git a/projects/js-packages/babel-plugin-replace-textdomain/changelog/restore-jp_test_coverage b/projects/js-packages/babel-plugin-replace-textdomain/changelog/restore-jp_test_coverage
@@ -0,0 +1,4 @@
+Significance: patch
+Type: added
+
+Enable test coverage.