build: Move "Test plugin upgrades" to a post-build workflow (#28241)

GitHub has an unfortunate behavior in that workflow artifacts are not available via the API until the entire workflow has completed. This means that the "Test plugin upgrades" job in the build workflow unnecessarily delays the availability of the build artifact for Jetpack Live Branches and future wpcom automated testing for 6 or so minutes. To get around this, we create a new "Post-Build" workflow that is triggered by `workflow_run` after the Build workflow completes. But since this new workflow isn't automatically attached to the PR, we have to make API calls to get it to show up there. This is a second attempt at #25892, which had to be reverted as using the Actions app put the checks into the "suite" of a random workflow.
Automattic · Jan 19, 2023 · c94eb1e · c94eb1e
1 parent 6a2ee27
commit c94eb1e
Show file tree

Hide file tree

Showing 6 changed files with 390 additions and 62 deletions.
diff --git a/.github/actions/check-run/action.yml b/.github/actions/check-run/action.yml
@@ -0,0 +1,70 @@
+name: "Check Run"
+description: "Create or update a check run. This is a simple wrapper around GitHub's checks API, https://docs.github.com/en/rest/checks/runs."
+inputs:
+  id:
+    description: "Check run ID to update. If not given, a new run will be created"
+  sha:
+    description: "The SHA of the commit."
+  name:
+    description: "The name of the check. For example, \"code-coverage\"."
+  status:
+    description: "The current status. Can be one of: `queued`, `in_progress`, `completed`"
+  conclusion:
+    description: "The final conclusion of the check. Can be one of: `action_required`, `cancelled`, `failure`, `neutral`, `success`, `skipped`, `timed_out`"
+  title:
+    description: "Title of the check. Shown in the PR's checks list."
+  summary:
+    description: "Summary of the check. Can contain Markdown."
+  token:
+    description: "App access token. See https://docs.github.com/en/developers/apps/building-github-apps/authenticating-with-github-apps#authenticating-as-a-github-app for how to get one."
+outputs:
+  id:
+    description: "Check run ID."
+    value: ${{ steps.run.outputs.id }}
+runs:
+  using: composite
+  steps:
+    - id: run
+      shell: bash
+      env:
+        ID: ${{ inputs.id }}
+        SHA: ${{ inputs.sha }}
+        NAME: ${{ inputs.name }}
+        STATUS: ${{ inputs.status }}
+        CONCLUSION: ${{ inputs.conclusion }}
+        TITLE: ${{ inputs.title }}
+        SUMMARY: ${{ inputs.summary }}
+        TOKEN: ${{ inputs.token }}
+      run: |
+        if [[ -n "$ID" ]]; then
+          METHOD=PATCH
+          URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/check-runs/$ID"
+        else
+          METHOD=POST
+          URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/check-runs"
+        fi
+
+        DATA="{}"
+        if [[ -n "$NAME" ]]; then
+          DATA="$(jq --arg v "$NAME" '.name |= $v' <<<"$DATA")"
+        fi
+        if [[ -n "$SHA" ]]; then
+          DATA="$(jq --arg v "$SHA" '.head_sha |= $v' <<<"$DATA")"
+        fi
+        if [[ -n "$STATUS" ]]; then
+          DATA="$(jq --arg v "$STATUS" '.status |= $v' <<<"$DATA")"
+        fi
+        if [[ -n "$CONCLUSION" ]]; then
+          DATA="$(jq --arg v "$CONCLUSION" '.conclusion |= $v' <<<"$DATA")"
+        fi
+        if [[ -n "$TITLE" ]]; then
+          DATA="$(jq --arg v "$TITLE" '.output.title |= $v' <<<"$DATA")"
+        fi
+        if [[ -n "$SUMMARY" ]]; then
+          DATA="$(jq --arg v "$SUMMARY" '.output.summary |= $v' <<<"$DATA")"
+        fi
+
+        echo "Data: $DATA"
+        JSON="$(curl -v -X "$METHOD" --header "authorization: Bearer $TOKEN" --url "$URL" --data "$DATA")"
+        echo "$JSON"
+        echo "id=$(jq -r .id <<<"$JSON")" >> "$GITHUB_OUTPUT"
diff --git a/.github/files/test-plugin-update/prepare-zips.sh b/.github/files/test-plugin-update/prepare-zips.sh
@@ -39,6 +39,7 @@ while IFS=$'\t' read -r SRC MIRROR SLUG; do
 			else
 				echo "::error::Unexpected response from betadownload.jetpack.me for $SLUG"
 				echo "$JSON"
+				echo "info=❌ Unexpected response from betadownload.jetpack.me for $SLUG" >> "$GITHUB_OUTPUT"
 				exit 1
 			fi
 		fi
@@ -59,7 +60,10 @@ while IFS=$'\t' read -r SRC MIRROR SLUG; do
 	else
 		echo "::error::Unexpected response from WordPress.org API for $SLUG"
 		echo "$JSON"
+		echo "info=❌ Unexpected response from WordPress.org API for $SLUG" >> "$GITHUB_OUTPUT"
 		exit 1
 	fi
 	echo "::endgroup::"
 done < build/plugins.tsv
+
+echo 'info=' >> "$GITHUB_OUTPUT"
diff --git a/.github/files/test-plugin-update/test.sh b/.github/files/test-plugin-update/test.sh
@@ -2,6 +2,8 @@
 
 set -eo pipefail
 
+source "$GITHUB_WORKSPACE/monorepo/.github/files/gh-funcs.sh"
+
 ZIPDIR="$GITHUB_WORKSPACE/zips"
 EXIT=0
 
@@ -11,20 +13,38 @@ for ZIP in *-dev.zip; do
 	SLUGS+=( "${ZIP%-dev.zip}" )
 done
 
+FINISHED=false
+OUTPUT=()
+
+function onexit {
+	if ! "$FINISHED"; then
+		OUTPUT+=( "💣 The testing script exited unexpectedly." )
+	fi
+	gh_set_output info "$( printf "%s\n" "${OUTPUT[@]}" )"
+}
+trap "onexit" EXIT
+
+function failed {
+	ERRMSG="$1"
+	OUTPUT+=( "❌ $ERRMSG" )
+	FAILED=1
+	EXIT=1
+}
+
 cd /var/www/html
 for SLUG in "${SLUGS[@]}"; do
 	for FROM in stable trunk dev; do
 		for HOW in web cli; do
 			[[ -e "$ZIPDIR/$SLUG-$FROM.zip" ]] || continue
 
+			FAILED=
 			printf '\n\e[1mTest upgrade of %s from %s via %s\e[0m\n' "$SLUG" "$FROM" "$HOW"
 
 			ERRMSG=
 			echo "::group::Installing $SLUG $FROM"
 			: > /var/www/html/wp-content/debug.log
 			if ! wp --allow-root plugin install --activate "$ZIPDIR/$SLUG-$FROM.zip"; then
-				ERRMSG="Plugin install failed for $SLUG $FROM!"
-				EXIT=1
+				failed "Plugin install failed for $SLUG $FROM!"
 			fi
 			echo '== Debug log =='
 			cat /var/www/html/wp-content/debug.log
@@ -46,8 +66,7 @@ for SLUG in "${SLUGS[@]}"; do
 			: > /var/www/html/wp-content/debug.log
 			if [[ "$HOW" == 'cli' ]]; then
 				if ! wp --allow-root plugin upgrade "$SLUG" 2>&1 | tee "$GITHUB_WORKSPACE/out.txt"; then
-					ERRMSG="CLI upgrade of $SLUG from $FROM exited with a non-zero status"
-					EXIT=1
+					failed "CLI upgrade of $SLUG from $FROM exited with a non-zero status"
 				fi
 			else
 				# Everything needs to be owned by www-data for the web upgrade to proceed.
@@ -63,19 +82,18 @@ for SLUG in "${SLUGS[@]}"; do
 			fi
 			ERR="$(grep -i 'Fatal error' /var/www/html/wp-content/debug.log || true)"
 			if [[ -n "$ERR" ]]; then
-				echo "::error::Mid-upgrade fatal detected for $SLUG $HOW update from $FROM!%0A$ERR"
-				EXIT=1
+				failed "Mid-upgrade fatal detected for $SLUG $HOW update from $FROM!%0A$ERR"
+				echo "::error::$ERRMSG"
 			elif [[ ! -e "/var/www/html/wp-content/plugins/$SLUG/ci-flag.txt" ]]; then
-				echo "::error::Plugin $SLUG ($HOW update from $FROM) does not seem to have been updated?"
-				EXIT=1
+				failed "Plugin $SLUG ($HOW update from $FROM) does not seem to have been updated?"
+				echo "::error::$ERRMSG"
 			fi
 
 			ERRMSG=
 			echo "::group::Deactivating $SLUG"
 			: > /var/www/html/wp-content/debug.log
 			if ! wp --allow-root plugin deactivate "$SLUG"; then
-				ERRMSG="Plugin deactivate failed after $SLUG $HOW update from $FROM!"
-				EXIT=1
+				failed "Plugin deactivate failed after $SLUG $HOW update from $FROM!"
 			fi
 			echo '== Debug log =='
 			cat /var/www/html/wp-content/debug.log
@@ -88,8 +106,7 @@ for SLUG in "${SLUGS[@]}"; do
 			echo "::group::Uninstalling $SLUG"
 			: > /var/www/html/wp-content/debug.log
 			if ! wp --allow-root plugin uninstall "$SLUG"; then
-				ERRMSG="Plugin uninstall failed after $SLUG $HOW update from $FROM!"
-				EXIT=1
+				failed "Plugin uninstall failed after $SLUG $HOW update from $FROM!"
 				rm -rf "/var/www/html/wp-content/plugins/$SLUG"
 			fi
 			echo '== Debug log =='
@@ -101,8 +118,13 @@ for SLUG in "${SLUGS[@]}"; do
 
 			wp --allow-root --quiet option delete fake_plugin_update_plugin
 			wp --allow-root --quiet option delete fake_plugin_update_url
+
+			if [[ -z "$FAILED" ]]; then
+				OUTPUT+=( "✅ Upgrade of $SLUG from $FROM via $HOW succeeded!" )
+			fi
 		done
 	done
 done
 
+FINISHED=true
 exit $EXIT
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -132,56 +132,6 @@ jobs:
               comment_id: +COMMENT_ID,
             } );
 
-  upgrade_test:
-    name: Test plugin upgrades
-    runs-on: ubuntu-latest
-    needs: build
-    if: needs.build.outputs.any_plugins == 'true'
-    timeout-minutes: 10  # 2022-03-04: Successful runs seem to take about 3 minutes, but give some extra time for the downloads.
-    services:
-      db:
-        image: mariadb:latest
-        env:
-          MARIADB_ROOT_PASSWORD: wordpress
-        ports:
-          - 3306:3306
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5
-    container:
-      image: ghcr.io/automattic/jetpack-wordpress-dev:latest
-      env:
-        WP_DOMAIN: localhost
-        WP_ADMIN_USER: wordpress
-        WP_ADMIN_EMAIL: wordpress@example.com
-        WP_ADMIN_PASSWORD: wordpress
-        WP_TITLE: Hello World
-        MYSQL_HOST: db:3306
-        MYSQL_DATABASE: wordpress
-        MYSQL_USER: root
-        MYSQL_PASSWORD: wordpress
-        HOST_PORT: 80
-      ports:
-        - 80:80
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          path: monorepo
-
-      - name: Download build artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: jetpack-build
-      - name: Extract build archive
-        run: tar --xz -xvvf build.tar.xz
-
-      - name: Setup WordPress
-        run: monorepo/.github/files/test-plugin-update/setup.sh
-
-      - name: Prepare plugin zips
-        run: monorepo/.github/files/test-plugin-update/prepare-zips.sh
-
-      - name: Test upgrades
-        run: monorepo/.github/files/test-plugin-update/test.sh
-
   jetpack_beta:
     name: Create artifact for Jetpack Beta plugin
     runs-on: ubuntu-latest