diff --git a/.github/workflows/pull-request-automation.yml b/.github/workflows/pull-request-automation.yml
index 0cd5452ddd09e..c4fb350b898ef 100644
--- a/.github/workflows/pull-request-automation.yml
+++ b/.github/workflows/pull-request-automation.yml
@@ -6,10 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master
-      - run: npm install
-        env:
-          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
-      - run: npm run build:packages
+      - run: cd packages/pull-request-automation && npm install
       - uses: ./packages/pull-request-automation
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/packages/pull-request-automation/action.yml b/packages/pull-request-automation/action.yml
index feef10aa42658..5fea66ee443fc 100644
--- a/packages/pull-request-automation/action.yml
+++ b/packages/pull-request-automation/action.yml
@@ -8,4 +8,4 @@ inputs:
     required: true
 runs:
   using: node12
-  main: build/index.js
+  main: lib/index.js
diff --git a/packages/pull-request-automation/src/add-first-time-contributor-label.js b/packages/pull-request-automation/lib/add-first-time-contributor-label.js
similarity index 80%
rename from packages/pull-request-automation/src/add-first-time-contributor-label.js
rename to packages/pull-request-automation/lib/add-first-time-contributor-label.js
index b6f1c0b7e821f..7a8b868ad0e08 100644
--- a/packages/pull-request-automation/src/add-first-time-contributor-label.js
+++ b/packages/pull-request-automation/lib/add-first-time-contributor-label.js
@@ -1,4 +1,4 @@
-export default async function addFirstTimeContributorLabel( context, octokit ) {
+async function addFirstTimeContributorLabel( context, octokit ) {
 	const owner = context.payload.repository.owner.login;
 	const repo = context.payload.repository.name;
 	const author = context.payload.pull_request.user.login;
@@ -18,3 +18,5 @@ export default async function addFirstTimeContributorLabel( context, octokit ) {
 		labels: [ 'First-time Contributor' ],
 	} );
 }
+
+module.exports = addFirstTimeContributorLabel;
diff --git a/packages/pull-request-automation/src/add-milestone.js b/packages/pull-request-automation/lib/add-milestone.js
similarity index 96%
rename from packages/pull-request-automation/src/add-milestone.js
rename to packages/pull-request-automation/lib/add-milestone.js
index 77a52c8baf505..18eb3b8966e0d 100644
--- a/packages/pull-request-automation/src/add-milestone.js
+++ b/packages/pull-request-automation/lib/add-milestone.js
@@ -7,7 +7,7 @@ const REFERENCE_DATE = '2019-08-12';
 // Releases are every 14 days.
 const DAYS_PER_RELEASE = 14;
 
-export default async function addMilestone( context, octokit ) {
+async function addMilestone( context, octokit ) {
 	if ( ! context.payload.pull_request.merged ) {
 		return;
 	}
@@ -73,3 +73,5 @@ export default async function addMilestone( context, octokit ) {
 		milestone: number,
 	} );
 }
+
+module.exports = addMilestone;
diff --git a/packages/pull-request-automation/src/assign-fixed-issues.js b/packages/pull-request-automation/lib/assign-fixed-issues.js
similarity index 88%
rename from packages/pull-request-automation/src/assign-fixed-issues.js
rename to packages/pull-request-automation/lib/assign-fixed-issues.js
index e1c01e3e22c1c..f57b5860145de 100644
--- a/packages/pull-request-automation/src/assign-fixed-issues.js
+++ b/packages/pull-request-automation/lib/assign-fixed-issues.js
@@ -1,4 +1,4 @@
-export default async function assignFixedIssues( context, octokit ) {
+async function assignFixedIssues( context, octokit ) {
 	const regex = /(?:close|closes|closed|fix|fixes|fixed|resolve|resolves|resolved):? +(?:\#?|https?:\/\/github\.com\/WordPress\/gutenberg\/issues\/)(\d+)/gi;
 
 	let match;
@@ -20,3 +20,5 @@ export default async function assignFixedIssues( context, octokit ) {
 		} );
 	}
 }
+
+module.exports = assignFixedIssues;
diff --git a/packages/pull-request-automation/src/index.js b/packages/pull-request-automation/lib/index.js
similarity index 71%
rename from packages/pull-request-automation/src/index.js
rename to packages/pull-request-automation/lib/index.js
index 0c63706bf95c9..87c21e2e1b6d6 100644
--- a/packages/pull-request-automation/src/index.js
+++ b/packages/pull-request-automation/lib/index.js
@@ -1,15 +1,15 @@
 /**
- * External dependencies
+ * GitHub dependencies
  */
-import { setFailed, getInput, debug } from '@actions/core';
-import { context, GitHub } from '@actions/github';
+const { setFailed, getInput, debug } = require( '@actions/core' );
+const { context, GitHub } = require( '@actions/github' );
 
 /**
  * Internal dependencies
  */
-import assignFixedIssues from './assign-fixed-issues';
-import addFirstTimeContributorLabel from './add-first-time-contributor-label';
-import addMilestone from './add-milestone';
+const assignFixedIssues = require( './assign-fixed-issues' );
+const addFirstTimeContributorLabel = require( './add-first-time-contributor-label' );
+const addMilestone = require( './add-milestone' );
 
 ( async function main() {
 	if ( context.eventName !== 'pull_request' ) {
diff --git a/packages/pull-request-automation/src/test/add-first-time-contributor-label.js b/packages/pull-request-automation/lib/test/add-first-time-contributor-label.js
similarity index 100%
rename from packages/pull-request-automation/src/test/add-first-time-contributor-label.js
rename to packages/pull-request-automation/lib/test/add-first-time-contributor-label.js
diff --git a/packages/pull-request-automation/src/test/add-milestone.js b/packages/pull-request-automation/lib/test/add-milestone.js
similarity index 100%
rename from packages/pull-request-automation/src/test/add-milestone.js
rename to packages/pull-request-automation/lib/test/add-milestone.js
diff --git a/packages/pull-request-automation/src/test/assign-fixed-issues.js b/packages/pull-request-automation/lib/test/assign-fixed-issues.js
similarity index 100%
rename from packages/pull-request-automation/src/test/assign-fixed-issues.js
rename to packages/pull-request-automation/lib/test/assign-fixed-issues.js