Add e2e tests to CI

cloudflare · Jun 13, 2023 · d861067 · d861067
1 parent 05099f4
commit d861067
Show file tree

Hide file tree

Showing 12 changed files with 327 additions and 67 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -53,3 +53,58 @@ jobs:
           TMP_CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           TMP_CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
           NODE_OPTIONS: "--max_old_space_size=8192"
+  e2e-test:
+    if: ${{ github.repository_owner == 'cloudflare' }}
+    name: "E2E Test"
+    strategy:
+      matrix:
+        os: [
+            # macos-11,
+            # macos-12,
+            macos-13,
+            windows-2019,
+            windows-2022,
+            # ubuntu-20.04,
+            ubuntu-22.04,
+          ]
+        node: ["16", "18"]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Use Node.js ${{ matrix.node }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          cache: "npm"
+
+      - name: Install workerd Dependencies
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          export DEBIAN_FRONTEND=noninteractive
+          sudo apt-get update
+          sudo apt-get install -y libc++1
+
+        # Attempt to cache all the node_modules directories based on the OS and package lock.
+      - name: Cache node_modules
+        id: npm-cache
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-node-modules
+        with:
+          key: ${{ runner.os }}-${{ matrix.node }}-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          path: "**/node_modules"
+
+      # If the cache missed then install using `npm ci` to follow package lock exactly
+      - if: ${{ steps.npm-cache.outputs.cache-hit != 'true'}}
+        name: Install NPM Dependencies
+        run: npm ci
+
+      - name: Run tests
+        run: cd packages/wrangler && npm run test:e2e
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.TEST_CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.TEST_CLOUDFLARE_ACCOUNT_ID }}
+          NODE_OPTIONS: "--max_old_space_size=8192"
diff --git a/.github/workflows/pullrequests.yml b/.github/workflows/pullrequests.yml
@@ -113,3 +113,58 @@ jobs:
 
       - name: Report Code Coverage
         uses: codecov/codecov-action@v3
+  e2e-test:
+    if: ${{ github.repository_owner == 'cloudflare' }}
+    name: "E2E Test"
+    strategy:
+      matrix:
+        os: [
+            # macos-11,
+            # macos-12,
+            macos-13,
+            windows-2019,
+            windows-2022,
+            # ubuntu-20.04,
+            ubuntu-22.04,
+          ]
+        node: ["16", "18"]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Use Node.js ${{ matrix.node }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          cache: "npm"
+
+      - name: Install workerd Dependencies
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          export DEBIAN_FRONTEND=noninteractive
+          sudo apt-get update
+          sudo apt-get install -y libc++1
+
+        # Attempt to cache all the node_modules directories based on the OS and package lock.
+      - name: Cache node_modules
+        id: npm-cache
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-node-modules
+        with:
+          key: ${{ runner.os }}-${{ matrix.node }}-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          path: "**/node_modules"
+
+      # If the cache missed then install using `npm ci` to follow package lock exactly
+      - if: ${{ steps.npm-cache.outputs.cache-hit != 'true'}}
+        name: Install NPM Dependencies
+        run: npm ci
+
+      - name: Run tests
+        run: cd packages/wrangler && npm run test:e2e
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.TEST_CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.TEST_CLOUDFLARE_ACCOUNT_ID }}
+          NODE_OPTIONS: "--max_old_space_size=8192"
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/wrangler/e2e/deploy.test.ts b/packages/wrangler/e2e/deploy.test.ts
@@ -1,8 +1,8 @@
 import crypto from "node:crypto";
 import path from "node:path";
-import { setTimeout } from "node:timers/promises";
 import { fetch } from "undici";
 import { describe, expect, it } from "vitest";
+import { retry } from "./helpers/retry";
 import { RUN, runIn } from "./helpers/run";
 import { dedent, makeRoot, seed } from "./helpers/setup";
 
@@ -33,15 +33,11 @@ describe("deploy", async () => {
 			✨ Created smoke-test-worker/src/index.ts
 			Your project will use Vitest to run your tests.
 			✨ Created smoke-test-worker/src/index.test.ts
-
 			added (N) packages, and audited (N) packages in (TIMINGS)
-
 			(N) packages are looking for funding
 			  run \`npm fund\` for details
-
 			found 0 vulnerabilities
 			✨ Installed @cloudflare/workers-types, typescript, and vitest into devDependencies
-
 			To start developing your Worker, run \`cd smoke-test-worker && npm start\`
 			To start testing your Worker, run \`npm test\`
 			To publish your Worker to the Internet, run \`npm run deploy\`"
@@ -65,10 +61,11 @@ describe("deploy", async () => {
 		expect(stderr).toMatchInlineSnapshot('""');
 		workersDev = matchWorkersDev(rawStdout);
 
-		await setTimeout(5_000);
-		await expect(
-			fetch(`https://${workerName}.${workersDev}`).then((r) => r.text())
-		).resolves.toMatchInlineSnapshot('"Hello World!"');
+		await retry(() =>
+			expect(
+				fetch(`https://${workerName}.${workersDev}`).then((r) => r.text())
+			).resolves.toMatchInlineSnapshot('"Hello World!"')
+		);
 	});
 	it("modify & deploy worker", async () => {
 		await seed(workerPath, {
@@ -96,10 +93,11 @@ describe("deploy", async () => {
 		expect(stderr).toMatchInlineSnapshot('""');
 		workersDev = matchWorkersDev(rawStdout);
 
-		await setTimeout(10_000);
-		await expect(
-			fetch(`https://${workerName}.${workersDev}`).then((r) => r.text())
-		).resolves.toMatchInlineSnapshot('"Updated Worker!"');
+		await retry(() =>
+			expect(
+				fetch(`https://${workerName}.${workersDev}`).then((r) => r.text())
+			).resolves.toMatchInlineSnapshot('"Updated Worker!"')
+		);
 	});
 
 	it("delete worker", async () => {
@@ -114,9 +112,10 @@ describe("deploy", async () => {
 			Successfully deleted smoke-test-worker"
 		`);
 		expect(stderr).toMatchInlineSnapshot('""');
-		await setTimeout(10_000);
-		await expect(
-			fetch(`https://${workerName}.${workersDev}`).then((r) => r.status)
-		).resolves.toBe(404);
+		await retry(() =>
+			expect(
+				fetch(`https://${workerName}.${workersDev}`).then((r) => r.status)
+			).resolves.toBe(404)
+		);
 	});
 });
diff --git a/packages/wrangler/e2e/deployments.test.ts b/packages/wrangler/e2e/deployments.test.ts
@@ -1,8 +1,8 @@
 import crypto from "node:crypto";
 import path from "node:path";
-import { setTimeout } from "node:timers/promises";
 import { fetch } from "undici";
 import { describe, expect, it } from "vitest";
+import { retry } from "./helpers/retry";
 import { RUN, runIn } from "./helpers/run";
 import { dedent, makeRoot, seed } from "./helpers/setup";
 
@@ -46,15 +46,11 @@ describe("deployments", async () => {
 			✨ Created smoke-test-worker/src/index.ts
 			Your project will use Vitest to run your tests.
 			✨ Created smoke-test-worker/src/index.test.ts
-
 			added (N) packages, and audited (N) packages in (TIMINGS)
-
 			(N) packages are looking for funding
 			  run \`npm fund\` for details
-
 			found 0 vulnerabilities
 			✨ Installed @cloudflare/workers-types, typescript, and vitest into devDependencies
-
 			To start developing your Worker, run \`cd smoke-test-worker && npm start\`
 			To start testing your Worker, run \`npm test\`
 			To publish your Worker to the Internet, run \`npm run deploy\`"
@@ -78,10 +74,11 @@ describe("deployments", async () => {
 		expect(stderr).toMatchInlineSnapshot('""');
 		workersDev = matchWorkersDev(rawStdout);
 
-		await setTimeout(2_000);
-		await expect(
-			fetch(`https://${workerName}.${workersDev}`).then((r) => r.text())
-		).resolves.toMatchInlineSnapshot('"Hello World!"');
+		await retry(() =>
+			expect(
+				fetch(`https://${workerName}.${workersDev}`).then((r) => r.text())
+			).resolves.toMatchInlineSnapshot('"Hello World!"')
+		);
 	});
 
 	it("list 1 deployment", async () => {
@@ -93,8 +90,6 @@ describe("deployments", async () => {
 	`;
 		expect(stdout).toMatchInlineSnapshot(`
 			"🚧\`wrangler deployments\` is a beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose
-
-
 			Deployment ID: 00000000-0000-0000-0000-000000000000
 			Created on:    TIMESTAMP
 			Author:        person@example.com
@@ -130,10 +125,11 @@ describe("deployments", async () => {
 		expect(stderr).toMatchInlineSnapshot('""');
 		workersDev = matchWorkersDev(rawStdout);
 
-		await setTimeout(10_000);
-		await expect(
-			fetch(`https://${workerName}.${workersDev}`).then((r) => r.text())
-		).resolves.toMatchInlineSnapshot('"Updated Worker!"');
+		await retry(() =>
+			expect(
+				fetch(`https://${workerName}.${workersDev}`).then((r) => r.text())
+			).resolves.toMatchInlineSnapshot('"Updated Worker!"')
+		);
 	});
 
 	it("list 2 deployments", async () => {
@@ -145,13 +141,10 @@ describe("deployments", async () => {
 	`;
 		expect(stdout).toMatchInlineSnapshot(`
 			"🚧\`wrangler deployments\` is a beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose
-
-
 			Deployment ID: 00000000-0000-0000-0000-000000000000
 			Created on:    TIMESTAMP
 			Author:        person@example.com
 			Source:        Upload from Wrangler 🤠
-
 			Deployment ID: 00000000-0000-0000-0000-000000000000
 			Created on:    TIMESTAMP
 			Author:        person@example.com
@@ -170,8 +163,6 @@ describe("deployments", async () => {
 	`;
 		expect(stdout).toMatchInlineSnapshot(`
 			"🚧\`wrangler rollback\` is a beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose
-
-
 			Successfully rolled back to Deployment ID: 00000000-0000-0000-0000-000000000000
 			Current Deployment ID: 00000000-0000-0000-0000-000000000000"
 		`);
@@ -187,18 +178,14 @@ describe("deployments", async () => {
 	`;
 		expect(stdout).toMatchInlineSnapshot(`
 			"🚧\`wrangler deployments\` is a beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose
-
-
 			Deployment ID: 00000000-0000-0000-0000-000000000000
 			Created on:    TIMESTAMP
 			Author:        person@example.com
 			Source:        Upload from Wrangler 🤠
-
 			Deployment ID: 00000000-0000-0000-0000-000000000000
 			Created on:    TIMESTAMP
 			Author:        person@example.com
 			Source:        Upload from Wrangler 🤠
-
 			Deployment ID: 00000000-0000-0000-0000-000000000000
 			Created on:    TIMESTAMP
 			Author:        person@example.com
@@ -222,9 +209,10 @@ describe("deployments", async () => {
 			Successfully deleted smoke-test-worker"
 		`);
 		expect(stderr).toMatchInlineSnapshot('""');
-		await setTimeout(10_000);
-		await expect(
-			fetch(`https://${workerName}.${workersDev}`).then((r) => r.status)
-		).resolves.toBe(404);
+		await retry(() =>
+			expect(
+				fetch(`https://${workerName}.${workersDev}`).then((r) => r.status)
+			).resolves.toBe(404)
+		);
 	});
 });