Extract all helper methods into smoke-test-utils and add version to s…

…moke test application
aws-observability · May 18, 2022 · a56f22d · a56f22d
1 parent 45079f6
commit a56f22d
Show file tree

Hide file tree

Showing 6 changed files with 153 additions and 114 deletions.
diff --git a/.github/scripts/upload_smoke_test.sh b/.github/scripts/upload_smoke_test.sh
@@ -1,2 +1,3 @@
 bucket=$1
-aws s3api put-object --bucket $bucket --key "smoke.html" --body processed_smoke.html --content-type "text/html"
+key=smoke-$(npm pkg get version | sed 's/"//g').html
+aws s3api put-object --bucket $bucket --key "$key" --body processed_smoke.html --content-type "text/html"
diff --git a/package.json b/package.json
@@ -51,16 +51,11 @@
         "preinteg:local:nightwatch:firefox": "http-server ./build/dev -s &",
         "integ:local:nightwatch:firefox": "nightwatch -e firefox",
         "postinteg:local:nightwatch:firefox": "kill $(lsof -t -i:8080)",
-        "smoke:local:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.local.config.ts",
-        "smoke:local": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.local.config.ts --headed",
-<<<<<<< HEAD
-        "smoke": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --headed",
-        "smoke:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test",
+        "smoke:local:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME VERSION=$npm_package_version npx playwright test --config=playwright.local.config.ts",
+        "smoke:local": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME VERSION=$npm_package_version npx playwright test --config=playwright.local.config.ts --headed",
+        "smoke": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME VERSION=$npm_package_version npx playwright test --config=playwright.config.ts --headed",
+        "smoke:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME VERSION=$npm_package_version npx playwright test --config=playwright.config.ts",
         "prepare": "husky install"
-=======
-        "smoke": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.config.ts --headed",
-        "smoke:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.config.ts"
->>>>>>> a4de281 (test: Integrate automated smoke test into release workflow)
     },
     "devDependencies": {
         "@babel/plugin-transform-runtime": "^7.16.0",

diff --git a/playwright.config.ts b/playwright.config.ts
@@ -5,7 +5,7 @@ const { devices } = require('@playwright/test');
 const config = {
     forbidOnly: !!process.env.CI,
     reporter: 'list',
-    workers: process.env.CI ? 6 : undefined,
+    workers: process.env.CI ? 4 : undefined,
     testDir: 'src/__smoke-test__',
     retries: process.env.CI ? 2 : 2,
     timeout: 300000,

diff --git a/src/__smoke-test__/dataplane-integ.spec.ts b/src/__smoke-test__/dataplane-integ.spec.ts
@@ -1,4 +1,9 @@
 import { test, expect } from '@playwright/test';
+import {
+    getEventsByType,
+    getUrl,
+    isDataPlaneRequest
+} from 'test-utils/smoke-test-utils';
 import {
     PERFORMANCE_NAVIGATION_EVENT_TYPE,
     PERFORMANCE_RESOURCE_EVENT_TYPE,
@@ -9,26 +14,9 @@ import {
 // Environment variables set through CLI command
 const ENDPOINT = process.env.ENDPOINT;
 const MONITOR_ID = process.env.MONITOR;
-const TEST_URL = process.env.URL || 'http://localhost:9000/smoke_local.html';
-
+const TEST_URL = getUrl(process.env.URL, process.env.VERSION);
 const TARGET_URL = ENDPOINT + MONITOR_ID + '/';
 
-function getEventsByType(requestBody, eventType) {
-    return requestBody.RumEvents.filter((e) => e.type === eventType);
-}
-
-/**
- * Returns true if the request is a successful PutRumEvents request
- */
-function isDataPlaneRequest(response): boolean {
-    const request = response.request();
-    return (
-        request.method() === 'POST' &&
-        response.status() === 200 &&
-        response.url() === TARGET_URL
-    );
-}
-
 test('when web client calls PutRumEvents then the response code is 200', async ({
     page
 }) => {
@@ -37,7 +25,7 @@ test('when web client calls PutRumEvents then the response code is 200', async (
 
     // Test will timeout if no successful dataplane request is found
     await page.waitForResponse(async (response) =>
-        isDataPlaneRequest(response)
+        isDataPlaneRequest(response, TARGET_URL)
     );
 });
 
@@ -55,7 +43,7 @@ test('when web client calls PutRumEvents then the payload contains all events',
 
     // Test will timeout if no successful dataplane request is found
     const response = await page.waitForResponse(async (response) =>
-        isDataPlaneRequest(response)
+        isDataPlaneRequest(response, TARGET_URL)
     );
 
     // Parse payload to verify event count

diff --git a/src/__smoke-test__/ingestion-integ.spec.ts b/src/__smoke-test__/ingestion-integ.spec.ts
@@ -1,9 +1,5 @@
 import { test, expect } from '@playwright/test';
-import {
-    RUMClient,
-    GetAppMonitorDataCommand,
-    GetAppMonitorDataCommandInput
-} from '@aws-sdk/client-rum';
+import { RUMClient } from '@aws-sdk/client-rum';
 import {
     HTTP_EVENT_TYPE,
     JS_ERROR_EVENT_TYPE,
@@ -14,87 +10,25 @@ import {
     PAGE_VIEW_EVENT_TYPE,
     SESSION_START_EVENT_TYPE
 } from '../plugins/utils/constant';
+import {
+    getEventIds,
+    getEventsByType,
+    getUrl,
+    isDataPlaneRequest,
+    verifyIngestionWithRetry
+} from 'test-utils/smoke-test-utils';
 
 // Environment variables set through CLI command
 const ENDPOINT = process.env.ENDPOINT;
 const MONITOR_ID = process.env.MONITOR;
-const TEST_URL = process.env.URL || 'http://localhost:9000/smoke_local.html';
+const TEST_URL = getUrl(process.env.URL, process.env.VERSION);
 const MONITOR_NAME = process.env.NAME;
-
 const REGION = ENDPOINT.split('.')[2];
 const TARGET_URL = ENDPOINT + MONITOR_ID + '/';
 
 // Parse region from endpoint
 const rumClient = new RUMClient({ region: REGION });
 
-function getEventsByType(requestBody, eventType) {
-    return requestBody.RumEvents.filter((e) => e.type === eventType);
-}
-function getEventIds(events) {
-    return events.map((e) => e.id);
-}
-
-/**
- * Returns true if the request is a successful PutRumEvents request
- */
-function isDataPlaneRequest(response): boolean {
-    const request = response.request();
-    return (
-        request.method() === 'POST' &&
-        response.status() === 200 &&
-        response.url() === TARGET_URL
-    );
-}
-
-/** Returns true when all events were ingested */
-async function verifyIngestionWithRetry(eventIds, timestamp, retryCount) {
-    while (true) {
-        if (retryCount === 0) {
-            console.log('Retry attempt exhausted.');
-            return false;
-        }
-        try {
-            await isEachEventIngested(eventIds, timestamp);
-            return true;
-        } catch (error) {
-            retryCount -= 1;
-            console.log(`${error.message} Waiting for next retry.`);
-            await new Promise((r) => setTimeout(r, 60000));
-        }
-    }
-}
-
-async function isEachEventIngested(eventIds, timestamp) {
-    let ingestedEvents = new Set();
-    const input: GetAppMonitorDataCommandInput = {
-        Name: MONITOR_NAME,
-        TimeRange: {
-            After: timestamp
-        }
-    };
-    let command = new GetAppMonitorDataCommand(input);
-    // Running tests in parallel require pagination logic, as several test cases have the same timestamp
-    while (true) {
-        const data = await rumClient.send(command);
-        for (let i = 0; i < data.Events.length; i++) {
-            ingestedEvents.add(JSON.parse(data.Events[i]).event_id);
-        }
-        if (data.NextToken) {
-            input.NextToken = data.NextToken;
-            command = new GetAppMonitorDataCommand(input);
-        } else {
-            // If there are no more pages, we can finish the loop
-            break;
-        }
-    }
-
-    for (let i = 0; i < eventIds.length; i++) {
-        if (!ingestedEvents.has(eventIds[i])) {
-            throw new Error(`Event ${eventIds[i]} not ingested.`);
-        }
-    }
-}
-
 // Run the tests in parallel
 test.describe.configure({ mode: 'parallel' });
 test('when session start event is sent then event is ingested', async ({
@@ -107,7 +41,7 @@ test('when session start event is sent then event is ingested', async ({
 
     // Test will timeout if no successful dataplane request is found
     const response = await page.waitForResponse(async (response) =>
-        isDataPlaneRequest(response)
+        isDataPlaneRequest(response, TARGET_URL)
     );
 
     // Parse payload to verify event count
@@ -117,8 +51,10 @@ test('when session start event is sent then event is ingested', async ({
     const eventIds = getEventIds(session);
 
     const isIngestionCompleted = await verifyIngestionWithRetry(
+        rumClient,
         eventIds,
         timestamp,
+        MONITOR_NAME,
         5
     );
     expect(isIngestionCompleted).toEqual(true);
@@ -132,7 +68,7 @@ test('when resource event is sent then event is ingested', async ({ page }) => {
 
     // Test will timeout if no successful dataplane request is found
     const response = await page.waitForResponse(async (response) =>
-        isDataPlaneRequest(response)
+        isDataPlaneRequest(response, TARGET_URL)
     );
 
     // Parse payload to verify event count
@@ -145,8 +81,10 @@ test('when resource event is sent then event is ingested', async ({ page }) => {
     const eventIds = getEventIds(resource);
 
     const isIngestionCompleted = await verifyIngestionWithRetry(
+        rumClient,
         eventIds,
         timestamp,
+        MONITOR_NAME,
         5
     );
     expect(isIngestionCompleted).toEqual(true);
@@ -162,7 +100,7 @@ test('when LCP event is sent then event is ingested', async ({ page }) => {
 
     // Test will timeout if no successful dataplane request is found
     const response = await page.waitForResponse(async (response) =>
-        isDataPlaneRequest(response)
+        isDataPlaneRequest(response, TARGET_URL)
     );
 
     // Parse payload to verify event count
@@ -173,8 +111,10 @@ test('when LCP event is sent then event is ingested', async ({ page }) => {
 
     expect(eventIds.length).not.toEqual(0);
     const isIngestionCompleted = await verifyIngestionWithRetry(
+        rumClient,
         eventIds,
         timestamp,
+        MONITOR_NAME,
         5
     );
     expect(isIngestionCompleted).toEqual(true);
@@ -190,7 +130,7 @@ test('when FID event is sent then event is ingested', async ({ page }) => {
 
     // Test will timeout if no successful dataplane request is found
     const response = await page.waitForResponse(async (response) =>
-        isDataPlaneRequest(response)
+        isDataPlaneRequest(response, TARGET_URL)
     );
 
     // Parse payload to verify event count
@@ -201,8 +141,10 @@ test('when FID event is sent then event is ingested', async ({ page }) => {
 
     expect(eventIds.length).not.toEqual(0);
     const isIngestionCompleted = await verifyIngestionWithRetry(
+        rumClient,
         eventIds,
         timestamp,
+        MONITOR_NAME,
         5
     );
     expect(isIngestionCompleted).toEqual(true);
@@ -220,7 +162,7 @@ test('when navigation events are sent then events are ingested', async ({
 
     // Test will timeout if no successful dataplane request is found
     const response = await page.waitForResponse(async (response) =>
-        isDataPlaneRequest(response)
+        isDataPlaneRequest(response, TARGET_URL)
     );
 
     // Parse payload to verify event count
@@ -235,8 +177,10 @@ test('when navigation events are sent then events are ingested', async ({
     // One initial load, one route change
     expect(eventIds.length).toEqual(2);
     const isIngestionCompleted = await verifyIngestionWithRetry(
+        rumClient,
         eventIds,
         timestamp,
+        MONITOR_NAME,
         5
     );
     expect(isIngestionCompleted).toEqual(true);
@@ -254,7 +198,7 @@ test('when page view event is sent then the event is ingested', async ({
 
     // Test will timeout if no successful dataplane request is found
     const response = await page.waitForResponse(async (response) =>
-        isDataPlaneRequest(response)
+        isDataPlaneRequest(response, TARGET_URL)
     );
 
     // Parse payload to verify event count
@@ -266,8 +210,10 @@ test('when page view event is sent then the event is ingested', async ({
     // One initial load, one route change
     expect(eventIds.length).toEqual(2);
     const isIngestionCompleted = await verifyIngestionWithRetry(
+        rumClient,
         eventIds,
         timestamp,
+        MONITOR_NAME,
         5
     );
     expect(isIngestionCompleted).toEqual(true);
@@ -289,7 +235,7 @@ test('when error events are sent then the events are ingested', async ({
 
     // Test will timeout if no successful dataplane request is found
     const response = await page.waitForResponse(async (response) =>
-        isDataPlaneRequest(response)
+        isDataPlaneRequest(response, TARGET_URL)
     );
 
     // Parse payload to verify event count
@@ -301,8 +247,10 @@ test('when error events are sent then the events are ingested', async ({
     // Expect three js error events
     expect(eventIds.length).toEqual(3);
     const isIngestionCompleted = await verifyIngestionWithRetry(
+        rumClient,
         eventIds,
         timestamp,
+        MONITOR_NAME,
         5
     );
     expect(isIngestionCompleted).toEqual(true);
@@ -322,7 +270,7 @@ test('when http events are sent then the events are ingested', async ({
 
     // Test will timeout if no successful dataplane request is found
     const response = await page.waitForResponse(async (response) =>
-        isDataPlaneRequest(response)
+        isDataPlaneRequest(response, TARGET_URL)
     );
 
     // Parse payload to verify event count
@@ -334,8 +282,10 @@ test('when http events are sent then the events are ingested', async ({
     // Expect three js error events
     expect(eventIds.length).toEqual(2);
     const isIngestionCompleted = await verifyIngestionWithRetry(
+        rumClient,
         eventIds,
         timestamp,
+        MONITOR_NAME,
         5
     );
     expect(isIngestionCompleted).toEqual(true);