From 651dec6db2114abd975f0280f0e09a4606a8fee9 Mon Sep 17 00:00:00 2001 From: Reno Seo Date: Mon, 9 May 2022 16:10:48 -0700 Subject: [PATCH 1/3] test: Integrate automated smoke test into release workflow --- .github/scripts/update_smoke_test.sh | 11 + .github/scripts/upload_smoke_test.sh | 2 + .github/workflows/cd.yaml | 37 +++- .gitignore | 2 +- app/smoke.html | 221 +++++++++++++++++++++ package.json | 5 + playwright.config.ts | 2 + playwright.local.config.ts | 1 + src/__smoke-test__/dataplane-integ.spec.ts | 2 +- src/__smoke-test__/ingestion-integ.spec.ts | 6 +- 10 files changed, 283 insertions(+), 6 deletions(-) create mode 100644 .github/scripts/update_smoke_test.sh create mode 100644 .github/scripts/upload_smoke_test.sh create mode 100644 app/smoke.html diff --git a/.github/scripts/update_smoke_test.sh b/.github/scripts/update_smoke_test.sh new file mode 100644 index 00000000..e0725162 --- /dev/null +++ b/.github/scripts/update_smoke_test.sh @@ -0,0 +1,11 @@ +MONITOR_ID=$1 +REGION=$2 +GUEST_ARN=$3 +IDENTITY_POOL=$4 +ENDPOINT=$5 +CDN=$6 +VERSION=$(npm pkg get version | sed 's/"//g')/cwr.js +CDN+=${VERSION} +awk '{sub(/\$MONITOR_ID/,MONITOR_ID);sub(/\$REGION/,REGION);sub(/\$CDN/,CDN);sub(/\$GUEST_ARN/,GUEST_ARN);sub(/\$IDENTITY_POOL/,IDENTITY_POOL);sub(/\$ENDPOINT/,ENDPOINT);}1' \ + MONITOR_ID="'$MONITOR_ID'" REGION="'$REGION'" CDN="'$CDN'" GUEST_ARN="'$GUEST_ARN'" IDENTITY_POOL="'$IDENTITY_POOL'" ENDPOINT="'$ENDPOINT'" app/smoke.html + \ No newline at end of file diff --git a/.github/scripts/upload_smoke_test.sh b/.github/scripts/upload_smoke_test.sh new file mode 100644 index 00000000..4df3eac5 --- /dev/null +++ b/.github/scripts/upload_smoke_test.sh @@ -0,0 +1,2 @@ +bucket=$1 +aws s3api put-object --bucket $bucket --key "smoke.html" --body processed_smoke.html --content-type "text/html" diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index 0a82436d..44e599c8 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -19,7 +19,7 @@ jobs: node-version: '16.x' registry-url: 'https://registry.npmjs.org' - - name: Fetch AWS Credentials + - name: Fetch AWS Credentials for Deployment run: | export AWS_ROLE_ARN=${{ secrets.ROLE }} export AWS_WEB_IDENTITY_TOKEN_FILE=/tmp/awscreds @@ -50,6 +50,41 @@ jobs: chmod u+x .github/scripts/deploy.sh .github/scripts/deploy.sh ${{ secrets.BUCKET }} + - name: Fetch AWS Credentials for Smoke Test + run: | + export AWS_ROLE_ARN=${{ secrets.SMOKE_TEST_ROLE }} + export AWS_WEB_IDENTITY_TOKEN_FILE=/tmp/awscreds + export AWS_DEFAULT_REGION=us-east-1 + + echo AWS_WEB_IDENTITY_TOKEN_FILE=$AWS_WEB_IDENTITY_TOKEN_FILE >> $GITHUB_ENV + echo AWS_ROLE_ARN=$AWS_ROLE_ARN >> $GITHUB_ENV + echo AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION >> $GITHUB_ENV + + curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" "$ACTIONS_ID_TOKEN_REQUEST_URL" | jq -r '.value' > $AWS_WEB_IDENTITY_TOKEN_FILE + + - name: Update Smoke Test GAMMA + id: update-smoke-test-gamma + run: | + chmod u+x .github/scripts/update_smoke_test.sh + .github/scripts/update_smoke_test.sh ${{ secrets.SMOKE_MONITOR }} ${{ secrets.SMOKE_REGION }} ${{ secrets.SMOKE_ARN }} ${{ secrets.SMOKE_IDENTITY }} ${{ secrets.CONFIG_ENDPOINT }} ${{ secrets.CDN }} >> processed_smoke.html + + - name: Upload Smoke Test + id: upload-smoke-test + run: | + chmod u+x .github/scripts/upload_smoke_test.sh + .github/scripts/upload_smoke_test.sh ${{ secrets.SMOKE_BUCKET }} + + - name: Install PlayWright + run: npx playwright install --with-deps chromium + + - name: Run Smoke Test + env: + URL: ${{ secrets.SMOKE_URL }} + MONITOR: ${{ secrets.SMOKE_MONITOR }} + ENDPOINT: ${{ secrets.SMOKE_ENDPOINT }} + NAME: ${{ secrets.SMOKE_MONITOR_NAME }} + run: npm run smoke:headless + - name: Publish to NPM run: npm publish env: diff --git a/.gitignore b/.gitignore index 90ade100..0462e73f 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,6 @@ chromedriver.log geckodriver.log safaridriver.log tests_output -app/smoke.html logs .idea +app/smoke_local.html diff --git a/app/smoke.html b/app/smoke.html new file mode 100644 index 00000000..abe89281 --- /dev/null +++ b/app/smoke.html @@ -0,0 +1,221 @@ + + + + + RUM Smoke Test + + + + + + + + + +

This application is used for RUM smoke testing.

+
+ + + + + + + + +
+ + + +
+ + + + +
+ + + +
+ + diff --git a/package.json b/package.json index aade0e3b..016af54d 100644 --- a/package.json +++ b/package.json @@ -53,9 +53,14 @@ "postinteg:local:nightwatch:firefox": "kill $(lsof -t -i:8080)", "smoke:local:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.local.config.ts", "smoke:local": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.local.config.ts --headed", +<<<<<<< HEAD "smoke": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --headed", "smoke:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test", "prepare": "husky install" +======= + "smoke": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.config.ts --headed", + "smoke:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.config.ts" +>>>>>>> a4de281 (test: Integrate automated smoke test into release workflow) }, "devDependencies": { "@babel/plugin-transform-runtime": "^7.16.0", diff --git a/playwright.config.ts b/playwright.config.ts index e10a29ee..9fac1497 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -4,6 +4,8 @@ const { devices } = require('@playwright/test'); const config = { forbidOnly: !!process.env.CI, + reporter: 'list', + workers: process.env.CI ? 6 : undefined, testDir: 'src/__smoke-test__', retries: process.env.CI ? 2 : 2, timeout: 300000, diff --git a/playwright.local.config.ts b/playwright.local.config.ts index b74ad71c..624d7ccd 100644 --- a/playwright.local.config.ts +++ b/playwright.local.config.ts @@ -5,6 +5,7 @@ const { devices } = require('@playwright/test'); const config = { forbidOnly: !!process.env.CI, testDir: 'src/__smoke-test__', + reporter: 'list', retries: process.env.CI ? 2 : 2, timeout: 300000, webServer: { diff --git a/src/__smoke-test__/dataplane-integ.spec.ts b/src/__smoke-test__/dataplane-integ.spec.ts index 91196902..6237acf1 100644 --- a/src/__smoke-test__/dataplane-integ.spec.ts +++ b/src/__smoke-test__/dataplane-integ.spec.ts @@ -9,7 +9,7 @@ import { // Environment variables set through CLI command const ENDPOINT = process.env.ENDPOINT; const MONITOR_ID = process.env.MONITOR; -const TEST_URL = process.env.URL || 'http://localhost:9000/smoke.html'; +const TEST_URL = process.env.URL || 'http://localhost:9000/smoke_local.html'; const TARGET_URL = ENDPOINT + MONITOR_ID + '/'; diff --git a/src/__smoke-test__/ingestion-integ.spec.ts b/src/__smoke-test__/ingestion-integ.spec.ts index ab5e7ef2..869bd16f 100644 --- a/src/__smoke-test__/ingestion-integ.spec.ts +++ b/src/__smoke-test__/ingestion-integ.spec.ts @@ -18,7 +18,7 @@ import { // Environment variables set through CLI command const ENDPOINT = process.env.ENDPOINT; const MONITOR_ID = process.env.MONITOR; -const TEST_URL = process.env.URL || 'http://localhost:9000/smoke.html'; +const TEST_URL = process.env.URL || 'http://localhost:9000/smoke_local.html'; const MONITOR_NAME = process.env.NAME; const REGION = ENDPOINT.split('.')[2]; @@ -157,7 +157,7 @@ test('when LCP event is sent then event is ingested', async ({ page }) => { // Open page await page.goto(TEST_URL); - const clearButton = page.locator('[id=clearRequestResponse]'); + const clearButton = page.locator('[id=dummyButton]'); await clearButton.click(); // Test will timeout if no successful dataplane request is found @@ -185,7 +185,7 @@ test('when FID event is sent then event is ingested', async ({ page }) => { // Open page await page.goto(TEST_URL); - const clearButton = page.locator('[id=clearRequestResponse]'); + const clearButton = page.locator('[id=dummyButton]'); await clearButton.click(); // Test will timeout if no successful dataplane request is found From 45079f64b22b41efb8a7686bdc8e294c396219ff Mon Sep 17 00:00:00 2001 From: Reno Seo Date: Wed, 11 May 2022 13:44:37 -0700 Subject: [PATCH 2/3] fix: Change step names to remove Gamma and add more details --- .github/workflows/cd.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index 44e599c8..dad4291e 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -62,13 +62,13 @@ jobs: curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" "$ACTIONS_ID_TOKEN_REQUEST_URL" | jq -r '.value' > $AWS_WEB_IDENTITY_TOKEN_FILE - - name: Update Smoke Test GAMMA - id: update-smoke-test-gamma + - name: Update Smoke Test Application + id: update-smoke-test run: | chmod u+x .github/scripts/update_smoke_test.sh .github/scripts/update_smoke_test.sh ${{ secrets.SMOKE_MONITOR }} ${{ secrets.SMOKE_REGION }} ${{ secrets.SMOKE_ARN }} ${{ secrets.SMOKE_IDENTITY }} ${{ secrets.CONFIG_ENDPOINT }} ${{ secrets.CDN }} >> processed_smoke.html - - name: Upload Smoke Test + - name: Upload Smoke Test to CloudFront id: upload-smoke-test run: | chmod u+x .github/scripts/upload_smoke_test.sh From a56f22d352556bce1144d51e03e5b80e26f401f2 Mon Sep 17 00:00:00 2001 From: Reno Seo Date: Tue, 17 May 2022 16:24:50 -0700 Subject: [PATCH 3/3] Extract all helper methods into smoke-test-utils and add version to smoke test application --- .github/scripts/upload_smoke_test.sh | 3 +- package.json | 13 +-- playwright.config.ts | 2 +- src/__smoke-test__/dataplane-integ.spec.ts | 28 ++--- src/__smoke-test__/ingestion-integ.spec.ts | 116 ++++++--------------- src/test-utils/smoke-test-utils.ts | 105 +++++++++++++++++++ 6 files changed, 153 insertions(+), 114 deletions(-) create mode 100644 src/test-utils/smoke-test-utils.ts diff --git a/.github/scripts/upload_smoke_test.sh b/.github/scripts/upload_smoke_test.sh index 4df3eac5..af387310 100644 --- a/.github/scripts/upload_smoke_test.sh +++ b/.github/scripts/upload_smoke_test.sh @@ -1,2 +1,3 @@ bucket=$1 -aws s3api put-object --bucket $bucket --key "smoke.html" --body processed_smoke.html --content-type "text/html" +key=smoke-$(npm pkg get version | sed 's/"//g').html +aws s3api put-object --bucket $bucket --key "$key" --body processed_smoke.html --content-type "text/html" diff --git a/package.json b/package.json index 016af54d..ee382185 100644 --- a/package.json +++ b/package.json @@ -51,16 +51,11 @@ "preinteg:local:nightwatch:firefox": "http-server ./build/dev -s &", "integ:local:nightwatch:firefox": "nightwatch -e firefox", "postinteg:local:nightwatch:firefox": "kill $(lsof -t -i:8080)", - "smoke:local:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.local.config.ts", - "smoke:local": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.local.config.ts --headed", -<<<<<<< HEAD - "smoke": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --headed", - "smoke:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test", + "smoke:local:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME VERSION=$npm_package_version npx playwright test --config=playwright.local.config.ts", + "smoke:local": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME VERSION=$npm_package_version npx playwright test --config=playwright.local.config.ts --headed", + "smoke": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME VERSION=$npm_package_version npx playwright test --config=playwright.config.ts --headed", + "smoke:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME VERSION=$npm_package_version npx playwright test --config=playwright.config.ts", "prepare": "husky install" -======= - "smoke": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.config.ts --headed", - "smoke:headless": "cross-env URL=$URL MONITOR_ID=$MONITOR ENDPOINT=$ENDPOINT NAME=$NAME npx playwright test --config=playwright.config.ts" ->>>>>>> a4de281 (test: Integrate automated smoke test into release workflow) }, "devDependencies": { "@babel/plugin-transform-runtime": "^7.16.0", diff --git a/playwright.config.ts b/playwright.config.ts index 9fac1497..8b0be1ae 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -5,7 +5,7 @@ const { devices } = require('@playwright/test'); const config = { forbidOnly: !!process.env.CI, reporter: 'list', - workers: process.env.CI ? 6 : undefined, + workers: process.env.CI ? 4 : undefined, testDir: 'src/__smoke-test__', retries: process.env.CI ? 2 : 2, timeout: 300000, diff --git a/src/__smoke-test__/dataplane-integ.spec.ts b/src/__smoke-test__/dataplane-integ.spec.ts index 6237acf1..a663a6ea 100644 --- a/src/__smoke-test__/dataplane-integ.spec.ts +++ b/src/__smoke-test__/dataplane-integ.spec.ts @@ -1,4 +1,9 @@ import { test, expect } from '@playwright/test'; +import { + getEventsByType, + getUrl, + isDataPlaneRequest +} from 'test-utils/smoke-test-utils'; import { PERFORMANCE_NAVIGATION_EVENT_TYPE, PERFORMANCE_RESOURCE_EVENT_TYPE, @@ -9,26 +14,9 @@ import { // Environment variables set through CLI command const ENDPOINT = process.env.ENDPOINT; const MONITOR_ID = process.env.MONITOR; -const TEST_URL = process.env.URL || 'http://localhost:9000/smoke_local.html'; - +const TEST_URL = getUrl(process.env.URL, process.env.VERSION); const TARGET_URL = ENDPOINT + MONITOR_ID + '/'; -function getEventsByType(requestBody, eventType) { - return requestBody.RumEvents.filter((e) => e.type === eventType); -} - -/** - * Returns true if the request is a successful PutRumEvents request - */ -function isDataPlaneRequest(response): boolean { - const request = response.request(); - return ( - request.method() === 'POST' && - response.status() === 200 && - response.url() === TARGET_URL - ); -} - test('when web client calls PutRumEvents then the response code is 200', async ({ page }) => { @@ -37,7 +25,7 @@ test('when web client calls PutRumEvents then the response code is 200', async ( // Test will timeout if no successful dataplane request is found await page.waitForResponse(async (response) => - isDataPlaneRequest(response) + isDataPlaneRequest(response, TARGET_URL) ); }); @@ -55,7 +43,7 @@ test('when web client calls PutRumEvents then the payload contains all events', // Test will timeout if no successful dataplane request is found const response = await page.waitForResponse(async (response) => - isDataPlaneRequest(response) + isDataPlaneRequest(response, TARGET_URL) ); // Parse payload to verify event count diff --git a/src/__smoke-test__/ingestion-integ.spec.ts b/src/__smoke-test__/ingestion-integ.spec.ts index 869bd16f..394968b6 100644 --- a/src/__smoke-test__/ingestion-integ.spec.ts +++ b/src/__smoke-test__/ingestion-integ.spec.ts @@ -1,9 +1,5 @@ import { test, expect } from '@playwright/test'; -import { - RUMClient, - GetAppMonitorDataCommand, - GetAppMonitorDataCommandInput -} from '@aws-sdk/client-rum'; +import { RUMClient } from '@aws-sdk/client-rum'; import { HTTP_EVENT_TYPE, JS_ERROR_EVENT_TYPE, @@ -14,87 +10,25 @@ import { PAGE_VIEW_EVENT_TYPE, SESSION_START_EVENT_TYPE } from '../plugins/utils/constant'; +import { + getEventIds, + getEventsByType, + getUrl, + isDataPlaneRequest, + verifyIngestionWithRetry +} from 'test-utils/smoke-test-utils'; // Environment variables set through CLI command const ENDPOINT = process.env.ENDPOINT; const MONITOR_ID = process.env.MONITOR; -const TEST_URL = process.env.URL || 'http://localhost:9000/smoke_local.html'; +const TEST_URL = getUrl(process.env.URL, process.env.VERSION); const MONITOR_NAME = process.env.NAME; - const REGION = ENDPOINT.split('.')[2]; const TARGET_URL = ENDPOINT + MONITOR_ID + '/'; // Parse region from endpoint const rumClient = new RUMClient({ region: REGION }); -function getEventsByType(requestBody, eventType) { - return requestBody.RumEvents.filter((e) => e.type === eventType); -} -function getEventIds(events) { - return events.map((e) => e.id); -} - -/** - * Returns true if the request is a successful PutRumEvents request - */ -function isDataPlaneRequest(response): boolean { - const request = response.request(); - return ( - request.method() === 'POST' && - response.status() === 200 && - response.url() === TARGET_URL - ); -} - -/** Returns true when all events were ingested */ -async function verifyIngestionWithRetry(eventIds, timestamp, retryCount) { - while (true) { - if (retryCount === 0) { - console.log('Retry attempt exhausted.'); - return false; - } - try { - await isEachEventIngested(eventIds, timestamp); - return true; - } catch (error) { - retryCount -= 1; - console.log(`${error.message} Waiting for next retry.`); - await new Promise((r) => setTimeout(r, 60000)); - } - } -} - -async function isEachEventIngested(eventIds, timestamp) { - let ingestedEvents = new Set(); - const input: GetAppMonitorDataCommandInput = { - Name: MONITOR_NAME, - TimeRange: { - After: timestamp - } - }; - let command = new GetAppMonitorDataCommand(input); - // Running tests in parallel require pagination logic, as several test cases have the same timestamp - while (true) { - const data = await rumClient.send(command); - for (let i = 0; i < data.Events.length; i++) { - ingestedEvents.add(JSON.parse(data.Events[i]).event_id); - } - if (data.NextToken) { - input.NextToken = data.NextToken; - command = new GetAppMonitorDataCommand(input); - } else { - // If there are no more pages, we can finish the loop - break; - } - } - - for (let i = 0; i < eventIds.length; i++) { - if (!ingestedEvents.has(eventIds[i])) { - throw new Error(`Event ${eventIds[i]} not ingested.`); - } - } -} - // Run the tests in parallel test.describe.configure({ mode: 'parallel' }); test('when session start event is sent then event is ingested', async ({ @@ -107,7 +41,7 @@ test('when session start event is sent then event is ingested', async ({ // Test will timeout if no successful dataplane request is found const response = await page.waitForResponse(async (response) => - isDataPlaneRequest(response) + isDataPlaneRequest(response, TARGET_URL) ); // Parse payload to verify event count @@ -117,8 +51,10 @@ test('when session start event is sent then event is ingested', async ({ const eventIds = getEventIds(session); const isIngestionCompleted = await verifyIngestionWithRetry( + rumClient, eventIds, timestamp, + MONITOR_NAME, 5 ); expect(isIngestionCompleted).toEqual(true); @@ -132,7 +68,7 @@ test('when resource event is sent then event is ingested', async ({ page }) => { // Test will timeout if no successful dataplane request is found const response = await page.waitForResponse(async (response) => - isDataPlaneRequest(response) + isDataPlaneRequest(response, TARGET_URL) ); // Parse payload to verify event count @@ -145,8 +81,10 @@ test('when resource event is sent then event is ingested', async ({ page }) => { const eventIds = getEventIds(resource); const isIngestionCompleted = await verifyIngestionWithRetry( + rumClient, eventIds, timestamp, + MONITOR_NAME, 5 ); expect(isIngestionCompleted).toEqual(true); @@ -162,7 +100,7 @@ test('when LCP event is sent then event is ingested', async ({ page }) => { // Test will timeout if no successful dataplane request is found const response = await page.waitForResponse(async (response) => - isDataPlaneRequest(response) + isDataPlaneRequest(response, TARGET_URL) ); // Parse payload to verify event count @@ -173,8 +111,10 @@ test('when LCP event is sent then event is ingested', async ({ page }) => { expect(eventIds.length).not.toEqual(0); const isIngestionCompleted = await verifyIngestionWithRetry( + rumClient, eventIds, timestamp, + MONITOR_NAME, 5 ); expect(isIngestionCompleted).toEqual(true); @@ -190,7 +130,7 @@ test('when FID event is sent then event is ingested', async ({ page }) => { // Test will timeout if no successful dataplane request is found const response = await page.waitForResponse(async (response) => - isDataPlaneRequest(response) + isDataPlaneRequest(response, TARGET_URL) ); // Parse payload to verify event count @@ -201,8 +141,10 @@ test('when FID event is sent then event is ingested', async ({ page }) => { expect(eventIds.length).not.toEqual(0); const isIngestionCompleted = await verifyIngestionWithRetry( + rumClient, eventIds, timestamp, + MONITOR_NAME, 5 ); expect(isIngestionCompleted).toEqual(true); @@ -220,7 +162,7 @@ test('when navigation events are sent then events are ingested', async ({ // Test will timeout if no successful dataplane request is found const response = await page.waitForResponse(async (response) => - isDataPlaneRequest(response) + isDataPlaneRequest(response, TARGET_URL) ); // Parse payload to verify event count @@ -235,8 +177,10 @@ test('when navigation events are sent then events are ingested', async ({ // One initial load, one route change expect(eventIds.length).toEqual(2); const isIngestionCompleted = await verifyIngestionWithRetry( + rumClient, eventIds, timestamp, + MONITOR_NAME, 5 ); expect(isIngestionCompleted).toEqual(true); @@ -254,7 +198,7 @@ test('when page view event is sent then the event is ingested', async ({ // Test will timeout if no successful dataplane request is found const response = await page.waitForResponse(async (response) => - isDataPlaneRequest(response) + isDataPlaneRequest(response, TARGET_URL) ); // Parse payload to verify event count @@ -266,8 +210,10 @@ test('when page view event is sent then the event is ingested', async ({ // One initial load, one route change expect(eventIds.length).toEqual(2); const isIngestionCompleted = await verifyIngestionWithRetry( + rumClient, eventIds, timestamp, + MONITOR_NAME, 5 ); expect(isIngestionCompleted).toEqual(true); @@ -289,7 +235,7 @@ test('when error events are sent then the events are ingested', async ({ // Test will timeout if no successful dataplane request is found const response = await page.waitForResponse(async (response) => - isDataPlaneRequest(response) + isDataPlaneRequest(response, TARGET_URL) ); // Parse payload to verify event count @@ -301,8 +247,10 @@ test('when error events are sent then the events are ingested', async ({ // Expect three js error events expect(eventIds.length).toEqual(3); const isIngestionCompleted = await verifyIngestionWithRetry( + rumClient, eventIds, timestamp, + MONITOR_NAME, 5 ); expect(isIngestionCompleted).toEqual(true); @@ -322,7 +270,7 @@ test('when http events are sent then the events are ingested', async ({ // Test will timeout if no successful dataplane request is found const response = await page.waitForResponse(async (response) => - isDataPlaneRequest(response) + isDataPlaneRequest(response, TARGET_URL) ); // Parse payload to verify event count @@ -334,8 +282,10 @@ test('when http events are sent then the events are ingested', async ({ // Expect three js error events expect(eventIds.length).toEqual(2); const isIngestionCompleted = await verifyIngestionWithRetry( + rumClient, eventIds, timestamp, + MONITOR_NAME, 5 ); expect(isIngestionCompleted).toEqual(true); diff --git a/src/test-utils/smoke-test-utils.ts b/src/test-utils/smoke-test-utils.ts new file mode 100644 index 00000000..a9d3a161 --- /dev/null +++ b/src/test-utils/smoke-test-utils.ts @@ -0,0 +1,105 @@ +import { + GetAppMonitorDataCommand, + GetAppMonitorDataCommandInput +} from '@aws-sdk/client-rum'; + +/** Returns filtered events by type */ +export const getEventsByType = (requestBody, eventType) => { + return requestBody.RumEvents.filter((e) => e.type === eventType); +}; + +/** Returns an array of eventIds */ +export const getEventIds = (events) => { + return events.map((e) => e.id); +}; + +/** Returns the smoke test URL with the right version */ +export const getUrl = (test_url, version) => { + if (!test_url) { + return 'http://localhost:9000/smoke_local.html'; + } + const url = new URL(test_url); + if (url.pathname === '/') { + return url + `smoke-${version}.html`; + } else { + return url.toString(); + } +}; + +/** + * Returns true if the request is a successful PutRumEvents request + */ +export const isDataPlaneRequest = (response, targetUrl) => { + const request = response.request(); + return ( + request.method() === 'POST' && + response.status() === 200 && + response.url() === targetUrl + ); +}; + +/** Returns true when all events were ingested */ +export const verifyIngestionWithRetry = async ( + rumClient, + eventIds, + timestamp, + monitorName, + retryCount +) => { + while (true) { + if (retryCount === 0) { + console.log('Retry attempt exhausted.'); + return false; + } + try { + await isEachEventIngested( + rumClient, + eventIds, + timestamp, + monitorName + ); + return true; + } catch (error) { + retryCount -= 1; + console.log(`${error.message} Waiting for next retry.`); + await new Promise((r) => setTimeout(r, 60000)); + } + } +}; + +/** Returns true when every event is ingested */ +export const isEachEventIngested = async ( + rumClient, + eventIds, + timestamp, + monitorName +) => { + let ingestedEvents = new Set(); + const input: GetAppMonitorDataCommandInput = { + Name: monitorName, + TimeRange: { + After: timestamp + } + }; + let command = new GetAppMonitorDataCommand(input); + // Running tests in parallel require pagination logic, as several test cases have the same timestamp + while (true) { + const data = await rumClient.send(command); + for (let i = 0; i < data.Events.length; i++) { + ingestedEvents.add(JSON.parse(data.Events[i]).event_id); + } + if (data.NextToken) { + input.NextToken = data.NextToken; + command = new GetAppMonitorDataCommand(input); + } else { + // If there are no more pages, we can finish the loop + break; + } + } + + for (let i = 0; i < eventIds.length; i++) { + if (!ingestedEvents.has(eventIds[i])) { + throw new Error(`Event ${eventIds[i]} not ingested.`); + } + } +};