From 367e8634fcc8c370af4ca9e337bfb15790bbd9f1 Mon Sep 17 00:00:00 2001 From: John Whitlock Date: Mon, 9 Sep 2024 13:21:46 -0500 Subject: [PATCH 1/7] Check that premium user has subdomain --- e2e-tests/pages/dashboardPage.ts | 5 +++++ e2e-tests/specs/relay-premium-functionality.spec.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/e2e-tests/pages/dashboardPage.ts b/e2e-tests/pages/dashboardPage.ts index 355409f584..07e9f512a8 100644 --- a/e2e-tests/pages/dashboardPage.ts +++ b/e2e-tests/pages/dashboardPage.ts @@ -56,6 +56,7 @@ export class DashboardPage { readonly customMaskDoneButton: Locator; readonly maskCardBottomMeta: Locator; readonly maskCardTrackersCount: Locator; + readonly chooseSubdomain: Locator; constructor(page: Page) { this.page = page; @@ -128,6 +129,7 @@ export class DashboardPage { this.dashBoardWithoutMasksEmail = page.locator( '//section[starts-with(@class, "profile_no-premium-header")]', ); + this.chooseSubdomain = page.locator("id=mpp-choose-subdomain"); // mask card elements this.maskCard = page.getByRole("button", { name: "Generate new mask" }); @@ -218,6 +220,9 @@ export class DashboardPage { if (numberOfMasks === 0) { return; } + // Check that the subdomain has been set for the premium user + expect(await this.chooseSubdomain.count()).toBe(0); + await this.generateNewMaskPremiumButton.click(); await this.premiumDomainMask.click(); diff --git a/e2e-tests/specs/relay-premium-functionality.spec.ts b/e2e-tests/specs/relay-premium-functionality.spec.ts index d031aac67b..8f51ee7da2 100644 --- a/e2e-tests/specs/relay-premium-functionality.spec.ts +++ b/e2e-tests/specs/relay-premium-functionality.spec.ts @@ -40,7 +40,7 @@ test.describe("Premium - General Functionalities, Desktop", () => { ); }); - test("Verify that a premium user can generate a custom mask @health_check", async ({ + test("Verify that a premium user can generate a custom mask @health_check", async ({ dashboardPage, }) => { // When there are zero masks, a random mask must be generated first From c77c404e19743810bbb6e51e20530d112910712d Mon Sep 17 00:00:00 2001 From: John Whitlock Date: Mon, 9 Sep 2024 14:26:19 -0500 Subject: [PATCH 2/7] Remove unused getVerificationCode --- e2e-tests/pages/dashboardPage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/pages/dashboardPage.ts b/e2e-tests/pages/dashboardPage.ts index 07e9f512a8..cb48783f3d 100644 --- a/e2e-tests/pages/dashboardPage.ts +++ b/e2e-tests/pages/dashboardPage.ts @@ -1,5 +1,5 @@ import { Locator, Page, expect } from "@playwright/test"; -import { checkAuthState, getVerificationCode } from "../e2eTestUtils/helpers"; +import { checkAuthState } from "../e2eTestUtils/helpers"; export class DashboardPage { readonly page: Page; From d05fbcc6806cb85dceb9aae78d24d4ec99e87c99 Mon Sep 17 00:00:00 2001 From: John Whitlock Date: Mon, 9 Sep 2024 14:26:44 -0500 Subject: [PATCH 3/7] Check for error condititions in maks creation * Check that the initial number of masks is 0 * Check that the error banner "The mask could not be created." does not appear * Check that exactly one mask was created --- e2e-tests/pages/dashboardPage.ts | 22 +++++++++++++++++-- .../specs/relay-premium-functionality.spec.ts | 1 + 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/e2e-tests/pages/dashboardPage.ts b/e2e-tests/pages/dashboardPage.ts index cb48783f3d..5d0e4ed5d1 100644 --- a/e2e-tests/pages/dashboardPage.ts +++ b/e2e-tests/pages/dashboardPage.ts @@ -57,6 +57,7 @@ export class DashboardPage { readonly maskCardBottomMeta: Locator; readonly maskCardTrackersCount: Locator; readonly chooseSubdomain: Locator; + readonly bannerEmailError: Locator; constructor(page: Page) { this.page = page; @@ -130,6 +131,9 @@ export class DashboardPage { '//section[starts-with(@class, "profile_no-premium-header")]', ); this.chooseSubdomain = page.locator("id=mpp-choose-subdomain"); + this.bannerEmailError = page.getByText( + "The mask could not be created. Please try again.", + ); // mask card elements this.maskCard = page.getByRole("button", { name: "Generate new mask" }); @@ -245,14 +249,28 @@ export class DashboardPage { ? this.generateNewMaskPremiumButton : this.generateNewMaskButton; - // generate a new mask and confirm + const maskCards = this.page.locator(this.maskCardString); + const preMaskCardsCount = await maskCards.count(); + + // generate a new mask await generateMaskBtn.click(); const randomMaskShown = await this.premiumRandomMask.isVisible(); if (randomMaskShown) { await this.premiumRandomMask.click(); } - await this.page.waitForSelector(this.maskCardString, { timeout: 3000 }); + + // Wait for the mask card count to increase, or the error banner + expect( + this.bannerEmailError.or(maskCards.nth(preMaskCardsCount)), + ).toBeVisible({ timeout: 3000 }); + expect( + this.bannerEmailError, + "No mask error banner. If fails, maybe rate-limited?", + ).not.toBeVisible(); + expect(await maskCards, "Mask cards should go up by one").toHaveCount( + preMaskCardsCount + 1, + ); // randomize between 1.5-2.5 secs between each generate to deal with issue of multiple quick clicks await this.page.waitForTimeout(Math.random() * 2500 + 1500); diff --git a/e2e-tests/specs/relay-premium-functionality.spec.ts b/e2e-tests/specs/relay-premium-functionality.spec.ts index 8f51ee7da2..7f36f7f86a 100644 --- a/e2e-tests/specs/relay-premium-functionality.spec.ts +++ b/e2e-tests/specs/relay-premium-functionality.spec.ts @@ -12,6 +12,7 @@ test.describe("Premium - General Functionalities, Desktop", () => { test("Verify that a premium user can make more than 5 masks @health_check", async ({ dashboardPage, }) => { + expect(await dashboardPage.emailMasksUsedAmount.textContent()).toBe("0"); await dashboardPage.generateMask(6, true); await expect From 7de99b58b7be86cc5cf4d23e3a5d8e96491a2cf5 Mon Sep 17 00:00:00 2001 From: John Whitlock Date: Tue, 10 Sep 2024 09:48:45 -0500 Subject: [PATCH 4/7] Speed up bulk mask creation Since we're now waiting for the UI to show the new mask, we don't have to add (much) of a pause. --- e2e-tests/pages/dashboardPage.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/pages/dashboardPage.ts b/e2e-tests/pages/dashboardPage.ts index 5d0e4ed5d1..7c6be6247a 100644 --- a/e2e-tests/pages/dashboardPage.ts +++ b/e2e-tests/pages/dashboardPage.ts @@ -272,8 +272,8 @@ export class DashboardPage { preMaskCardsCount + 1, ); - // randomize between 1.5-2.5 secs between each generate to deal with issue of multiple quick clicks - await this.page.waitForTimeout(Math.random() * 2500 + 1500); + // randomize between .5-1.0 secs between each generate to deal with issue of multiple quick clicks + await this.page.waitForTimeout(Math.random() * 500 + 500); if (await this.closeCornerUpsell.isVisible()) { await this.closeCornerUpsell.click(); } From 37a66e8a2ccfa7e287df74613bbf0424fb415bce Mon Sep 17 00:00:00 2001 From: John Whitlock Date: Tue, 10 Sep 2024 09:54:58 -0500 Subject: [PATCH 5/7] Change test to auto-retry assertion This makes the firefox version as reliable as the chromium one. --- e2e-tests/specs/relay-premium-upgrade.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/specs/relay-premium-upgrade.spec.ts b/e2e-tests/specs/relay-premium-upgrade.spec.ts index e7438eae14..afa350a5b6 100644 --- a/e2e-tests/specs/relay-premium-upgrade.spec.ts +++ b/e2e-tests/specs/relay-premium-upgrade.spec.ts @@ -15,7 +15,7 @@ test.describe("Premium Relay - Purchase Premium Flow, Desktop", () => { page, }) => { await dashboardPage.upgrade(); - expect(page.url()).toContain("/premium/"); + expect(page).toHaveURL("/premium/#pricing"); }); }); @@ -31,6 +31,6 @@ test.describe("Premium Relay - Purchase Premium Flow, Desktop - Visual Regressio page, }) => { await dashboardPage.upgradeNow(); - expect(page.url()).toContain("premium"); + expect(page).toHaveURL("/premium/#pricing"); }); }); From b34310f52bfdb0ca034012588b2c3b4cfb7c6b04 Mon Sep 17 00:00:00 2001 From: John Whitlock Date: Tue, 10 Sep 2024 10:50:17 -0500 Subject: [PATCH 6/7] Expand e2e README --- e2e-tests/README.md | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/e2e-tests/README.md b/e2e-tests/README.md index b3d008ff83..b292c8a51a 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -59,7 +59,21 @@ npm run test:e2e By default, `npm run test:e2e` will run the tests on https://stage.fxprivaterelay.nonprod.cloudops.mozgcp.net/. -You can also run tests locally, on our dev server (https://dev.fxprivaterelay.nonprod.cloudops.mozgcp.net/), and in production (https://relay.firefox.com/). You can find the commands [here](https://github.com/mozilla/fx-private-relay/blob/main/package.json#L26-L31), or you can run `E2E_TEST_ENV= npx playwright test`. To view the tests live in the browser, you can add `--headed` to the end of the command. See https://playwright.dev/docs/test-cli for more flags. +You can also run tests locally, on our dev server (https://dev.fxprivaterelay.nonprod.cloudops.mozgcp.net/), and in production (https://relay.firefox.com/). You can find the commands [here](https://github.com/mozilla/fx-private-relay/blob/main/package.json#L26-L31), or you can run `E2E_TEST_ENV= npx playwright test`. + +To view the tests live in the browser, you can add `--headed` to the end of the command: + +``` +npx playwright test --headed +``` + +To interactively develop tests, you can use `--ui --debug`: + +``` +npx playwright test --ui --debug +``` + +See for more flags. Our github actions workflows can be found here, [![Relay e2e Tests](https://github.com/mozilla/fx-private-relay/actions/workflows/playwright.yml/badge.svg)](https://github.com/mozilla/fx-private-relay/actions/workflows/playwright.yml). You can run the tests against different branches. @@ -80,3 +94,26 @@ Our ![health check](https://github.com/mozilla/fx-private-relay/actions/workflow `test.describe("Subscription flows @health_check", ...)` To run the health check manually, go to ![Relay e2e tests](https://github.com/mozilla/fx-private-relay/actions/workflows/playwright.yml), click run workflow, and check off "enable health check" before clicking "run workflow". + +### 9. Diagnosing Test Failures + +If the end-to-end automated test suite fails, a good first step is to manually run the test in stage with similar steps. This will help determine if the playwright tests need updates or if it has detected a regression. + +The end-to-end tests rely on several external services: + +- Relay deployments +- [Mozilla Accounts](https://accounts.firefox.com/) +- [Mozilla Monitor](https://monitor.mozilla.org/) +- [Development That Pays](https://pages.developmentthatpays.com) +- [Restmail.net](https://restmail.net/) + +If tests fail when checking these services, manually check that they are running. + +Relay includes abuse monitoring. For example, there is a limit to how many masks can be created in a time period. When developing tests, it is possible to hit these abuse limits. + +If a test is flaky, consider making the tests more reliable by using the [locators][playwright-locators], [auto-retrying assertions][playwright-auto-retrying-assertions], or [fixtures][playwright-fixtures]. For more suggestions on making Playwright tests more reliable or efficient, see [documentation on FxA test improvements][fxa-test-improvements]. + +[playwright-locators]: https://playwright.dev/docs/locators +[playwright-auto-retrying-assertions]: https://playwright.dev/docs/test-assertions#auto-retrying-assertions +[playwright-fixtures]: https://playwright.dev/docs/test-fixtures +[fxa-test-improvements]: https://docs.google.com/presentation/d/1dSASq9xcaA8DuQM_1_Ab6q5_ScBpvqI9NPHvovkA-wU/edit#slide=id.g276e3207c4d_1_427 From 8082a0e522669803052dc7e484bea9f760602caf Mon Sep 17 00:00:00 2001 From: John Whitlock Date: Tue, 10 Sep 2024 10:53:36 -0500 Subject: [PATCH 7/7] Note, premium account needs subdomain --- e2e-tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/README.md b/e2e-tests/README.md index b292c8a51a..547d9293af 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -49,7 +49,7 @@ E2E_TEST_ACCOUNT_PREMIUM= E2E_TEST_ACCOUNT_PASSWORD= ``` -Any free account created during the initial setup of tests will also use `E2E_TEST_ACCOUNT_PASSWORD`. If you do not want to use a personal premium account, reach out to Luke for `relay-team` premium account details. +The premium account needs to have a chosen subdomain for the premium tests to pass. Any free account created during the initial setup of tests will also use `E2E_TEST_ACCOUNT_PASSWORD`. If you do not want to use a personal premium account, reach out to Luke for `relay-team` premium account details. ### 6. Run Tests