Merge pull request #5022 from mozilla/e2e-tests-mpp-3892

MPP-3892: e2e test improvements

e2e-tests/README.md

@@ -49,7 +49,7 @@ E2E_TEST_ACCOUNT_PREMIUM=<your_premium_account_email>
 E2E_TEST_ACCOUNT_PASSWORD=<your_premium_account_password>
 ```
 
-Any free account created during the initial setup of tests will also use `E2E_TEST_ACCOUNT_PASSWORD`. If you do not want to use a personal premium account, reach out to Luke for `relay-team` premium account details.
+The premium account needs to have a chosen subdomain for the premium tests to pass. Any free account created during the initial setup of tests will also use `E2E_TEST_ACCOUNT_PASSWORD`. If you do not want to use a personal premium account, reach out to Luke for `relay-team` premium account details.
 
 ### 6. Run Tests
 
@@ -59,7 +59,21 @@ npm run test:e2e
 
 By default, `npm run test:e2e` will run the tests on https://stage.fxprivaterelay.nonprod.cloudops.mozgcp.net/.
 
-You can also run tests locally, on our dev server (https://dev.fxprivaterelay.nonprod.cloudops.mozgcp.net/), and in production (https://relay.firefox.com/). You can find the commands [here](https://github.com/mozilla/fx-private-relay/blob/main/package.json#L26-L31), or you can run `E2E_TEST_ENV=<env (prod, dev, stage)> npx playwright test`. To view the tests live in the browser, you can add `--headed` to the end of the command. See https://playwright.dev/docs/test-cli for more flags.
+You can also run tests locally, on our dev server (https://dev.fxprivaterelay.nonprod.cloudops.mozgcp.net/), and in production (https://relay.firefox.com/). You can find the commands [here](https://github.com/mozilla/fx-private-relay/blob/main/package.json#L26-L31), or you can run `E2E_TEST_ENV=<env (prod, dev, stage)> npx playwright test`.
+
+To view the tests live in the browser, you can add `--headed` to the end of the command:
+
+```
+npx playwright test --headed
+```
+
+To interactively develop tests, you can use `--ui --debug`:
+
+```
+npx playwright test --ui --debug
+```
+
+See <https://playwright.dev/docs/test-cli> for more flags.
 
 Our github actions workflows can be found here, [![Relay e2e Tests](https://github.com/mozilla/fx-private-relay/actions/workflows/playwright.yml/badge.svg)](https://github.com/mozilla/fx-private-relay/actions/workflows/playwright.yml). You can run the tests against different branches.
 
@@ -80,3 +94,26 @@ Our ![health check](https://github.com/mozilla/fx-private-relay/actions/workflow
 `test.describe("Subscription flows @health_check", ...)`
 
 To run the health check manually, go to ![Relay e2e tests](https://github.com/mozilla/fx-private-relay/actions/workflows/playwright.yml), click run workflow, and check off "enable health check" before clicking "run workflow".
+
+### 9. Diagnosing Test Failures
+
+If the end-to-end automated test suite fails, a good first step is to manually run the test in stage with similar steps. This will help determine if the playwright tests need updates or if it has detected a regression.
+
+The end-to-end tests rely on several external services:
+
+- Relay deployments
+- [Mozilla Accounts](https://accounts.firefox.com/)
+- [Mozilla Monitor](https://monitor.mozilla.org/)
+- [Development That Pays](https://pages.developmentthatpays.com)
+- [Restmail.net](https://restmail.net/)
+
+If tests fail when checking these services, manually check that they are running.
+
+Relay includes abuse monitoring. For example, there is a limit to how many masks can be created in a time period. When developing tests, it is possible to hit these abuse limits.
+
+If a test is flaky, consider making the tests more reliable by using the [locators][playwright-locators], [auto-retrying assertions][playwright-auto-retrying-assertions], or [fixtures][playwright-fixtures]. For more suggestions on making Playwright tests more reliable or efficient, see [documentation on FxA test improvements][fxa-test-improvements].
+
+[playwright-locators]: https://playwright.dev/docs/locators
+[playwright-auto-retrying-assertions]: https://playwright.dev/docs/test-assertions#auto-retrying-assertions
+[playwright-fixtures]: https://playwright.dev/docs/test-fixtures
+[fxa-test-improvements]: https://docs.google.com/presentation/d/1dSASq9xcaA8DuQM_1_Ab6q5_ScBpvqI9NPHvovkA-wU/edit#slide=id.g276e3207c4d_1_427

e2e-tests/pages/dashboardPage.ts

@@ -1,5 +1,5 @@
 import { Locator, Page, expect } from "@playwright/test";
-import { checkAuthState, getVerificationCode } from "../e2eTestUtils/helpers";
+import { checkAuthState } from "../e2eTestUtils/helpers";
 
 export class DashboardPage {
   readonly page: Page;
@@ -56,6 +56,8 @@ export class DashboardPage {
   readonly customMaskDoneButton: Locator;
   readonly maskCardBottomMeta: Locator;
   readonly maskCardTrackersCount: Locator;
+  readonly chooseSubdomain: Locator;
+  readonly bannerEmailError: Locator;
 
   constructor(page: Page) {
     this.page = page;
@@ -128,6 +130,10 @@ export class DashboardPage {
     this.dashBoardWithoutMasksEmail = page.locator(
       '//section[starts-with(@class, "profile_no-premium-header")]',
     );
+    this.chooseSubdomain = page.locator("id=mpp-choose-subdomain");
+    this.bannerEmailError = page.getByText(
+      "The mask could not be created. Please try again.",
+    );
 
     // mask card elements
     this.maskCard = page.getByRole("button", { name: "Generate new mask" });
@@ -218,6 +224,9 @@ export class DashboardPage {
     if (numberOfMasks === 0) {
       return;
     }
+    // Check that the subdomain has been set for the premium user
+    expect(await this.chooseSubdomain.count()).toBe(0);
+
     await this.generateNewMaskPremiumButton.click();
     await this.premiumDomainMask.click();
 
@@ -240,17 +249,31 @@ export class DashboardPage {
       ? this.generateNewMaskPremiumButton
       : this.generateNewMaskButton;
 
-    // generate a new mask and confirm
+    const maskCards = this.page.locator(this.maskCardString);
+    const preMaskCardsCount = await maskCards.count();
+
+    // generate a new mask
     await generateMaskBtn.click();
 
     const randomMaskShown = await this.premiumRandomMask.isVisible();
     if (randomMaskShown) {
       await this.premiumRandomMask.click();
     }
-    await this.page.waitForSelector(this.maskCardString, { timeout: 3000 });
 
-    // randomize between 1.5-2.5 secs between each generate to deal with issue of multiple quick clicks
-    await this.page.waitForTimeout(Math.random() * 2500 + 1500);
+    // Wait for the mask card count to increase, or the error banner
+    expect(
+      this.bannerEmailError.or(maskCards.nth(preMaskCardsCount)),
+    ).toBeVisible({ timeout: 3000 });
+    expect(
+      this.bannerEmailError,
+      "No mask error banner. If fails, maybe rate-limited?",
+    ).not.toBeVisible();
+    expect(await maskCards, "Mask cards should go up by one").toHaveCount(
+      preMaskCardsCount + 1,
+    );
+
+    // randomize between .5-1.0 secs between each generate to deal with issue of multiple quick clicks
+    await this.page.waitForTimeout(Math.random() * 500 + 500);
     if (await this.closeCornerUpsell.isVisible()) {
       await this.closeCornerUpsell.click();
     }

e2e-tests/specs/relay-premium-functionality.spec.ts

@@ -12,6 +12,7 @@ test.describe("Premium - General Functionalities, Desktop", () => {
   test("Verify that a premium user can make more than 5 masks @health_check", async ({
     dashboardPage,
   }) => {
+    expect(await dashboardPage.emailMasksUsedAmount.textContent()).toBe("0");
     await dashboardPage.generateMask(6, true);
 
     await expect
@@ -40,7 +41,7 @@ test.describe("Premium - General Functionalities, Desktop", () => {
     );
   });
 
-  test("Verify that a premium  user can generate a custom mask @health_check", async ({
+  test("Verify that a premium user can generate a custom mask @health_check", async ({
     dashboardPage,
   }) => {
     // When there are zero masks, a random mask must be generated first

e2e-tests/specs/relay-premium-upgrade.spec.ts

@@ -15,7 +15,7 @@ test.describe("Premium Relay - Purchase Premium Flow, Desktop", () => {
     page,
   }) => {
     await dashboardPage.upgrade();
-    expect(page.url()).toContain("/premium/");
+    expect(page).toHaveURL("/premium/#pricing");
   });
 });
 
@@ -31,6 +31,6 @@ test.describe("Premium Relay - Purchase Premium Flow, Desktop - Visual Regressio
     page,
   }) => {
     await dashboardPage.upgradeNow();
-    expect(page.url()).toContain("premium");
+    expect(page).toHaveURL("/premium/#pricing");
   });
 });