Merge pull request #713 from Atom-Learning/feat/visual-regression

Feat/visual regression

.github/workflows/test.yml

@@ -5,6 +5,10 @@ on:
     branches:
       - main
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest

.github/workflows/visual-regression.yml

@@ -1,21 +1,27 @@
 name: Visual Regression
 
 on:
-  workflow_call:
-    inputs:
-      dsDocumentationUrl:
-        description: 'URL of environment to run VR tests against. Should include protocol (e.g. https://)'
-        required: false
-        default: ''
-        type: string
+  push:
+    branches: -main
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
 
 permissions:
   actions: write
   contents: read
   pull-requests: write
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  test:
+  visual-regression:
+    if: ${{ github.ref_name == 'main' && true || contains(github.event.pull_request.labels.*.name, 'visual-regression') }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -31,15 +37,12 @@ jobs:
         run: yarn install
 
       - name: Build Library
-        if: ${{ inputs.dsDocumentationUrl != '' }}
         run: yarn build:lib
 
       - name: Build Documentation
-        if: ${{ inputs.dsDocumentationUrl != '' }}
         run: yarn build:docs
 
       - name: Start Web Server as a Daemon
-        if: ${{ inputs.dsDocumentationUrl != '' }}
         run: |
           yarn start:docs &
           disown -h %1
@@ -48,7 +51,6 @@ jobs:
         run: yarn playwright install --with-deps
 
       - name: Run Visual Regression
-        run: yarn visual-regression
+        run: yarn vr:chromium
         env:
-          DS_DOCUMENTATION_URL: '${{ inputs.dsDocumentationUrl }}'
           ARGOS_TOKEN: ${{ secrets.ARGOS_TOKEN }}

.gitignore

@@ -16,6 +16,7 @@
 !.yarn/releases
 !.yarn/sdks
 !.yarn/versions
-
-playwright-report
-test-results
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

e2e/constants.ts

@@ -0,0 +1,2 @@
+export const DS_DOCUMENTATION_URL =
+  process.env.DS_DOCUMENTATION_URL || 'http://localhost:3000'

e2e/fixtures/baseTest.ts

@@ -0,0 +1,30 @@
+import { expect, test } from '@playwright/test'
+import { argosScreenshot } from '@argos-ci/playwright'
+
+import { waitForNetworkIdle } from '../utils/domUtils'
+import { HomePage } from '../pom/HomePage'
+import { DS_DOCUMENTATION_URL } from '../constants'
+
+interface BaseTest {
+  homePage: HomePage
+  visualSnapshot: (snapshotName?: string) => Promise<void>
+}
+
+export const baseTest = test.extend<BaseTest>({
+  homePage: async ({ page }, use) => {
+    await page.goto(DS_DOCUMENTATION_URL)
+    await waitForNetworkIdle(page)
+    await expect(
+      page.getByRole('heading', { name: 'Atom Learning Design System' })
+    ).toBeVisible()
+
+    const homePage = new HomePage(page)
+
+    await use(homePage)
+  },
+  visualSnapshot: async ({ page }, use, testInfo) =>
+    use(async (snapshotName: string = testInfo.title) => {
+      await waitForNetworkIdle(page)
+      await argosScreenshot(page, snapshotName, { animations: 'disabled' })
+    })
+})

e2e/fixtures/index.ts

@@ -0,0 +1 @@
+export * from './baseTest'

e2e/pom/HomePage.ts

@@ -0,0 +1,49 @@
+import { Locator, Page } from '@playwright/test'
+
+export class HomePage {
+  public nav: Locator
+  public menu: Locator
+  public tabList: Locator
+
+  constructor(public readonly page: Page) {
+    this.nav = page.getByRole('navigation', { name: 'Main' })
+    this.menu = this.nav.getByRole('list')
+    this.tabList = page.getByRole('tablist').first()
+  }
+
+  async getAllLinks() {
+    const anchorElements = await this.menu
+      .getByRole('listitem')
+      .getByRole('link')
+      .all()
+
+    const links: Array<{ href: string; label: string }> = []
+
+    for (const anchorElement of anchorElements) {
+      const href = await anchorElement.getAttribute('href')
+      const label = await anchorElement.innerText()
+
+      if (href) {
+        links.push({
+          href,
+          label
+        })
+      }
+    }
+
+    return links
+  }
+
+  async getAllTabs() {
+    const tabs = await this.tabList.getByRole('tab').all()
+    return Promise.all(tabs.map((tab) => tab.innerText()))
+  }
+
+  async showTab(tabName: string) {
+    const tab = this.tabList.getByRole('tab', { name: tabName })
+
+    if (await tab.isEnabled()) {
+      await tab.click()
+    }
+  }
+}

e2e/specs/visual-regression.spec.ts

@@ -0,0 +1,39 @@
+import { baseTest as test } from '../fixtures'
+import { prepareURL } from '../utils/urlUtils'
+
+test.describe('@visual Design System - Documentation Visual Regression', () => {
+  test('Scenario: Visits all links and takes snapshot', async ({
+    homePage,
+    page,
+    visualSnapshot
+  }) => {
+    // Take screenshot of home page
+    await visualSnapshot()
+
+    // Get all links in the side navbar
+    const links = await homePage.getAllLinks()
+
+    // Iterate of the links
+    for (const { href, label } of links) {
+      // Visit each link
+      console.log(`Visiting ${label}`)
+      await page.goto(prepareURL(href))
+
+      // Check if a page has tabs
+      const tabs = await homePage.getAllTabs()
+      // If tabs are present, iterate over them and take screenshot
+      if (tabs.length > 0) {
+        for (const tab of tabs) {
+          await homePage.showTab(tab)
+          await visualSnapshot(
+            `[Visual Snapshot - ${label}] [Link - ${href}] [Tab - ${tab}]`
+          )
+        }
+      }
+      // If tabs are not present, take screenshot and move on
+      else {
+        await visualSnapshot(`[Visual Snapshot - ${label}] [Link - ${href}]`)
+      }
+    }
+  })
+})

e2e/utils/domUtils.ts

@@ -0,0 +1,13 @@
+import { Page } from '@playwright/test'
+
+export const waitForNetworkIdle = async (
+  page: Page,
+  timeout: number = 10000
+) => {
+  try {
+    // eslint-disable-next-line playwright/no-networkidle
+    await page.waitForLoadState('networkidle', { timeout })
+  } catch (error) {
+    // Do nothing
+  }
+}

e2e/utils/urlUtils.ts

@@ -0,0 +1,3 @@
+import { DS_DOCUMENTATION_URL } from '../constants'
+
+export const prepareURL = (link: string) => `${DS_DOCUMENTATION_URL}${link}`

package.json

@@ -17,9 +17,14 @@
     "build:types": "cd lib && NODE_ENV=production tsc --emitDeclarationOnly && tsc-alias",
     "lint-staged": "cd lib && yarn run precommit",
     "build-all": "run-s build:*",
-    "visual-regression": "echo 'vr:chromium'"
+    "vr:chromium": "playwright test --project chromium --grep '@visual'",
+    "vr:firefox": "playwright test --project firefox --grep '@visual'",
+    "vr:webkit": "playwright test --project webkit --grep '@visual'"
   },
   "devDependencies": {
+    "@argos-ci/playwright": "^3.9.1",
+    "@playwright/test": "^1.48.2",
+    "@types/node": "^22.9.0",
     "husky": "^4.3.8",
     "npm-run-all": "^4.1.5"
   },

playwright.config.ts

@@ -0,0 +1,92 @@
+import { defineConfig, devices } from '@playwright/test'
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// import path from 'path';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './e2e',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /**
+   * We can only run one test in parallel because ubuntu-latest in private repositories has 2vCPUs
+   * and the proportion of workers is # vCPU / 2 (because one per browser and test runner)
+   */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: [
+    ['list'],
+    ['html'],
+    [
+      '@argos-ci/playwright/reporter',
+      {
+        uploadToArgos: Boolean(process.env.ARGOS_TOKEN),
+        token: Boolean(process.env.ARGOS_TOKEN)
+      }
+    ]
+  ],
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    trace: process.env.CI ? 'off' : 'on'
+  },
+
+  timeout: 5 * 60 * 60 * 1000,
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] }
+    },
+
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] }
+    },
+
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] }
+    }
+
+    // Potential configs in case we want to cover more screen sizes / browsers
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ]
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   url: 'http://127.0.0.1:3000',
+  //   reuseExistingServer: !process.env.CI,
+  // },
+})