Using playwright to generate snapshots (#1589)

.devcontainer/devcontainer.json

@@ -35,8 +35,7 @@
   "appPort": [4000, 5400],
   "portsAttributes": {
     "4000": {
-      "label": "demo",
-      "onAutoForward": "openBrowser"
+      "label": "demo"
     },
     "5400": {
       "label": "doctocat"

.github/workflows/test.yml

@@ -174,3 +174,59 @@ jobs:
     - name: Test
       run: |
         bundle exec rake test:system
+  visual:
+    name: Visual Regressions
+    if: ${{ github.event_name == 'pull_request' }}
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.1'
+        bundler-cache: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+        cache: 'npm'
+        cache-dependency-path: |
+          package-lock.json
+          demo/package-lock.json
+    - name: Build
+      run: |
+        npm ci
+        bundle install
+        cd demo
+        npm ci
+        bundle install
+    - name: Install Playwright Browsers
+      run: npx playwright install --with-deps
+    - name: Run Playwright tests
+      id: playwright-run
+      continue-on-error: true
+      run: npx playwright test
+    - id: auto-commit
+      uses: stefanzweifel/git-auto-commit-action@v4
+      with:
+        commit_message: Generating component snapshots
+        file_pattern: .playwright/screenshots/**/*.png
+    - name: "Changes detected"
+      if: steps.auto-commit.outputs.changes_detected == 'true'
+      uses: phulsechinmay/rewritable-pr-comment@v0.3.0
+      with:
+        message: |
+          ### ⚠️ Visual differences found
+
+          Our visual comparison tests found UI differences. Please review the differences by viewing the files changed tab to ensure that the changes were intentional.
+
+          [Review visual differences](https://github.com/primer/view_components/pull/${{ github.event.number }}/files?file-filters%5B%5D=.png&show-viewed-files=false)
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        COMMENT_IDENTIFIER: 'visual-comparison-diff'
+    - uses: actions/upload-artifact@v3
+      with:
+        name: playwright-report
+        path: .playwright/report/
+        retention-days: 30
+    - name: Failure
+      if: ${{ steps.auto-commit.outputs.changes_detected == 'true' || steps.playwright-run.outcome == 'failure' }}
+      run: exit 1

.playwright/.gitignore

@@ -0,0 +1,3 @@
+report/
+results/
+results.json

docs/contributors/playwright-testing.md

@@ -0,0 +1,52 @@
+# Testing
+
+<!-- prettier-ignore-start -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+## Table of Contents
+
+- [Testing](#testing)
+  - [Table of Contents](#table-of-contents)
+  - [Overview](#overview)
+  - [Prerequisites](#prerequisites)
+  - [Continous Integration](#continous-integration)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+<!-- prettier-ignore-end -->
+
+## Overview
+
+We use Playwright to run visual regression tests against our components along with automated accessibility checks. These tests are authored within the `test/playwright` directory and match the file pattern:
+`*.test.ts`.
+
+You can run these tests using Playwright locally but **we recommend you check the results** of
+these tests on GitHub through the CI workflow.
+
+To get started locally, make sure to follow the [Prerequisites](#prerequisites)
+section to setup your machine. If you're looking for a quick overview of the commands
+available, check out the table below.
+
+| Task                                                  | Command                                         |
+| :---------------------------------------------------- | :---------------------------------------------- |
+| Run playwright tests                                  | `npx playwright test`                           |
+| Run a specific test                                   | `npx playwright test path/to/test`              |
+| View the report from a test run                       | `npx playwright show-report .playwright/report` |
+
+## Prerequisites
+
+To run Playwright locally, it is recommended to open the repo in a codespace. This is to ensure that the browser
+rendering the screenshots will match the browser in CI.
+
+Once you have the codespace open, make sure you're up to date with `script/setup`. Then you can run the `npx playwright test` command.
+
+## Continous Integration
+
+Playwright tests are included in the `Tests > Visual Regressions` jobs of the CI workflow.
+The results of the test run are uploaded at the end of the job and are available
+to download and view locally.
+
+If you notice that `Tests > Visual Regressions` is failing, you can view the report of the
+failing run by visiting the CI workflow, clicking into the job that has failed
+and downloading the relevant report.
+
+When the workflow runs, it will check in screenshots of previews for visual diff checking.

package.json

@@ -57,6 +57,7 @@
     "@changesets/cli": "^2.24.1",
     "@github/browserslist-config": "^1.0.0",
     "@github/prettier-config": "0.0.4",
+    "@playwright/test": "^1.27.1",
     "@primer/css": "20.4.7",
     "@primer/primitives": "^7.9.0",
     "@rollup/plugin-node-resolve": "^13.3.0",
@@ -71,6 +72,7 @@
     "eslint-plugin-prettier": "^4.2.1",
     "markdownlint-cli": "^0.32.2",
     "mocha": "^10.0.0",
+    "playwright": "^1.27.1",
     "postcss": "^8.4.16",
     "postcss-cli": "^10.0.0",
     "postcss-import": "^14.1.0",

playwright.config.ts

@@ -0,0 +1,48 @@
+// eslint-disable-next-line import/no-nodejs-modules
+import path from 'node:path'
+import type {PlaywrightTestConfig} from '@playwright/test'
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config: PlaywrightTestConfig = {
+  testDir: path.join(__dirname, 'test', 'playwright'),
+  testMatch: '**/*.test.ts',
+  /* Maximum time one test can run for. */
+  timeout: 30 * 1000,
+
+  // https://playwright.dev/docs/api/class-testconfig#test-config-output-dir
+  outputDir: path.join(__dirname, '.playwright', 'results'),
+  snapshotDir: path.join(__dirname, '.playwright', 'screenshots'),
+
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  workers: process.env.CI ? 4 : undefined,
+  updateSnapshots: 'all',
+  use: {
+    baseURL: 'http://127.0.0.1:4000',
+    browserName: 'chromium',
+    headless: true,
+    screenshot: 'only-on-failure'
+  },
+  expect: {
+    toHaveScreenshot: {
+      animations: 'disabled'
+    }
+  },
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+
+  reporter: [
+    ['line'],
+    ['html', {open: 'never', outputFolder: path.join(__dirname, '.playwright/report')}],
+    ['json', {outputFile: path.join(__dirname, '.playwright', 'results.json')}]
+  ],
+
+  webServer: {
+    command: 'cd demo; bin/rails s -p 4000',
+    port: 4000
+  }
+}
+
+export default config

test/playwright/helpers.ts

@@ -0,0 +1,41 @@
+import {glob} from 'glob'
+
+// Previews we're ignoring because they're not ready to be tested yet
+const ignoredPreviews = ['primer/forms/forms']
+
+interface ComponentPreview {
+  componentName: string
+  previewURL: string
+}
+
+export function componentPreviews(): ComponentPreview[] {
+  // Glob for all the preview files
+  const previews = glob.sync('previews/**/*_preview.rb')
+  const result: ComponentPreview[] = []
+
+  for (const preview of previews) {
+    // Remove prefixes and suffixes to get the component name and preview URL
+    const previewURL = preview.replace('previews/', '').replace('_preview.rb', '').replace('_component', '')
+
+    // If the preview is in the ignored list, skip it
+    if (ignoredPreviews.includes(previewURL)) {
+      continue
+    }
+
+    // Covert the preview URL to a component name ie. primer/beta/button => Primer::Beta::Button
+    const componentName = previewURL
+      .split('/')
+      .map(module => {
+        return module
+          .split('_')
+          .map(word => {
+            return word[0].toUpperCase() + word.substring(1)
+          })
+          .join('')
+      })
+      .join('::')
+
+    result.push({componentName, previewURL})
+  }
+  return result
+}

test/playwright/previews.test.ts

@@ -0,0 +1,21 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+/* eslint-disable no-unused-vars */
+import {test, expect} from '@playwright/test'
+import {componentPreviews} from './helpers'
+
+test.beforeEach(async ({page}, testInfo) => {
+  testInfo.snapshotSuffix = ''
+})
+
+for (const {componentName, previewURL} of componentPreviews()) {
+  test(`renders ${componentName} preview`, async ({page}) => {
+    await page.goto(`/lookbook/preview/${previewURL}/default`)
+    const defaultScreenshot = await page.locator('#component-preview').screenshot({animations: 'disabled'})
+    expect(defaultScreenshot).toMatchSnapshot([previewURL, 'default.png'])
+
+    // Focus state
+    await page.keyboard.press('Tab')
+    const focusedScreenshot = await page.locator('#component-preview').screenshot({animations: 'disabled'})
+    expect(focusedScreenshot).toMatchSnapshot([previewURL, 'focused.png'])
+  })
+}