Automate WCAG testing

.github/workflows/percy_snapshots.yml

@@ -23,6 +23,6 @@ jobs:
     - run: npm ci
     - run: npm run build
     - name: Percy Snapshots
-      run: npx percy exec -- node tests/acceptance/percy/snapshots.mjs
+      run: npx percy exec -- node tests/acceptance/percy/snapshots.js
       env:
         PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}

.github/workflows/wcag_test.yml

@@ -0,0 +1,22 @@
+name: WCAG tests
+
+on:
+  pull_request:
+    branches: [develop, hotfix/*, release/*, support/*]
+
+jobs:
+  WCAG-test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [14.x]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v2
+      with:
+        node-version: ${{ matrix.node-version }}
+    - run: npm ci
+    - run: npm run wcag

package.json

@@ -44,6 +44,7 @@
     "uuid": "^8.3.2"
   },
   "devDependencies": {
+    "@axe-core/puppeteer": "^4.3.1",
     "@babel/plugin-proposal-object-rest-spread": "^7.9.5",
     "@babel/plugin-syntax-dynamic-import": "^7.2.0",
     "@babel/plugin-transform-arrow-functions": "^7.8.3",
@@ -92,7 +93,7 @@
     "jsdoc": "^3.6.3",
     "npm-force-resolutions": "0.0.10",
     "postcss-pxtorem": "4.0.1",
-    "puppeteer": "^10.1.0",
+    "puppeteer": "^10.4.0",
     "regenerator-runtime": "^0.13.7",
     "rollup": "^1.4.1",
     "rollup-plugin-babel": "^4.4.0",
@@ -119,6 +120,7 @@
     "lint": "eslint .",
     "test": "eslint . && stylelint src/**/*.scss && cross-env NODE_ICU_DATA=node_modules/full-icu jest",
     "acceptance": "testcafe safari,chrome --config-file ./.circleci/testcafe.json",
+    "wcag": "./tests/acceptance/wcag/index.js",
     "size": "size-limit",
     "fix": "eslint . --fix",
     "extract-translations": "gulp extractTranslations",
@@ -239,4 +241,4 @@
     "./zh-Hant/modern.min": "./dist/zh-Hant-answers-modern.min.js",
     "./zh-Hant/template": "./dist/zh-Hant-answerstemplates.compiled.min.js"
   }
-}
+}

tests/acceptance/pagenavigator.js

@@ -0,0 +1,30 @@
+const { waitTillHTMLRendered, getQueryParamsString } = require('./wcag/utils');
+
+/**
+ * Responsible for navigating pages
+ */
+class PageNavigator {
+  constructor (page, siteUrl) {
+    this._page = page;
+    this._siteUrl = siteUrl;
+  }
+
+  async goto (pageName, queryParams = {}) {
+    const queryParamsString = getQueryParamsString(queryParams);
+    const url = `${this._siteUrl}/${pageName}?${queryParamsString}`;
+    await this._page.goto(url);
+    await waitTillHTMLRendered(this._page);
+  }
+
+  /**
+   * Click on an element of the current page
+   *
+   * @param {string} selector The CSS selector to click on
+   */
+  async click (selector) {
+    await this._page.click(selector);
+    await waitTillHTMLRendered(this._page);
+  }
+}
+
+module.exports = PageNavigator;

tests/acceptance/percy/snapshots.mjs → tests/acceptance/percy/snapshots.js

@@ -1,18 +1,19 @@
-import puppeteer from 'puppeteer';
-import percySnapshot from '@percy/puppeteer';
-import http from 'http';
-import handler from 'serve-handler';
+const puppeteer = require('puppeteer');
+const percySnapshot = require('@percy/puppeteer');
+const http = require('http');
+const handler = require('serve-handler');
+const PageNavigator = require('../pagenavigator');
 
 (async () => {
   const server = await setupServer();
   const browser = await puppeteer.launch();
   const page = await browser.newPage();
+  const navigator = new PageNavigator(page, 'http://localhost:9999/tests/acceptance/fixtures/html');
 
   async function snap (slug, name) {
-    await page.goto(`http://localhost:9999/tests/acceptance/fixtures/html/${slug}`);
+    await navigator.goto(slug);
     await percySnapshot(page, name);
   }
-
   await snap('facets.html', 'facets page pre search');
   await snap('no-unsafe-eval.html', 'no unsafe eval CSP');
 

tests/acceptance/wcag/index.js

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+
+const puppeteer = require('puppeteer');
+const { AxePuppeteer } = require('@axe-core/puppeteer');
+const PageNavigator = require('../pagenavigator');
+const WcagReporter = require('./wcagreporter');
+const handler = require('serve-handler');
+const http = require('http');
+const PORT = 9999;
+
+/**
+ * specifies options to be used by axe-core engine within axe-core/puppeteer.
+ * API documentation: https://github.com/dequelabs/axe-core/blob/master/doc/API.md
+ * - set reporter to 'no-passes' to only return violation results
+ * - set runOnly with tag values below to run WCAG standards:
+ *    - WCAG 2.0 Level A, AA, AAA
+ *    - WCAG 2.1 Level A, AA
+ */
+const axeCoreConfig = {
+  reporter: 'no-passes',
+  runOnly: {
+    type: 'tag',
+    values: ['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa', 'wcag2aaa']
+  }
+};
+
+async function setupServer () {
+  const server = http.createServer((request, response) => {
+    return handler(request, response);
+  });
+  server.listen(PORT);
+  return server;
+}
+
+async function wcagTester () {
+  const server = await setupServer();
+  const browser = await puppeteer.launch();
+  const page = await browser.newPage();
+
+  const pageNavigator = new PageNavigator(page, `http://localhost:${PORT}/tests/acceptance/fixtures/html`);
+  const analyzer = await new AxePuppeteer(page).options(axeCoreConfig);
+  let results = [];
+  try {
+    results = await new WcagReporter(pageNavigator, analyzer).analyze();
+  } catch (e) {
+    console.log(e);
+    await browser.close();
+    await server.close();
+    process.exit(1);
+  }
+
+  const failedResults = [];
+  results.forEach(result => {
+    const { url, violations } = result;
+    if (violations && violations.length > 0) {
+      failedResults.push({ url, violations });
+    }
+  });
+
+  await browser.close();
+  await server.close();
+
+  if (failedResults.length > 0) {
+    console.error(`${failedResults.length} pages failed to be WCAG complaint:`);
+    console.log(JSON.stringify(failedResults, null, 2));
+    process.exit(1);
+  }
+}
+
+wcagTester();

tests/acceptance/wcag/utils.js

@@ -0,0 +1,44 @@
+
+/**
+ * Waits until the content of the page stops changing.
+ *
+ * This methods polls the html content length and waits until the value is the same for three
+ * consecutive samples before returning.
+ *
+ * @param {import('puppeteer').Page} page
+ */
+module.exports.waitTillHTMLRendered = async function (page) {
+  const pollingIntervalMsecs = 750;
+  const minNumStableIntervals = 3;
+
+  let previousHTMLSize = 0;
+  let numStableIntervals = 0;
+  let isHTMLStabilized = false;
+
+  while (!isHTMLStabilized) {
+    await page.waitForTimeout(pollingIntervalMsecs);
+    const currentHTMLSize = (await page.content()).length;
+
+    if (currentHTMLSize === previousHTMLSize) {
+      numStableIntervals++;
+    } else {
+      numStableIntervals = 0;
+    }
+
+    isHTMLStabilized = (numStableIntervals >= minNumStableIntervals && currentHTMLSize > 0);
+    previousHTMLSize = currentHTMLSize;
+  }
+};
+
+/**
+ * Returns a string represenation of a query params object
+ *
+ * @example
+ * The input of {query: 'test', url: 'localhost'} returns 'query=test&url=localhost'
+ *
+ * @param {Object} queryParams
+ * @returns {string}
+ */
+module.exports.getQueryParamsString = function (queryParams) {
+  return new URLSearchParams(queryParams).toString();
+};

tests/acceptance/wcag/wcagreporter.js

@@ -0,0 +1,37 @@
+class WcagReporter {
+  /**
+   * @param {PageNavigator} pageNavigator
+   * @param {AxePuppeteer} analyzer
+   */
+  constructor (pageNavigator, analyzer) {
+    this._pageNavigator = pageNavigator;
+    this._analyzer = analyzer;
+    this.results = [];
+  }
+
+  async analyze () {
+    await this._analyzeUniversalPages();
+    await this._analyzeVerticalPages();
+    return this.results;
+  }
+
+  async _analyzeUniversalPages () {
+    await this._pageNavigator.goto('universal');
+    this.results.push(await this._analyzer.analyze());
+    await this._pageNavigator.goto('universal', { query: 'virginia' });
+    this.results.push(await this._analyzer.analyze());
+  }
+
+  async _analyzeVerticalPages () {
+    await this._pageNavigator.goto('facets', { query: 'tom' });
+    this.results.push(await this._analyzer.analyze());
+    await this._pageNavigator.goto('filterbox', { query: 'sales' });
+    this.results.push(await this._analyzer.analyze());
+    await this._pageNavigator.goto('vertical');
+    this.results.push(await this._analyzer.analyze());
+    await this._pageNavigator.goto('vertical', { query: 'technology' });
+    this.results.push(await this._analyzer.analyze());
+  }
+}
+
+module.exports = WcagReporter;