Merge pull request #196 from wikimedia/bernard/wmf-a11y

Add a11y testing to pixel

.eslintignore

@@ -1,4 +1,5 @@
 report
+report-a11y
 package-lock.json
 coverage
 context.json

Dockerfile.a11y-regression

@@ -0,0 +1,58 @@
+FROM node:16.15.1-bullseye
+
+ARG PA11Y_VERSION
+
+ENV \
+	PA11Y_VERSION=$PA11Y_VERSION
+
+# Base packages
+RUN apt-get update && \
+	apt-get install -y git sudo software-properties-common
+
+RUN set -ex && \
+    DEBIAN_VERSION=$(cat /etc/os-release | grep VERSION_ID | cut -d '=' -f2 | tr -d '"') && \
+    export DEBIAN_VERSION=$DEBIAN_VERSION && \
+    ARCH=`uname -m` && \
+    if [ "$ARCH" = "x86_64" ]; then \
+       sudo npm install -g --unsafe-perm=true --allow-root pa11y@${PA11Y_VERSION}; \
+    else \
+       sudo PUPPETEER_SKIP_DOWNLOAD=true PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true npm install -g --unsafe-perm=true --allow-root pa11y@${PA11Y_VERSION} && puppeteer-chromium-version-finder@^1.0.1 chromium-version-deb-finder@^2.0.1 && \
+       NODE_PATH="$(npm root -g):$(npm root -g)/backstopjs/node_modules" node -e " \
+          const versionFinder = require('puppeteer-chromium-version-finder'); \
+          const debFinder = require('chromium-version-deb-finder'); \
+          (async () => { \
+            const version = await versionFinder.getPuppeteerChromiumVersion(); \
+            const debUrls = await debFinder.getDebUrlsForVersionAndArch(version.MAJOR, version.MINOR, process.env.DEBIAN_VERSION, 'arm64'); \
+            console.log(debUrls.join('\n')); \
+          })(); \
+       " | \
+       xargs -I % sh -c 'set -x; echo "Downloading: %"; wget -c -t 10 -w 10 -T 120 %; echo "Downloaded: %"' && \
+       apt install -y ./*.deb && \
+       rm -f ./*.deb && \
+       sudo test -f /usr/bin/chromium && sudo ln -s /usr/bin/chromium /usr/bin/chromium-browser && sudo ln -s /usr/bin/chromium /usr/bin/chrome; \
+    fi
+
+RUN sudo npm install -g mustache
+
+RUN wget https://dl-ssl.google.com/linux/linux_signing_key.pub && sudo apt-key add linux_signing_key.pub
+RUN sudo add-apt-repository "deb http://dl.google.com/linux/chrome/deb/ stable main"
+
+RUN apt-get -qqy update \
+  && apt-get -qqy --no-install-recommends install \
+    libxshmfence-dev \
+    libfontconfig \
+    libfreetype6 \
+    xfonts-cyrillic \
+    xfonts-scalable \
+    fonts-liberation \
+    fonts-ipafont-gothic \
+    fonts-wqy-zenhei \
+    libgbm-dev \
+    gconf-service libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxss1 libxtst6 libappindicator1 libnss3 libasound2 libatk1.0-0 libc6 ca-certificates fonts-liberation lsb-release xdg-utils wget \
+  && rm -rf /var/lib/apt/lists/* \
+  && apt-get -qyy clean
+
+ENV NODE_PATH=/usr/local/lib/node_modules
+
+#ENTRYPOINT ["tail", "-f", "/dev/null"]
+ENTRYPOINT ["node", "src/a11y/runA11yTests.js"]

configDesktopA11y.js

@@ -0,0 +1,48 @@
+// @ts-nocheck
+const utils = require( './utils' );
+
+const NAMESPACE = 'desktop';
+const BASE_URL = process.env.PIXEL_MW_SERVER;
+
+const testDefaults = {
+	viewport: {
+		width: 1200,
+		height: 1080
+	},
+	runners: [
+		'axe',
+		'htmlcs'
+	],
+	includeWarnings: true,
+	includeNotices: true,
+	ignore: [
+		// Prevent axe-core from flagging all TOC links as incorrect color contrast
+		'color-contrast',
+		// Prevent contrast ratio error on absolutely positioned elements
+		'WCAG2AA.Principle1.Guideline1_4.1_4_3.G18.Abs'
+	],
+	chromeLaunchConfig: {
+		headless: true,
+		args: [
+			'--no-sandbox',
+			'--disable-setuid-sandbox'
+		]
+	}
+};
+
+module.exports = {
+	namespace: NAMESPACE,
+	paths: utils.makeA11yPaths( NAMESPACE ),
+	tests: [
+		{
+			name: 'default',
+			url: BASE_URL + '/wiki/Test',
+			...testDefaults
+		},
+		{
+			name: 'logged_in',
+			url: BASE_URL + '/wiki/Test',
+			...testDefaults
+		}
+	]
+};

docker-compose.yml

@@ -78,3 +78,21 @@ services:
       - ./configCodex.js:/pixel/configCodex.js
       - ./src:/pixel/src
       - ./report:/pixel/report
+  a11y-regression:
+    init: true
+    network_mode: host
+    build:
+      context: .
+      dockerfile: Dockerfile.a11y-regression
+      args:
+        - PA11Y_VERSION=6.2.3
+    working_dir: /pixel
+    env_file:
+      - .env
+    volumes:
+      - ./context.json:/pixel/context.json
+      - ./viewports.js:/pixel/viewports.js
+      - ./utils.js:/pixel/utils.js
+      - ./configDesktopA11y.js:/pixel/configDesktopA11y.js
+      - ./src:/pixel/src
+      - ./report-a11y:/pixel/report-a11y

pixel.js

@@ -156,10 +156,19 @@ const GROUP_CONFIG = {
 
 /**
  * @param {string} groupName
+ * @param {boolean} a11y
  * @return {string} path to configuration file.
  * @throws {Error} for unknown group
  */
-const getGroupConfig = ( groupName ) => {
+const getGroupConfig = ( groupName, a11y ) => {
+	if ( a11y ) {
+		switch ( groupName ) {
+			case 'desktop':
+				return 'configDesktopA11y.js';
+			default:
+				throw new Error( `Unknown test group: ${groupName}` );
+		}
+	}
 	const c = GROUP_CONFIG[ groupName ];
 	if ( !c ) {
 		throw new Error( `Unknown test group: ${groupName}` );
@@ -241,10 +250,9 @@ async function processCommand( type, opts, runSilently = false ) {
 			context[ group ].description = description;
 		}
 		context[ group ][ type ] = active;
+
 		// store details of this run.
 		fs.writeFileSync( `${__dirname}/context.json`, JSON.stringify( context ) );
-		const configFile = getGroupConfig( group );
-		const config = require( `${__dirname}/${configFile}` );
 
 		// Start docker containers.
 		await batchSpawn.spawn(
@@ -260,54 +268,71 @@ async function processCommand( type, opts, runSilently = false ) {
 			[ 'compose', ...getComposeOpts( [ 'exec', ...( process.env.NONINTERACTIVE ? [ '-T' ] : [] ), 'mediawiki', '/src/main.js', JSON.stringify( opts ) ] ) ]
 		);
 
-		// Remove test screenshots folder (if present) so that its size doesn't
-		// increase with each `test` run. BackstopJS automatically removes the
-		// reference folder when the `reference` command is run, but not the test
-		// folder when the `test` command is run.
-		if ( type === 'test' ) {
-			removeFolder( config.paths.bitmaps_test );
-		}
-		// Execute Visual regression tests.
-		const finished = await batchSpawn.spawn(
-			'docker',
-			[ 'compose', ...getComposeOpts( [ 'run', ...( process.env.NONINTERACTIVE ? [ '--no-TTY' ] : [] ), '--rm', 'visual-regression', type, '--config', configFile ] ) ]
-		).then( async () => {
-			await writeBannerAndOpenReportIfNecessary(
-				type, group, config.paths.html_report, runSilently || process.env.NONINTERACTIVE
-			);
-		}, async ( /** @type {Error} */ err ) => {
-			// Don't check error message if caller asked us not to.
-			if ( err.message.includes( '130' ) ) {
-				// If user ends subprocess with a sigint, exit early.
-				if ( !runSilently ) {
-					// eslint-disable-next-line no-process-exit
-					process.exit( 1 );
+		const configFile = getGroupConfig( group, opts.a11y );
+		const config = require( `${__dirname}/${configFile}` );
+
+		if ( opts.a11y ) {
+			// Execute a11y regression tests.
+			return batchSpawn.spawn(
+				'docker',
+				[ 'compose', ...getComposeOpts( [ 'run', ...( process.env.NONINTERACTIVE ? [ '--no-TTY' ] : [] ), '--rm', 'a11y-regression', type, configFile ] ) ]
+			).finally( async () => {
+				// Reset the database if `--reset-db` option is passed.
+				if ( opts.resetDb ) {
+					console.log( 'Resetting database state...' );
+					await resetDb();
 				}
+			} );
+		} else {
+			// Remove test screenshots folder (if present) so that its size doesn't
+			// increase with each `test` run. BackstopJS automatically removes the
+			// reference folder when the `reference` command is run, but not the test
+			// folder when the `test` command is run.
+			if ( type === 'test' ) {
+				removeFolder( config.paths.bitmaps_test );
 			}
-
-			if ( err.message.includes( 'Exit with error code 1' ) ) {
+			// Execute Visual regression tests.
+			const finished = await batchSpawn.spawn(
+				'docker',
+				[ 'compose', ...getComposeOpts( [ 'run', ...( process.env.NONINTERACTIVE ? [ '--no-TTY' ] : [] ), '--rm', 'visual-regression', type, '--config', configFile ] ) ]
+			).then( async () => {
 				await writeBannerAndOpenReportIfNecessary(
-					type, group, config.paths.html_report, process.env.NONINTERACTIVE
+					type, group, config.paths.html_report, runSilently || process.env.NONINTERACTIVE
 				);
-				if ( !runSilently ) {
-					// eslint-disable-next-line no-process-exit
-					process.exit( 1 );
+			}, async ( /** @type {Error} */ err ) => {
+				// Don't check error message if caller asked us not to.
+				if ( err.message.includes( '130' ) ) {
+					// If user ends subprocess with a sigint, exit early.
+					if ( !runSilently ) {
+						// eslint-disable-next-line no-process-exit
+						process.exit( 1 );
+					}
 				}
-			}
 
-			if ( runSilently ) {
-				return Promise.resolve();
-			} else {
-				throw err;
-			}
-		} ).finally( async () => {
-			// Reset the database if `--reset-db` option is passed.
-			if ( opts.resetDb ) {
-				console.log( 'Resetting database state...' );
-				await resetDb();
-			}
-		} );
-		return finished;
+				if ( err.message.includes( 'Exit with error code 1' ) ) {
+					await writeBannerAndOpenReportIfNecessary(
+						type, group, config.paths.html_report, process.env.NONINTERACTIVE
+					);
+					if ( !runSilently ) {
+						// eslint-disable-next-line no-process-exit
+						process.exit( 1 );
+					}
+				}
+
+				if ( runSilently ) {
+					return Promise.resolve();
+				} else {
+					throw err;
+				}
+			} ).finally( async () => {
+				// Reset the database if `--reset-db` option is passed.
+				if ( opts.resetDb ) {
+					console.log( 'Resetting database state...' );
+					await resetDb();
+				}
+			} );
+			return finished;
+		}
 	} catch ( err ) {
 		// eslint-disable-next-line no-process-exit
 		process.exit( 1 );
@@ -320,6 +345,14 @@ function setEnvironmentFlagIfGroup( envVarName, soughtGroup, group ) {
 
 function setupCli() {
 	const { program } = require( 'commander' );
+	const a11yOpt = /** @type {const} */ ( [
+		'-a, --a11y',
+		'Run automated a11y tests in addition to visual regression.'
+	] );
+	const logResultsOpt = /** @type {const} */ ( [
+		'-l, --logResults',
+		'Log accessibility results to statsv.'
+	] );
 	const branchOpt = /** @type {const} */ ( [
 		'-b, --branch <name-of-branch>',
 		`Name of branch. Can be "${MAIN_BRANCH}" or a release branch (e.g. "origin/wmf/1.37.0-wmf.19"). Use "${LATEST_RELEASE_BRANCH}" to use the latest wmf release branch.`,
@@ -357,6 +390,8 @@ function setupCli() {
 		.command( 'reference' )
 		.description( 'Create reference (baseline) screenshots and delete the old reference screenshots.' )
 		.requiredOption( ...branchOpt )
+		.option( ...a11yOpt )
+		.option( ...logResultsOpt )
 		.option( ...changeIdOpt )
 		.option( ...groupOpt )
 		.option( ...resetDbOpt )
@@ -368,6 +403,8 @@ function setupCli() {
 		.command( 'test' )
 		.description( 'Create test screenshots and compare them against the reference screenshots.' )
 		.requiredOption( ...branchOpt )
+		.option( ...a11yOpt )
+		.option( ...logResultsOpt )
 		.option( ...changeIdOpt )
 		.option( ...groupOpt )
 		.option( ...resetDbOpt )

report-a11y/.gitignore

@@ -0,0 +1,3 @@
+# Ignore everything except this file
+*
+!.gitignore

src/a11y/reporter/Results.mustache

@@ -0,0 +1,22 @@
+<div id="{{type}}-{{name}}" class="container">
+	<h3>{{typeLabel}} ({{typeCount}})</h3>
+	{{#codes}}
+	<div class="message {{type}}">
+		<h4>{{message}}</h4>
+		<p class="rule"><b>Rule:</b> {{runner}}, {{code}}</p>
+		{{#runnerExtras}}<p><b>Impact:</b> {{impact}}</p>{{/runnerExtras}}
+		<details class="details">
+			<summary>
+				{{issueCount}} issue(s):
+			</summary>
+			<ul class="issues-list">
+				{{#issues}}
+				<li class="issue">
+					<pre>{{selector}}<hr>{{context}}</pre>
+				</li>
+				{{/issues}}
+			</ul>
+		</details>
+	</div>
+	{{/codes}}
+</div>