wikimedia/mediawiki-skins-Vector
1
0
Fork 0
mirror of https://gerrit.wikimedia.org/r/mediawiki/skins/Vector.git synced 2024-11-11 16:59:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add automated a11y tests with pa11y

Browse source 
Bug: T301184
Change-Id: If3d5aa74b9b1f5bf0f7aa2ea6950853c9a5f5ada
This commit is contained in:
bwang 2022-03-16 17:26:04 -05:00 committed by Jdlrobson
parent 094928736c
commit dd9b6e1dbc
7 changed files with 1410 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1010
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

6
package.json
View file

@ -4,13 +4,15 @@
"start": "bash dev-scripts/setup-storybook.sh && start-storybook --quiet -p 6006 -s resources/skins.vector.styles",
"test": "npm -s run lint && tsc && npm run test:unit && npm -s run doc",
"test:unit": "jest --silent",
"test:a11y": "node tests/a11y/runA11yTests.js --env development",
"selenium-daily": "node tests/a11y/runA11yTests.js --env ci --logResults",
"lint": "npm -s run lint:js && npm -s run lint:styles && npm -s run lint:i18n",
"lint:fix:js": "npm -s run lint:js -- --fix",
"lint:fix:styles": "npm -s run lint:styles -- --fix",
"lint:js": "eslint --cache .",
"lint:styles": "stylelint \"**/*.{less,css}\"",
"lint:i18n": "banana-checker --requireLowerCase=0 i18n/",
"doc": "jsdoc -c jsdoc.json && npm run build-storybook -s resources/skins.vector.styles",
"doc": "jsdoc -c jsdoc.json && npm run build-storybook -s resources/skins.vector.styles && npm run test:a11y",
"build-storybook": "bash dev-scripts/setup-storybook.sh && build-storybook --quiet --loglevel warn -o docs/ui",
"minify-svg": "svgo --config=.svgo.config.js --quiet --recursive --folder resources/",
"pre-commit": "[ \"${PRE_COMMIT:-1}\" -eq 0 ] || npm -s t"
@ -28,6 +30,7 @@
"@wikimedia/types-wikimedia": "0.2.0",
"@wikimedia/wvui": "0.3.5",
"babel-loader": "8.0.6",
"commander": "9.1.0",
"eslint-config-wikimedia": "0.20.0",
"grunt-banana-checker": "0.9.0",
"jest": "26.4.2",
@ -38,6 +41,7 @@
"less-loader": "4.1.0",
"mustache": "3.0.1",
"node-fetch": "2.6.1",
"pa11y": "6.1.1",
"pre-commit": "1.2.2",
"stylelint-config-wikimedia": "0.11.1",
"svgo": "2.8.0",

11
tests/a11y/.eslintrc.json Normal file
View file

@ -0,0 +1,11 @@
{
"extends": [
"../../.eslintrcEs6.json"
],
"parserOptions": {
"ecmaVersion": 2018
},
"env": {
"node": true
}
}

63
tests/a11y/a11y.config.js Normal file
View file

@ -0,0 +1,63 @@
// @ts-nocheck
const config = {
reportDir: 'docs/a11y',
namespace: 'Vector',
env: {
development: {
baseUrl: process.env.MW_SERVER,
defaultPage: '/wiki/Polar_bear?useskin=vector-2022'
},
ci: {
baseUrl: 'https://en.wikipedia.beta.wmflabs.org',
defaultPage: '/wiki/Polar_bear'
}
},
defaults: {
viewport: {
width: 1200,
height: 1080
},
runners: [
'axe',
'htmlcs'
],
includeWarnings: true,
includeNotices: true,
hideElements: '#content'
}
};
config.tests = ( envName ) => ( [
{
name: 'default',
url: config.env[ envName ].baseUrl + config.env[ envName ].defaultPage
},
{
name: 'logged_in',
url: config.env[ envName ].baseUrl + config.env[ envName ].defaultPage,
wait: '500',
actions: [
'click #p-personal-checkbox',
'wait for .vector-user-menu-login a to be visible',
'click .vector-user-menu-login a',
'wait for #wpName1 to be visible',
'set field #wpName1 to ' + process.env.MEDIAWIKI_USER,
'set field #wpPassword1 to ' + process.env.MEDIAWIKI_PASSWORD,
'click #wpLoginAttempt',
'wait for #pt-userpage-2 to be visible' // Confirm login was successful
]
},
{
name: 'search',
url: config.env[ envName ].baseUrl + config.env[ envName ].defaultPage,
rootElement: '#p-search',
wait: '500',
actions: [
'click #searchInput',
'wait for .wvui-input__input to be added',
'set field .wvui-input__input to Test'
]
}
] );
module.exports = config;

128
tests/a11y/reporter/report.mustache Normal file
View file

@ -0,0 +1,128 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<title>Accessibility Report For "{{pageUrl}}" ({{date}})</title>
<style>
html, body {
margin: 0;
padding: 0;
background-color: #fff;
font-family: Arial, Helvetica, sans-serif;
font-size: 16px;
line-height: 22px;
color: #333;
}
li {
margin-bottom: 15px;
}
pre {
white-space: pre-wrap;
overflow: auto;
}
.page {
max-width: 800px;
margin: 0 auto;
padding: 25px;
}
.counts {
margin-top: 30px;
font-size: 20px;
}
.count {
display: inline-block;
padding: 5px;
border-radius: 5px;
border: 1px solid #eee;
}
.container {
margin-top: 30px;
}
.message {
margin-top: 20px;
padding: 0 15px 0 15px;
border-radius: 5px;
border: 1px solid #eee;
}
.issues-list {
margin-left: 0;
padding-left: 0;
list-style: none;
}
.issue {
border-radius: 5px;
padding: 10px;
border: 1px solid #eee;
background-color: white;
}
pre {
margin: 0;
}
.error {
background-color: #fdd;
border-color: #ff9696;
}
.warning {
background-color: #ffd;
border-color: #e7c12b;
}
.notice {
background-color: #eef4ff;
border-color: #b6d0ff;
}
</style>
</head>
<body>
<div class="page">
<h1>Accessibility Report For {{{name}}}</h1>
<p>Run on: {{{pageUrl}}}</p>
<p>Generated at: {{date}}</p>
<nav class="counts">
<a href="#error" class="count error">{{errorCount}} total errors</a>
<a href="#warning" class="count warning">{{warningCount}} total warnings</a>
<a href="#notice" class="count notice">{{noticeCount}} total notices</a>
</nav>
{{#issueData}}
<div id="{{type}}" class="container">
<h2>{{typeLabel}}, {{typeCount}} rules</h2>
{{#messages}}
<div class="message {{type}}">
<h3>{{message}}</h3>
<p>Rule: {{runner}}, {{code}}</p>
{{#runnerExtras}}<p>Impact: {{impact}}</p>{{/runnerExtras}}
<h4>{{issueCount}} instance(s):</h4>
<ul class="issues-list">
{{#issues}}
<li class="issue">
<pre>{{selector}}</pre>
<hr>
<pre>{{context}}</pre>
</li>
{{/issues}}
</ul>
</div>
{{/messages}}
</div>
{{/issueData}}
</div>
</body>
</html>

77
tests/a11y/reporter/reporter.js Normal file
View file

@ -0,0 +1,77 @@
// @ts-nocheck
'use strict';
const fs = require( 'fs' );
const path = require( 'path' );
const mustache = require( 'mustache' );
const reportTemplate = fs.readFileSync( path.resolve( __dirname, 'report.mustache' ), 'utf8' );
const report = module.exports = {};
// Utility function to uppercase the first character of a string
function upperCaseFirst( string ) {
return string.charAt( 0 ).toUpperCase() + string.slice( 1 );
}
// Pa11y version support
report.supports = '^6.0.0 || ^6.0.0-alpha || ^6.0.0-beta';
// Compile template and output formatted results
report.results = async ( results ) => {
const messagesByType = results.issues.reduce( ( result, issue ) => {
if ( result[ issue.type ].indexOf( issue.message ) === -1 ) {
result[ issue.type ].push( issue.message );
}
return result;
}, { error: [], warning: [], notice: [] } );
const issuesByMessage = results.issues.reduce( ( result, issue ) => {
if ( result[ issue.message ] ) {
result[ issue.message ].push( issue );
} else {
result[ issue.message ] = [ issue ];
}
return result;
}, {} );
const issueData = [ 'error', 'warning', 'notice' ].map( ( type ) => ( {
type,
typeLabel: upperCaseFirst( type ) + 's',
typeCount: messagesByType[ type ].length,
messages: messagesByType[ type ].map( ( message ) => {
const firstIssue = issuesByMessage[ message ][ 0 ];
const hasRunnerExtras = Object.keys( firstIssue.runnerExtras ).length > 0;
return {
message,
issueCount: issuesByMessage[ message ].length,
runner: firstIssue.runner,
runnerExtras: hasRunnerExtras ? firstIssue.runnerExtras : false,
code: firstIssue.code,
issues: issuesByMessage[ message ]
};
} ).sort( ( a, b ) => {
// Sort messages by number of issues
return b.issueCount - a.issueCount;
} )
} ) );
return mustache.render( reportTemplate, {
// The current date
date: new Date(),
// Test information
name: results.name,
pageUrl: results.pageUrl,
// Results
issueData,
// Issue counts
errorCount: results.issues.filter( ( issue ) => issue.type === 'error' ).length,
warningCount: results.issues.filter( ( issue ) => issue.type === 'warning' ).length,
noticeCount: results.issues.filter( ( issue ) => issue.type === 'notice' ).length
} );
};
// Output error messages
report.error = ( message ) => {
return message;
};

125
tests/a11y/runA11yTests.js Normal file
View file

@ -0,0 +1,125 @@
/* eslint-disable no-console */
// @ts-nocheck
const fs = require( 'fs' );
const fetch = require( 'node-fetch' );
const path = require( 'path' );
const pa11y = require( 'pa11y' );
const htmlReporter = require( path.resolve( __dirname, './reporter/reporter.js' ) );
const config = require( path.resolve( __dirname, 'a11y.config.js' ) );
/**
* Delete and recreate the report directory
*/
function resetReportDir() {
// Delete and create report directory
fs.rmdirSync( config.reportDir, { recursive: true } );
fs.mkdirSync( config.reportDir, { recursive: true } );
}
/**
* Log test results to Graphite
*
* @param {string} namespace
* @param {string} name
* @param {number} count
* @return {Promise<any>}
*/
function sendMetrics( namespace, name, count ) {
const metricPrefix = 'MediaWiki.a11y';
const url = `${process.env.BEACON_URL}${metricPrefix}.${namespace}.${name}=${count}c`;
return fetch( url );
}
/**
* Run pa11y on tests specified by the config.
*
* @param {Object} opts
*/
async function runTests( opts ) {
try {
if ( !config.env[ opts.env ] ) {
throw new Error( `Invalid env value: '${opts.env}'` );
}
if ( opts.env !== 'ci' && opts.logResults ) {
throw new Error( "Results can only be logged with '--env ci'" );
}
const tests = config.tests( opts.env );
const allTestsHaveNames = tests.filter( ( test ) => test.name ).length === tests.length;
if ( !allTestsHaveNames ) {
throw new Error( 'Config missing test name' );
}
resetReportDir();
const testPromises = tests.map( ( test ) => {
const { url, name, ...testOptions } = test;
const options = { ...config.defaults, ...testOptions };
// Automatically enable screen capture for every test;
options.screenCapture = `${config.reportDir}/${name}.png`;
return pa11y( url, options ).then( ( testResult ) => {
testResult.name = name;
return testResult;
} );
} );
// Run tests against multiple URLs
const results = await Promise.all( testPromises ); // eslint-disable-line
results.forEach( async ( testResult ) => {
const name = testResult.name;
const errorNum = testResult.issues.filter( ( issue ) => issue.type === 'error' ).length;
const warningNum = testResult.issues.filter( ( issue ) => issue.type === 'warning' ).length;
const noticeNum = testResult.issues.filter( ( issue ) => issue.type === 'notice' ).length;
// Log results summary to console
if ( !opts.silent ) {
console.log( `'${name}'- ${errorNum} errors, ${warningNum} warnings, ${noticeNum} notices` );
}
// Send data to Graphite
// BEACON_URL is only defined in CI env
if ( opts.env === 'ci' && opts.logResults && process.env.BEACON_URL ) {
if ( !config.namespace ) {
throw new Error( 'Config missing namespace' );
}
await sendMetrics( config.namespace, testResult.name, errorNum )
.then( ( response ) => {
if ( response.ok ) {
console.log( `'${name}' results logged successfully` );
} else {
console.error( `Failed to log '${name}' results` );
}
} );
}
// Save in html report
const html = await htmlReporter.results( testResult );
fs.promises.writeFile( `${config.reportDir}/report-${name}.html`, html, 'utf8' );
// Save in json report
fs.promises.writeFile( `${config.reportDir}/report-${name}.json`, JSON.stringify( testResult, null, ' ' ), 'utf8' );
} );
} catch ( error ) {
// Output an error if it occurred
console.error( error.message );
}
}
function setupCLI() {
const { program } = require( 'commander' );
program
.requiredOption( '-e, --env <env>', 'determine which urls tests are run on, development or ci' )
.option( '-s, --silent', 'avoids logging results summary to console', false )
.option( '-l, --logResults', 'log a11y results to Graphite, should only be used with --env ci', false )
.action( ( opts ) => {
runTests( opts );
} );
program.parse();
}
setupCLI();