mirror of
https://gerrit.wikimedia.org/r/mediawiki/skins/Vector.git
synced 2024-11-14 11:15:33 +00:00
Merge "Replace automated testing code with npm package"
This commit is contained in:
jenkins-bot
Gerrit Code Review
commit
388a8cc6ce
12507
package-lock.json
generated
12507
package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -6,8 +6,8 @@
|
|||
"test": "npm -s run lint && tsc && npm run test:unit && npm -s run doc",
|
||||
"test:unit": "jest --silent",
|
||||
"test:unit:updateSnapshot": "jest --updateSnapshot",
|
||||
"test:a11y": "node tests/a11y/runA11yTests.js",
|
||||
"selenium-daily": "node tests/a11y/runA11yTests.js --logResults",
|
||||
"test:a11y": "wmf-a11y --config 'tests/a11y/a11y.config.js'",
|
||||
"selenium-daily": "wmf-a11y --config 'tests/a11y/a11y.config.js' --logResults",
|
||||
"lint": "npm -s run lint:js && npm -s run lint:styles && npm -s run lint:i18n",
|
||||
"lint:fix:js": "npm -s run lint:js -- --fix",
|
||||
"lint:fix:styles": "npm -s run lint:styles -- --fix",
|
||||
|
|
@ -38,7 +38,6 @@
|
|||
"@wikimedia/mw-node-qunit": "6.4.1",
|
||||
"@wikimedia/types-wikimedia": "0.3.3",
|
||||
"babel-loader": "8.0.6",
|
||||
"commander": "9.1.0",
|
||||
"eslint-config-wikimedia": "0.22.1",
|
||||
"grunt-banana-checker": "0.9.0",
|
||||
"jest": "27.4.7",
|
||||
|
|
@ -50,13 +49,13 @@
|
|||
"mustache": "3.0.1",
|
||||
"mustache-jest": "1.1.1",
|
||||
"node-fetch": "2.6.7",
|
||||
"pa11y": "6.1.1",
|
||||
"postcss-less": "6.0.0",
|
||||
"pre-commit": "1.2.2",
|
||||
"stylelint-config-wikimedia": "0.13.0",
|
||||
"svgo": "2.8.0",
|
||||
"ts-jest": "27.1.3",
|
||||
"typescript": "4.5.5",
|
||||
"vue": "^3.2.37"
|
||||
"vue": "3.2.37",
|
||||
"wmf-a11y": "0.0.0"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,142 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8"/>
|
||||
<title>Accessibility Report For "{{pageUrl}}" ({{date}})</title>
|
||||
|
||||
<style>
|
||||
|
||||
html, body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
background-color: #fff;
|
||||
font-family: Arial, Helvetica, sans-serif;
|
||||
font-size: 16px;
|
||||
line-height: 22px;
|
||||
color: #333;
|
||||
}
|
||||
li {
|
||||
margin-bottom: 15px;
|
||||
}
|
||||
pre {
|
||||
white-space: pre-wrap;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
.page {
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
padding: 25px;
|
||||
}
|
||||
|
||||
.counts {
|
||||
margin-top: 30px;
|
||||
font-size: 20px;
|
||||
}
|
||||
.count {
|
||||
display: inline-block;
|
||||
padding: 5px;
|
||||
border-radius: 5px;
|
||||
border: 1px solid #eee;
|
||||
}
|
||||
|
||||
.container {
|
||||
margin-top: 30px;
|
||||
}
|
||||
|
||||
.message {
|
||||
margin-top: 20px;
|
||||
padding: 0 15px 0 15px;
|
||||
border-radius: 5px;
|
||||
border: 1px solid #eee;
|
||||
}
|
||||
|
||||
details {
|
||||
margin: 16px 0 16px 0;
|
||||
}
|
||||
|
||||
summary {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.issues-list {
|
||||
margin-left: 0;
|
||||
padding-left: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.issue {
|
||||
border-radius: 5px;
|
||||
padding: 10px;
|
||||
border: 1px solid #eee;
|
||||
background-color: white;
|
||||
}
|
||||
|
||||
pre {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.error {
|
||||
background-color: #fdd;
|
||||
border-color: #ff9696;
|
||||
}
|
||||
.warning {
|
||||
background-color: #ffd;
|
||||
border-color: #e7c12b;
|
||||
}
|
||||
.notice {
|
||||
background-color: #eef4ff;
|
||||
border-color: #b6d0ff;
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="page">
|
||||
|
||||
<h1>Accessibility Report for "{{{name}}}"</h1>
|
||||
<p><b>Run on:</b> {{{pageUrl}}}</p>
|
||||
<p><b>Generated at:</b> {{date}}</p>
|
||||
<p><b>Page screenshot:</b> <a href="./{{{name}}}.png">{{{name}}}.png</a></p>
|
||||
|
||||
<nav class="counts">
|
||||
<a href="#error" class="count error">{{errorCount}} total errors</a>
|
||||
<a href="#warning" class="count warning">{{warningCount}} total warnings</a>
|
||||
<a href="#notice" class="count notice">{{noticeCount}} total notices</a>
|
||||
</nav>
|
||||
|
||||
{{#issueData}}
|
||||
<div id="{{type}}" class="container">
|
||||
<h2>{{typeLabel}}, {{typeCount}} rule(s)</h2>
|
||||
{{#messages}}
|
||||
<div class="message {{type}}">
|
||||
<h3>{{message}}</h3>
|
||||
<p><b>Rule:</b> {{runner}}, {{code}}</p>
|
||||
{{#runnerExtras}}<p><b>Impact:</b> {{impact}}</p>{{/runnerExtras}}
|
||||
<hr>
|
||||
<details>
|
||||
<summary>
|
||||
{{issueCount}} instance(s):
|
||||
</summary>
|
||||
<ul class="issues-list">
|
||||
{{#issues}}
|
||||
<li class="issue">
|
||||
<pre>{{selector}}</pre>
|
||||
<hr>
|
||||
<pre>{{context}}</pre>
|
||||
</li>
|
||||
{{/issues}}
|
||||
</ul>
|
||||
</details>
|
||||
</div>
|
||||
{{/messages}}
|
||||
</div>
|
||||
{{/issueData}}
|
||||
|
||||
</div>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
// @ts-nocheck
|
||||
'use strict';
|
||||
|
||||
const fs = require( 'fs' );
|
||||
const path = require( 'path' );
|
||||
const mustache = require( 'mustache' );
|
||||
const reportTemplate = fs.readFileSync( path.resolve( __dirname, 'report.mustache' ), 'utf8' );
|
||||
|
||||
const report = module.exports = {};
|
||||
|
||||
// Utility function to uppercase the first character of a string
|
||||
function upperCaseFirst( string ) {
|
||||
return string.charAt( 0 ).toUpperCase() + string.slice( 1 );
|
||||
}
|
||||
|
||||
// Pa11y version support
|
||||
report.supports = '^6.0.0 || ^6.0.0-alpha || ^6.0.0-beta';
|
||||
|
||||
// Compile template and output formatted results
|
||||
report.results = async ( results ) => {
|
||||
const messagesByType = results.issues.reduce( ( result, issue ) => {
|
||||
if ( result[ issue.type ].indexOf( issue.message ) === -1 ) {
|
||||
result[ issue.type ].push( issue.message );
|
||||
}
|
||||
return result;
|
||||
}, { error: [], warning: [], notice: [] } );
|
||||
const issuesByMessage = results.issues.reduce( ( result, issue ) => {
|
||||
if ( result[ issue.message ] ) {
|
||||
result[ issue.message ].push( issue );
|
||||
} else {
|
||||
result[ issue.message ] = [ issue ];
|
||||
}
|
||||
return result;
|
||||
}, {} );
|
||||
const issueData = [ 'error', 'warning', 'notice' ].map( ( type ) => ( {
|
||||
type,
|
||||
typeLabel: upperCaseFirst( type ) + 's',
|
||||
typeCount: messagesByType[ type ].length,
|
||||
messages: messagesByType[ type ].map( ( message ) => {
|
||||
const firstIssue = issuesByMessage[ message ][ 0 ];
|
||||
const hasRunnerExtras = Object.keys( firstIssue.runnerExtras ).length > 0;
|
||||
return {
|
||||
message,
|
||||
issueCount: issuesByMessage[ message ].length,
|
||||
runner: firstIssue.runner,
|
||||
runnerExtras: hasRunnerExtras ? firstIssue.runnerExtras : false,
|
||||
code: firstIssue.code,
|
||||
issues: issuesByMessage[ message ]
|
||||
};
|
||||
} ).sort( ( a, b ) => {
|
||||
// Sort messages by number of issues
|
||||
return b.issueCount - a.issueCount;
|
||||
} )
|
||||
} ) );
|
||||
|
||||
return mustache.render( reportTemplate, {
|
||||
// The current date
|
||||
date: new Date(),
|
||||
|
||||
// Test information
|
||||
name: results.name,
|
||||
pageUrl: results.pageUrl,
|
||||
|
||||
// Results
|
||||
issueData,
|
||||
|
||||
// Issue counts
|
||||
errorCount: results.issues.filter( ( issue ) => issue.type === 'error' ).length,
|
||||
warningCount: results.issues.filter( ( issue ) => issue.type === 'warning' ).length,
|
||||
noticeCount: results.issues.filter( ( issue ) => issue.type === 'notice' ).length
|
||||
} );
|
||||
};
|
||||
|
||||
// Output error messages
|
||||
report.error = ( message ) => {
|
||||
return message;
|
||||
};
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
/* eslint-disable no-console */
|
||||
// @ts-nocheck
|
||||
const fs = require( 'fs' );
|
||||
const fetch = require( 'node-fetch' );
|
||||
const path = require( 'path' );
|
||||
const pa11y = require( 'pa11y' );
|
||||
const { program } = require( 'commander' );
|
||||
|
||||
const htmlReporter = require( path.resolve( __dirname, './reporter/reporter.js' ) );
|
||||
const config = require( path.resolve( __dirname, 'a11y.config.js' ) );
|
||||
|
||||
/**
|
||||
* Delete and recreate the report directory
|
||||
*/
|
||||
function resetReportDir() {
|
||||
// Delete and create report directory
|
||||
if ( fs.existsSync( config.reportDir ) ) {
|
||||
fs.rmSync( config.reportDir, { recursive: true } );
|
||||
}
|
||||
fs.mkdirSync( config.reportDir, { recursive: true } );
|
||||
}
|
||||
|
||||
/**
|
||||
* Log test results to Graphite
|
||||
*
|
||||
* @param {string} namespace
|
||||
* @param {string} name
|
||||
* @param {number} count
|
||||
* @return {Promise<any>}
|
||||
*/
|
||||
function sendMetrics( namespace, name, count ) {
|
||||
const metricPrefix = 'ci_a11y';
|
||||
const url = `${process.env.WMF_JENKINS_BEACON_URL}${metricPrefix}.${namespace}.${name}=${count}c`;
|
||||
return fetch( url );
|
||||
}
|
||||
|
||||
/**
|
||||
* Run pa11y on tests specified by the config.
|
||||
*
|
||||
* @param {Object} opts
|
||||
*/
|
||||
async function runTests( opts ) {
|
||||
if ( !process.env.MW_SERVER ||
|
||||
!process.env.MEDIAWIKI_USER ||
|
||||
!process.env.MEDIAWIKI_PASSWORD ) {
|
||||
throw new Error( 'Missing env variables' );
|
||||
}
|
||||
|
||||
const tests = config.tests;
|
||||
const allValidTests = tests.filter( ( test ) => test.name ).length === tests.length;
|
||||
if ( !allValidTests ) {
|
||||
throw new Error( 'Config missing test name' );
|
||||
}
|
||||
|
||||
const canLogResults = process.env.WMF_JENKINS_BEACON_URL && config.namespace;
|
||||
if ( opts.logResults && !canLogResults ) {
|
||||
throw new Error( 'Unable to log results, missing config or env variables' );
|
||||
}
|
||||
|
||||
resetReportDir();
|
||||
|
||||
const testPromises = tests.map( ( test ) => {
|
||||
const { url, name, ...testOptions } = test;
|
||||
const options = { ...config.defaults, ...testOptions };
|
||||
// Automatically enable screen capture for every test;
|
||||
options.screenCapture = `${config.reportDir}/${name}.png`;
|
||||
|
||||
return pa11y( url, options ).then( ( testResult ) => {
|
||||
testResult.name = name;
|
||||
return testResult;
|
||||
} );
|
||||
} );
|
||||
|
||||
// Run tests against multiple URLs
|
||||
const results = await Promise.all( testPromises ); // eslint-disable-line
|
||||
results.forEach( async ( testResult ) => {
|
||||
const name = testResult.name;
|
||||
const errorNum = testResult.issues.filter( ( issue ) => issue.type === 'error' ).length;
|
||||
const warningNum = testResult.issues.filter( ( issue ) => issue.type === 'warning' ).length;
|
||||
const noticeNum = testResult.issues.filter( ( issue ) => issue.type === 'notice' ).length;
|
||||
|
||||
// Log results summary to console
|
||||
if ( !opts.silent ) {
|
||||
console.log( `'${name}'- ${errorNum} errors, ${warningNum} warnings, ${noticeNum} notices` );
|
||||
}
|
||||
|
||||
// Send data to Graphite
|
||||
// WMF_JENKINS_BEACON_URL is only defined in CI env
|
||||
if ( opts.logResults && canLogResults ) {
|
||||
await sendMetrics( config.namespace, testResult.name, errorNum )
|
||||
.then( ( response ) => {
|
||||
if ( response.ok ) {
|
||||
console.log( `'${name}' results logged successfully` );
|
||||
} else {
|
||||
console.error( `Failed to log '${name}' results` );
|
||||
}
|
||||
} );
|
||||
}
|
||||
|
||||
// Save in html report
|
||||
const html = await htmlReporter.results( testResult );
|
||||
fs.promises.writeFile( `${config.reportDir}/report-${name}.html`, html, 'utf8' );
|
||||
// Save in json report
|
||||
fs.promises.writeFile( `${config.reportDir}/report-${name}.json`, JSON.stringify( testResult, null, ' ' ), 'utf8' );
|
||||
} );
|
||||
}
|
||||
|
||||
function setupCLI() {
|
||||
program
|
||||
.option( '-s, --silent', 'avoids logging results summary to console', false )
|
||||
.option( '-l, --logResults', 'log a11y results to Graphite, should only be used with --env ci', false )
|
||||
.action( ( opts ) => {
|
||||
runTests( opts );
|
||||
} );
|
||||
|
||||
program.parse();
|
||||
}
|
||||
|
||||
setupCLI();
|
||||
Loading…
Reference in a new issue