mediawiki-skins-Vector/tests/a11y/runA11yTests.js

/* eslint-disable no-console */
// @ts-nocheck
const fs = require( 'fs' );
const fetch = require( 'node-fetch' );
const path = require( 'path' );
const pa11y = require( 'pa11y' );
const { program } = require( 'commander' );

const htmlReporter = require( path.resolve( __dirname, './reporter/reporter.js' ) );
const config = require( path.resolve( __dirname, 'a11y.config.js' ) );

/**
 *  Delete and recreate the report directory
 */
function resetReportDir() {
	// Delete and create report directory
	if ( fs.existsSync( config.reportDir ) ) {
		fs.rmSync( config.reportDir, { recursive: true } );
	}
	fs.mkdirSync( config.reportDir, { recursive: true } );
}

/**
 *  Log test results to Graphite
 *
 * @param {string} namespace
 * @param {string} name
 * @param {number} count
 * @return {Promise<any>}
 */
function sendMetrics( namespace, name, count ) {
	const metricPrefix = 'ci_a11y';
	const url = `${process.env.WMF_JENKINS_BEACON_URL}${metricPrefix}.${namespace}.${name}=${count}c`;
	return fetch( url );
}

/**
 *  Run pa11y on tests specified by the config.
 *
 * @param {Object} opts
 */
async function runTests( opts ) {
	if ( !process.env.MW_SERVER ||
		!process.env.MEDIAWIKI_USER ||
		!process.env.MEDIAWIKI_PASSWORD ) {
		throw new Error( 'Missing env variables' );
	}

	try {
		const tests = config.tests;
		const allValidTests = tests.filter( ( test ) => test.name ).length === tests.length;
		if ( !allValidTests ) {
			throw new Error( 'Config missing test name' );
		}

		const canLogResults = process.env.WMF_JENKINS_BEACON_URL && config.namespace;
		if ( opts.logResults && !canLogResults ) {
			throw new Error( 'Unable to log results, missing config or env variables' );
		}

		resetReportDir();

		const testPromises = tests.map( ( test ) => {
			const { url, name, ...testOptions } = test;
			const options = { ...config.defaults, ...testOptions };
			// Automatically enable screen capture for every test;
			options.screenCapture = `${config.reportDir}/${name}.png`;

			return pa11y( url, options ).then( ( testResult ) => {
				testResult.name = name;
				return testResult;
			} );
		} );

		// Run tests against multiple URLs
		const results = await Promise.all( testPromises ); // eslint-disable-line
		results.forEach( async ( testResult ) => {
			const name = testResult.name;
			const errorNum = testResult.issues.filter( ( issue ) => issue.type === 'error' ).length;
			const warningNum = testResult.issues.filter( ( issue ) => issue.type === 'warning' ).length;
			const noticeNum = testResult.issues.filter( ( issue ) => issue.type === 'notice' ).length;

			// Log results summary to console
			if ( !opts.silent ) {
				console.log( `'${name}'- ${errorNum} errors, ${warningNum} warnings, ${noticeNum} notices` );
			}

			// Send data to Graphite
			// WMF_JENKINS_BEACON_URL is only defined in CI env
			if ( opts.logResults && canLogResults ) {
				await sendMetrics( config.namespace, testResult.name, errorNum )
					.then( ( response ) => {
						if ( response.ok ) {
							console.log( `'${name}' results logged successfully` );
						} else {
							console.error( `Failed to log '${name}' results` );
						}
					} );
			}

			// Save in html report
			const html = await htmlReporter.results( testResult );
			fs.promises.writeFile( `${config.reportDir}/report-${name}.html`, html, 'utf8' );
			// Save in json report
			fs.promises.writeFile( `${config.reportDir}/report-${name}.json`, JSON.stringify( testResult, null, '  ' ), 'utf8' );
		} );

	} catch ( error ) {
		// Output an error if it occurred
		console.error( error.message );
	}
}

function setupCLI() {
	program
		.option( '-s, --silent', 'avoids logging results summary to console', false )
		.option( '-l, --logResults', 'log a11y results to Graphite, should only be used with --env ci', false )
		.action( ( opts ) => {
			runTests( opts );
		} );

	program.parse();
}

setupCLI();