mirror of
https://gerrit.wikimedia.org/r/mediawiki/extensions/VisualEditor
synced 2024-09-26 19:56:49 +00:00
Refactor parserTests somewhat into a class-like structure, and wire up the
TokenTransformer.
This commit is contained in:
Gabriel Wicke
parent
d616f07a79
commit
752b0990b2
|
|
@ -19,8 +19,6 @@
|
|||
|
||||
function TokenTransformer( callback ) {
|
||||
this.cb = callback; // Called with transformed token list when done
|
||||
this.accum = new TokenAccumulator();
|
||||
this.firstaccum = this.accum;
|
||||
this.transformers = {
|
||||
tag: {}, // for TAG, ENDTAG, SELFCLOSINGTAG, keyed on name
|
||||
text: [],
|
||||
|
|
@ -29,11 +27,17 @@ function TokenTransformer( callback ) {
|
|||
end: [], // eof
|
||||
martian: [] // none of the above
|
||||
};
|
||||
this.outstanding = 1; // Number of outstanding processing steps
|
||||
// (e.g., async template fetches/expansions)
|
||||
this.reset();
|
||||
return this;
|
||||
}
|
||||
|
||||
TokenTransformer.prototype.reset = function () {
|
||||
this.accum = new TokenAccumulator();
|
||||
this.firstaccum = this.accum;
|
||||
this.outstanding = 1; // Number of outstanding processing steps
|
||||
// (e.g., async template fetches/expansions)
|
||||
};
|
||||
|
||||
TokenTransformer.prototype.appendListener = function ( listener, type, name ) {
|
||||
if ( type === 'tag' ) {
|
||||
if ( $.isArray(this.transformers.tag.name) ) {
|
||||
|
|
@ -97,7 +101,7 @@ function TokenContext ( token, accum, transformer, lastToken ) {
|
|||
* @returns {TokenContext} Context with updated token and/or accum.
|
||||
*/
|
||||
TokenTransformer.prototype._transformTagToken = function ( tokenCTX ) {
|
||||
var ts = this.transformers.tag[token.name];
|
||||
var ts = this.transformers.tag[tokenCTX.token.name];
|
||||
if ( ts ) {
|
||||
for (var i = 0, l = ts.length; i < l; i++ ) {
|
||||
// Transform token with side effects
|
||||
|
|
@ -145,6 +149,7 @@ TokenTransformer.prototype._transformToken = function ( tokenCTX, ts ) {
|
|||
* */
|
||||
TokenTransformer.prototype.transformTokens = function ( tokens, accum ) {
|
||||
if ( accum === undefined ) {
|
||||
this.reset();
|
||||
accum = this.accum;
|
||||
} else {
|
||||
// Prepare to replace the last token in the current accumulator.
|
||||
|
|
@ -199,13 +204,13 @@ TokenTransformer.prototype.finish = function ( ) {
|
|||
if ( this.outstanding === 0 ) {
|
||||
// Join the token accumulators back into a single token list
|
||||
var a = this.firstaccum;
|
||||
var accums = [a.accum];
|
||||
var tokens = a.accum;
|
||||
while ( a.next !== undefined ) {
|
||||
a = a.next;
|
||||
accums.concat(a.accum);
|
||||
tokens.concat(a.accum);
|
||||
}
|
||||
// Call our callback with the flattened token list
|
||||
this.cb(accums);
|
||||
this.cb(tokens);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -237,3 +242,7 @@ TokenAccumulator.prototype.insertAccumulator = function ( ) {
|
|||
this.next = new TokenAccumulator(this.next, tokens);
|
||||
return this.next;
|
||||
};
|
||||
|
||||
if (typeof module == "object") {
|
||||
module.exports.TokenTransformer = TokenTransformer;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,27 +5,28 @@
|
|||
* Use along with a HTML5TreeBuilder and the DOMPostProcessor(s) for HTML
|
||||
* output.
|
||||
*/
|
||||
|
||||
var PEG = require('pegjs');
|
||||
|
||||
function PegTokenizer(env) {
|
||||
this.env = env || {};
|
||||
}
|
||||
|
||||
PegTokenizer.src = false;
|
||||
|
||||
PegTokenizer.prototype.tokenize = function(text, callback) {
|
||||
this.initSource(function() {
|
||||
var out, err;
|
||||
try {
|
||||
if ( !this.parser ) {
|
||||
this.parser = PEG.buildParser(PegTokenizer.src);
|
||||
}
|
||||
out = this.parser.parse(text);
|
||||
} catch (e) {
|
||||
err = e;
|
||||
console.trace();
|
||||
} finally {
|
||||
callback(out, err);
|
||||
}
|
||||
});
|
||||
PegTokenizer.prototype.tokenize = function( text ) {
|
||||
var out, err;
|
||||
if ( !this.parser ) {
|
||||
this.parser = PEG.buildParser(PegTokenizer.src);
|
||||
}
|
||||
try {
|
||||
out = this.parser.parse(text);
|
||||
} catch (e) {
|
||||
err = e;
|
||||
console.trace();
|
||||
} finally {
|
||||
return {tokens: out, err: err};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -19,89 +19,10 @@ var fs = require('fs'),
|
|||
jsDiff = require('diff'),
|
||||
colors = require('colors'),
|
||||
util = require( 'util' ),
|
||||
HTML5 = require('html5').HTML5;
|
||||
|
||||
// Name of file used to cache the parser tests cases
|
||||
var cache_file = "parserTests.cache";
|
||||
|
||||
// XXX: avoid a global here!
|
||||
global.PEG = require('pegjs');
|
||||
|
||||
// Handle options/arguments with optimist module
|
||||
var optimist = require('optimist');
|
||||
|
||||
var argv = optimist.usage( 'Usage: $0', {
|
||||
'quick': {
|
||||
description: 'Suppress diff output of failed tests',
|
||||
boolean: true,
|
||||
default: false,
|
||||
},
|
||||
'quiet': {
|
||||
description: 'Suppress notification of passed tests (shows only failed tests)',
|
||||
boolean: true,
|
||||
default: false,
|
||||
},
|
||||
'color': {
|
||||
description: 'Enable color output Ex: --no-color',
|
||||
boolean: true,
|
||||
default: true,
|
||||
},
|
||||
'cache': {
|
||||
description: 'Get tests cases from cache file ' + cache_file,
|
||||
boolean: true,
|
||||
default: false,
|
||||
},
|
||||
'filter': {
|
||||
description: 'Only run tests whose descriptions which match given regex',
|
||||
alias: 'regex',
|
||||
},
|
||||
'whitelist': {
|
||||
description: 'Alternatively compare against manually verified parser output from whitelist',
|
||||
default: true,
|
||||
boolean: true,
|
||||
},
|
||||
'help': {
|
||||
description: 'Show this help message',
|
||||
alias: 'h',
|
||||
},
|
||||
'disabled': {
|
||||
description: 'Run disabled tests (option not implemented)',
|
||||
default: false,
|
||||
boolean: true,
|
||||
},
|
||||
'printwhitelist': {
|
||||
description: 'Print out a whitelist entry for failing tests. Default false.',
|
||||
default: false,
|
||||
boolean: true,
|
||||
},
|
||||
}
|
||||
).check( function(argv) {
|
||||
if( argv.filter === true ) {
|
||||
throw "--filter need an argument";
|
||||
}
|
||||
}
|
||||
).argv // keep that
|
||||
;
|
||||
|
||||
|
||||
if( argv.help ) {
|
||||
optimist.showHelp();
|
||||
process.exit( 0 );
|
||||
}
|
||||
var test_filter = null;
|
||||
if( argv.filter ) { // null is the default by definition
|
||||
try {
|
||||
test_filter = new RegExp( argv.filter );
|
||||
} catch(e) {
|
||||
console.error( "\nERROR> --filter was given an invalid regular expression.");
|
||||
console.error( "ERROR> See below for JS engine error:\n" + e + "\n" );
|
||||
process.exit( 1 );
|
||||
}
|
||||
console.log( "Filtering title test using Regexp " + test_filter );
|
||||
}
|
||||
if( !argv.color ) {
|
||||
colors.mode = 'none';
|
||||
}
|
||||
HTML5 = require('html5').HTML5,
|
||||
PEG = require('pegjs'),
|
||||
// Handle options/arguments with optimist module
|
||||
optimist = require('optimist');
|
||||
|
||||
// @fixme wrap more or this setup in a common module
|
||||
|
||||
|
|
@ -125,25 +46,18 @@ function _import(filename, symbols) {
|
|||
}
|
||||
|
||||
|
||||
// needed for html5 parser adapter
|
||||
//var events = require('events');
|
||||
|
||||
// For now most modules only need this for $.extend and $.each :)
|
||||
global.$ = require('jquery');
|
||||
|
||||
// hack for renderer
|
||||
global.document = $('<div>')[0].ownerDocument;
|
||||
|
||||
var pj = path.join;
|
||||
|
||||
|
||||
|
||||
// Our code...
|
||||
|
||||
var testWhiteList = require('./parserTests-whitelist.js').testWhiteList;
|
||||
|
||||
_import(pj('parser', 'mediawiki.tokenizer.peg.js'), ['PegTokenizer']);
|
||||
_import(pj('parser', 'mediawiki.parser.environment.js'), ['MWParserEnvironment']);
|
||||
_import(pj('parser', 'mediawiki.TokenTransformer.js'), ['TokenTransformer']);
|
||||
_import(pj('parser', 'ext.cite.taghook.ref.js'), ['MWRefTagHook']);
|
||||
|
||||
_import(pj('parser', 'mediawiki.HTML5TreeBuilder.node.js'), ['FauxHTML5']);
|
||||
|
|
@ -157,57 +71,176 @@ _require(pj('es', 'serializers', 'es.HtmlSerializer.js'));
|
|||
_require(pj('es', 'serializers', 'es.WikitextSerializer.js'));
|
||||
_require(pj('es', 'serializers', 'es.JsonSerializer.js'));
|
||||
|
||||
// Preload the grammar file...
|
||||
PegTokenizer.src = fs.readFileSync(path.join(basePath, 'parser', 'pegTokenizer.pegjs.txt'), 'utf8');
|
||||
|
||||
var wikiTokenizer = new PegTokenizer();
|
||||
function ParserTests () {
|
||||
|
||||
var testFileName = '../../../../phase3/tests/parser/parserTests.txt'; // default
|
||||
var testFileName2 = '../../../../tests/parser/parserTests.txt'; // Fallback. Not everyone fetch at phase3 level
|
||||
this.argv = optimist.usage( 'Usage: $0', {
|
||||
'quick': {
|
||||
description: 'Suppress diff output of failed tests',
|
||||
boolean: true,
|
||||
default: false
|
||||
},
|
||||
'quiet': {
|
||||
description: 'Suppress notification of passed tests (shows only failed tests)',
|
||||
boolean: true,
|
||||
default: false
|
||||
},
|
||||
'color': {
|
||||
description: 'Enable color output Ex: --no-color',
|
||||
boolean: true,
|
||||
default: true
|
||||
},
|
||||
'cache': {
|
||||
description: 'Get tests cases from cache file ' + this.cache_file,
|
||||
boolean: true,
|
||||
default: false
|
||||
},
|
||||
'filter': {
|
||||
description: 'Only run tests whose descriptions which match given regex',
|
||||
alias: 'regex'
|
||||
},
|
||||
'whitelist': {
|
||||
description: 'Alternatively compare against manually verified parser output from whitelist',
|
||||
default: true,
|
||||
boolean: true
|
||||
},
|
||||
'help': {
|
||||
description: 'Show this help message',
|
||||
alias: 'h'
|
||||
},
|
||||
'disabled': {
|
||||
description: 'Run disabled tests (option not implemented)',
|
||||
default: false,
|
||||
boolean: true
|
||||
},
|
||||
'printwhitelist': {
|
||||
description: 'Print out a whitelist entry for failing tests. Default false.',
|
||||
default: false,
|
||||
boolean: true
|
||||
}
|
||||
}
|
||||
).check( function(argv) {
|
||||
if( argv.filter === true ) {
|
||||
throw "--filter need an argument";
|
||||
}
|
||||
}
|
||||
).argv; // keep that
|
||||
|
||||
if (argv._[0]) {
|
||||
// hack :D
|
||||
testFileName = argv._[0] ;
|
||||
testFileName2 = null;
|
||||
|
||||
if( this.argv.help ) {
|
||||
optimist.showHelp();
|
||||
process.exit( 0 );
|
||||
}
|
||||
this.test_filter = null;
|
||||
if( this.argv.filter ) { // null is the default by definition
|
||||
try {
|
||||
this.test_filter = new RegExp( this.argv.filter );
|
||||
} catch(e) {
|
||||
console.error( "\nERROR> --filter was given an invalid regular expression.");
|
||||
console.error( "ERROR> See below for JS engine error:\n" + e + "\n" );
|
||||
process.exit( 1 );
|
||||
}
|
||||
console.log( "Filtering title test using Regexp " + this.test_filter );
|
||||
}
|
||||
if( !this.argv.color ) {
|
||||
colors.mode = 'none';
|
||||
}
|
||||
|
||||
// Name of file used to cache the parser tests cases
|
||||
this.cache_file = "parserTests.cache";
|
||||
|
||||
// Preload the grammar file...
|
||||
PegTokenizer.src = fs.readFileSync(path.join(basePath, 'parser', 'pegTokenizer.pegjs.txt'), 'utf8');
|
||||
|
||||
this.wikiTokenizer = new PegTokenizer();
|
||||
|
||||
this.testFileName = '../../../../phase3/tests/parser/parserTests.txt'; // default
|
||||
this.testFileName2 = '../../../../tests/parser/parserTests.txt'; // Fallback. Not everyone fetch at phase3 level
|
||||
|
||||
if (this.argv._[0]) {
|
||||
// hack :D
|
||||
this.testFileName = this.argv._[0] ;
|
||||
this.testFileName2 = null;
|
||||
}
|
||||
|
||||
try {
|
||||
this.testParser = PEG.buildParser(fs.readFileSync('parserTests.pegjs', 'utf8'));
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
|
||||
this.cases = this.getTests();
|
||||
|
||||
this.articles = {};
|
||||
|
||||
this.htmlparser = new HTML5.Parser();
|
||||
|
||||
this.postProcessor = new DOMPostProcessor();
|
||||
|
||||
var pt = this;
|
||||
this.tokenTransformer = new TokenTransformer ( function ( tokens ) {
|
||||
//console.log("TOKENS: " + JSON.stringify(tokens, null, 2));
|
||||
// Create a new tree builder, which also creates a new document.
|
||||
var treeBuilder = new FauxHTML5.TreeBuilder();
|
||||
|
||||
// Build a DOM tree from tokens using the HTML tree
|
||||
// builder/parser.
|
||||
pt.buildTree( tokens, treeBuilder );
|
||||
|
||||
// Perform post-processing on DOM.
|
||||
pt.postProcessor.doPostProcess(treeBuilder.parser.document);
|
||||
|
||||
// And serialize the result.
|
||||
var out = treeBuilder.body().innerHTML;
|
||||
|
||||
pt.checkResult( pt.currentItem, out );
|
||||
});
|
||||
|
||||
// Test statistics
|
||||
this.passedTests = 0;
|
||||
this.passedTestsManual = 0;
|
||||
this.failParseTests = 0;
|
||||
this.failTreeTests = 0;
|
||||
this.failOutputTests = 0;
|
||||
|
||||
this.currentItem = undefined;
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
try {
|
||||
var testParser = PEG.buildParser(fs.readFileSync('parserTests.pegjs', 'utf8'));
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Get an object holding our tests cases. Eventually from a cache file
|
||||
*/
|
||||
function getTests() {
|
||||
ParserTests.prototype.getTests = function () {
|
||||
|
||||
// Startup by loading .txt test file
|
||||
var testFile;
|
||||
try {
|
||||
testFile = fs.readFileSync(testFileName, 'utf8');
|
||||
fileDependencies.push( testFileName );
|
||||
testFile = fs.readFileSync(this.testFileName, 'utf8');
|
||||
fileDependencies.push( this.testFileName );
|
||||
} catch (e) {
|
||||
// Try opening fallback file
|
||||
if( testFileName2 !== '' ) {
|
||||
if( this.testFileName2 !== '' ) {
|
||||
try {
|
||||
testFile = fs.readFileSync( testFileName2, 'utf8' );
|
||||
fileDependencies.push( testFileName2 );
|
||||
testFile = fs.readFileSync( this.testFileName2, 'utf8' );
|
||||
fileDependencies.push( this.testFileName2 );
|
||||
}
|
||||
catch(e) { console.log(e); }
|
||||
}
|
||||
}
|
||||
if( !argv.cache ) {
|
||||
if( !this.argv.cache ) {
|
||||
// Cache not wanted, parse file and return object
|
||||
return parseTestCase( testFile );
|
||||
return this.parseTestCase( testFile );
|
||||
}
|
||||
|
||||
// Find out modification time of all files depencies and then hashes those
|
||||
// as a unique value using sha1.
|
||||
var mtimes = '';
|
||||
fileDependencies.sort().forEach( function (file) {
|
||||
mtimes += fs.statSync( file )['mtime'];
|
||||
mtimes += fs.statSync( file ).mtime;
|
||||
});
|
||||
var sha1 = require('crypto').createHash('sha1')
|
||||
.update( mtimes ).digest( 'hex' );
|
||||
|
|
@ -216,8 +249,8 @@ function getTests() {
|
|||
var cache_content;
|
||||
var cache_file_digest;
|
||||
try {
|
||||
console.log( "Looking for cache file " + cache_file );
|
||||
cache_content = fs.readFileSync( cache_file, 'utf8' );
|
||||
console.log( "Looking for cache file " + this.cache_file );
|
||||
cache_content = fs.readFileSync( this.cache_file, 'utf8' );
|
||||
// Fetch previous digest
|
||||
cache_file_digest = cache_content.match( /^CACHE: (\w+)\n/ )[1];
|
||||
} catch(e) {
|
||||
|
|
@ -232,9 +265,9 @@ function getTests() {
|
|||
} else {
|
||||
// Write new file cache, content preprended with current digest
|
||||
console.log( "Cache file either inexistant or outdated" );
|
||||
var parse = parseTestCase( testFile )
|
||||
console.log( "Writing parse result to " +cache_file );
|
||||
fs.writeFileSync( cache_file,
|
||||
var parse = this.parseTestCase( testFile );
|
||||
console.log( "Writing parse result to " + this.cache_file );
|
||||
fs.writeFileSync( this.cache_file,
|
||||
"CACHE: " + sha1 + "\n" + JSON.stringify( parse ),
|
||||
'utf8'
|
||||
);
|
||||
|
|
@ -242,26 +275,23 @@ function getTests() {
|
|||
// We can now return the parsed object
|
||||
return parse;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse given tests cases given as plaintext
|
||||
*/
|
||||
function parseTestCase( content ) {
|
||||
ParserTests.prototype.parseTestCase = function ( content ) {
|
||||
console.log( "Parsing tests case from file, this takes a few seconds ..." );
|
||||
try {
|
||||
return testParser.parse(content);
|
||||
console.log( "Done parsing." );
|
||||
return this.testParser.parse(content);
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var cases = getTests();
|
||||
|
||||
var articles = {};
|
||||
|
||||
function normalizeTitle(name) {
|
||||
ParserTests.prototype.normalizeTitle = function(name) {
|
||||
if (typeof name !== 'string') {
|
||||
throw new Error('nooooooooo not a string');
|
||||
}
|
||||
|
|
@ -271,36 +301,32 @@ function normalizeTitle(name) {
|
|||
throw new Error('Invalid/empty title');
|
||||
}
|
||||
return name;
|
||||
}
|
||||
};
|
||||
|
||||
function fetchArticle(name) {
|
||||
ParserTests.prototype.fetchArticle = function(name) {
|
||||
// very simple for now..
|
||||
var norm = normalizeTitle(name);
|
||||
if (norm in articles) {
|
||||
return articles[norm];
|
||||
if (norm in this.articles) {
|
||||
return this.articles[norm];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
function processArticle(item) {
|
||||
var norm = normalizeTitle(item.title);
|
||||
articles[norm] = item.text;
|
||||
}
|
||||
ParserTests.prototype.processArticle = function(item) {
|
||||
var norm = this.normalizeTitle(item.title);
|
||||
this.articles[norm] = item.text;
|
||||
};
|
||||
|
||||
function nodeToHtml(node) {
|
||||
return $('<div>').append(node).html();
|
||||
}
|
||||
|
||||
var htmlparser = new HTML5.Parser();
|
||||
|
||||
/* Normalize the expected parser output by parsing it using a HTML5 parser and
|
||||
* re-serializing it to HTML. Ideally, the parser would normalize inter-tag
|
||||
* whitespace for us. For now, we fake that by simply stripping all newlines.
|
||||
*/
|
||||
function normalizeHTML(source) {
|
||||
ParserTests.prototype.normalizeHTML = function (source) {
|
||||
// TODO: Do not strip newlines in pre and nowiki blocks!
|
||||
source = source.replace(/\n/g, '');
|
||||
try {
|
||||
htmlparser.parse('<body>' + source + '</body>');
|
||||
return htmlparser.document
|
||||
this.htmlparser.parse('<body>' + source + '</body>');
|
||||
return this.htmlparser.document
|
||||
.getElementsByTagName('body')[0]
|
||||
.innerHTML
|
||||
// a few things we ignore for now..
|
||||
|
|
@ -323,34 +349,44 @@ function normalizeHTML(source) {
|
|||
return source;
|
||||
}
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
// Specialized normalization of the wiki parser output, mostly to ignore a few
|
||||
// known-ok differences.
|
||||
function normalizeOut ( out ) {
|
||||
ParserTests.prototype.normalizeOut = function ( out ) {
|
||||
// TODO: Do not strip newlines in pre and nowiki blocks!
|
||||
return out.replace(/\n| data-[a-zA-Z]+="[^">]*"/g, '')
|
||||
.replace(/<!--.*?-->\n?/gm, '');
|
||||
}
|
||||
};
|
||||
|
||||
function formatHTML ( source ) {
|
||||
ParserTests.prototype.formatHTML = function ( source ) {
|
||||
// Quick hack to insert newlines before some block level start tags
|
||||
return source.replace(
|
||||
/(?!^)<((div|dd|dt|li|p|table|tr|td|tbody|dl|ol|ul|h1|h2|h3|h4|h5|h6)[^>]*)>/g,
|
||||
'\n<$1>');
|
||||
}
|
||||
};
|
||||
|
||||
var passedTests = 0,
|
||||
passedTestsManual = 0,
|
||||
failParseTests = 0,
|
||||
failTreeTests = 0,
|
||||
failOutputTests = 0;
|
||||
|
||||
var postProcessor = new DOMPostProcessor();
|
||||
|
||||
function processTest(item) {
|
||||
// Create a new tree builder, which also creates a new document.
|
||||
var treeBuilder = new FauxHTML5.TreeBuilder();
|
||||
ParserTests.prototype.printTitle = function( item, failure_only ) {
|
||||
if( failure_only ) {
|
||||
console.log('FAILED'.red + ': ' + item.title.yellow);
|
||||
return;
|
||||
}
|
||||
console.log('=====================================================');
|
||||
console.log('FAILED'.red + ': ' + item.title.yellow);
|
||||
console.log(item.comments.join('\n'));
|
||||
if (item.options) {
|
||||
console.log("OPTIONS".cyan + ":");
|
||||
console.log(item.options + '\n');
|
||||
}
|
||||
console.log("INPUT".cyan + ":");
|
||||
console.log(item.input + "\n");
|
||||
};
|
||||
|
||||
|
||||
|
||||
ParserTests.prototype.processTest = function (item) {
|
||||
if (!('title' in item)) {
|
||||
console.log(item);
|
||||
throw new Error('Missing title from test case.');
|
||||
|
|
@ -364,130 +400,108 @@ function processTest(item) {
|
|||
throw new Error('Missing input from test case ' + item.title);
|
||||
}
|
||||
|
||||
function printTitle( failure_only ) {
|
||||
if( failure_only ) {
|
||||
console.log('FAILED'.red + ': ' + item.title.yellow);
|
||||
return;
|
||||
}
|
||||
console.log('=====================================================');
|
||||
console.log('FAILED'.red + ': ' + item.title.yellow);
|
||||
console.log(item.comments.join('\n'));
|
||||
if (item.options) {
|
||||
console.log("OPTIONS".cyan + ":");
|
||||
console.log(item.options + '\n');
|
||||
}
|
||||
console.log("INPUT".cyan + ":");
|
||||
console.log(item.input + "\n");
|
||||
this.currentItem = item;
|
||||
|
||||
// Tokenize the input
|
||||
var res = this.wikiTokenizer.tokenize(item.input + "\n");
|
||||
|
||||
// Check for errors
|
||||
if (res.err) {
|
||||
this.printTitle(item);
|
||||
this.failParseTests++;
|
||||
console.log('PARSE FAIL', res.err);
|
||||
} else {
|
||||
//var environment = new MWParserEnvironment({
|
||||
// tagHooks: {
|
||||
// 'ref': MWRefTagHook,
|
||||
// 'references': MWReferencesTagHook
|
||||
// }
|
||||
//});
|
||||
//var res = es.HtmlSerializer.stringify(tokens,environment);
|
||||
|
||||
//console.log(JSON.stringify(tokens));
|
||||
//Slightly better token output debugging:
|
||||
//console.log( util.inspect( tokens, false, null ).yellow);
|
||||
|
||||
// Transform tokens using the TokenTransformer. When done, the
|
||||
// TokenTransformer calls buildTree() and checkResult() with the
|
||||
// transformed tokens.
|
||||
this.tokenTransformer.transformTokens( res.tokens );
|
||||
}
|
||||
};
|
||||
|
||||
wikiTokenizer.tokenize(item.input + "\n", function(tokens, err) {
|
||||
if (err) {
|
||||
printTitle();
|
||||
failParseTests++;
|
||||
console.log('PARSE FAIL', err);
|
||||
} else {
|
||||
//var environment = new MWParserEnvironment({
|
||||
// tagHooks: {
|
||||
// 'ref': MWRefTagHook,
|
||||
// 'references': MWReferencesTagHook
|
||||
// }
|
||||
//});
|
||||
//var res = es.HtmlSerializer.stringify(tokens,environment);
|
||||
|
||||
//console.log(JSON.stringify(tokens));
|
||||
//Slightly better token output debugging:
|
||||
//console.log( util.inspect( tokens, false, null ).yellow);
|
||||
|
||||
try {
|
||||
// Build a DOM tree from tokens using the HTML tree
|
||||
// builder/parser.
|
||||
processTokens(tokens, treeBuilder);
|
||||
|
||||
// Perform post-processing on DOM.
|
||||
postProcessor.doPostProcess(treeBuilder.parser.document);
|
||||
|
||||
// And serialize the result.
|
||||
var out = treeBuilder.body()
|
||||
.innerHTML;
|
||||
} catch ( e ) {
|
||||
printTitle();
|
||||
failTreeTests++;
|
||||
console.log('RENDER FAIL', e);
|
||||
return;
|
||||
}
|
||||
|
||||
var normalizedOut = normalizeOut(out);
|
||||
var normalizedExpected = normalizeHTML(item.result);
|
||||
if ( normalizedOut !== normalizedExpected ) {
|
||||
if (argv.whitelist &&
|
||||
item.title in testWhiteList &&
|
||||
normalizeOut(testWhiteList[item.title]) === normalizedOut) {
|
||||
if( !argv.quiet ) {
|
||||
console.log( 'PASSED (whiteList)'.green + ': ' + item.title.yellow );
|
||||
}
|
||||
passedTestsManual++;
|
||||
return;
|
||||
}
|
||||
printTitle( argv.quick );
|
||||
failOutputTests++;
|
||||
|
||||
if( !argv.quick ) {
|
||||
console.log('RAW EXPECTED'.cyan + ':');
|
||||
console.log(item.result + "\n");
|
||||
|
||||
console.log('RAW RENDERED'.cyan + ':');
|
||||
console.log(formatHTML(out) + "\n");
|
||||
|
||||
var a = formatHTML(normalizedExpected);
|
||||
|
||||
console.log('NORMALIZED EXPECTED'.magenta + ':');
|
||||
console.log(a + "\n");
|
||||
|
||||
var b = formatHTML(normalizedOut);
|
||||
|
||||
console.log('NORMALIZED RENDERED'.magenta + ':')
|
||||
console.log(formatHTML(normalizeOut(out)) + "\n");
|
||||
var patch = jsDiff.createPatch('wikitext.txt', a, b, 'before', 'after');
|
||||
|
||||
console.log('DIFF'.cyan +': ');
|
||||
|
||||
// Strip the header from the patch, we know how diffs work..
|
||||
patch = patch.replace(/^[^\n]*\n[^\n]*\n[^\n]*\n[^\n]*\n/, '');
|
||||
|
||||
var colored_diff = patch.split( '\n' ).map( function(line) {
|
||||
// Add some colors to diff output
|
||||
switch( line.charAt(0) ) {
|
||||
case '-':
|
||||
return line.red;
|
||||
case '+':
|
||||
return line.blue;
|
||||
default:
|
||||
return line;
|
||||
ParserTests.prototype.checkResult = function ( item, out ) {
|
||||
var normalizedOut = this.normalizeOut(out);
|
||||
var normalizedExpected = this.normalizeHTML(item.result);
|
||||
if ( normalizedOut !== normalizedExpected ) {
|
||||
if (this.argv.whitelist &&
|
||||
item.title in testWhiteList &&
|
||||
this.normalizeOut(testWhiteList[item.title]) === normalizedOut) {
|
||||
if( !this.argv.quiet ) {
|
||||
console.log( 'PASSED (whiteList)'.green + ': ' + item.title.yellow );
|
||||
}
|
||||
}).join( "\n" );
|
||||
|
||||
this.passedTestsManual++;
|
||||
return;
|
||||
}
|
||||
this.printTitle( item, this.argv.quick );
|
||||
this.failOutputTests++;
|
||||
|
||||
console.log( colored_diff );
|
||||
|
||||
if(argv.printwhitelist) {
|
||||
console.log("WHITELIST ENTRY:".cyan);
|
||||
console.log("testWhiteList[" +
|
||||
JSON.stringify(item.title) + "] = " +
|
||||
JSON.stringify(out) +
|
||||
";\n");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
passedTests++;
|
||||
if( !argv.quiet ) {
|
||||
console.log( 'PASSED'.green + ': ' + item.title.yellow );
|
||||
if( !this.argv.quick ) {
|
||||
console.log('RAW EXPECTED'.cyan + ':');
|
||||
console.log(item.result + "\n");
|
||||
|
||||
console.log('RAW RENDERED'.cyan + ':');
|
||||
console.log(this.formatHTML(out) + "\n");
|
||||
|
||||
var a = this.formatHTML(normalizedExpected);
|
||||
|
||||
console.log('NORMALIZED EXPECTED'.magenta + ':');
|
||||
console.log(a + "\n");
|
||||
|
||||
var b = this.formatHTML(normalizedOut);
|
||||
|
||||
console.log('NORMALIZED RENDERED'.magenta + ':');
|
||||
console.log(this.formatHTML(this.normalizeOut(out)) + "\n");
|
||||
var patch = jsDiff.createPatch('wikitext.txt', a, b, 'before', 'after');
|
||||
|
||||
console.log('DIFF'.cyan +': ');
|
||||
|
||||
// Strip the header from the patch, we know how diffs work..
|
||||
patch = patch.replace(/^[^\n]*\n[^\n]*\n[^\n]*\n[^\n]*\n/, '');
|
||||
|
||||
var colored_diff = patch.split( '\n' ).map( function(line) {
|
||||
// Add some colors to diff output
|
||||
switch( line.charAt(0) ) {
|
||||
case '-':
|
||||
return line.red;
|
||||
case '+':
|
||||
return line.blue;
|
||||
default:
|
||||
return line;
|
||||
}
|
||||
}).join( "\n" );
|
||||
|
||||
|
||||
console.log( colored_diff );
|
||||
|
||||
if(this.argv.printwhitelist) {
|
||||
console.log("WHITELIST ENTRY:".cyan);
|
||||
console.log("testWhiteList[" +
|
||||
JSON.stringify(item.title) + "] = " +
|
||||
JSON.stringify(out) +
|
||||
";\n");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
} else {
|
||||
this.passedTests++;
|
||||
if( !this.argv.quiet ) {
|
||||
console.log( 'PASSED'.green + ': ' + item.title.yellow );
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
function processTokens ( tokens, treeBuilder ) {
|
||||
|
||||
ParserTests.prototype.buildTree = function ( tokens, treeBuilder ) {
|
||||
// push a body element, just to be sure to have one
|
||||
treeBuilder.processToken({type: 'TAG', name: 'body'});
|
||||
// Process all tokens
|
||||
|
|
@ -496,37 +510,7 @@ function processTokens ( tokens, treeBuilder ) {
|
|||
}
|
||||
// And signal the end
|
||||
treeBuilder.processToken({type: 'END'});
|
||||
}
|
||||
|
||||
var comments = [];
|
||||
|
||||
console.log( "Initialisation complete. Now launching tests." );
|
||||
cases.forEach(function(item) {
|
||||
if (typeof item == 'object') {
|
||||
switch(item.type) {
|
||||
case 'article':
|
||||
//processArticle(item);
|
||||
comments = [];
|
||||
break;
|
||||
case 'test':
|
||||
if( test_filter && -1 === item.title.search( test_filter ) ) {
|
||||
// Skip test whose title does not match --filter
|
||||
break;
|
||||
}
|
||||
// Add comments to following test.
|
||||
item.comments = comments;
|
||||
comments = [];
|
||||
processTest(item);
|
||||
break;
|
||||
case 'comment':
|
||||
comments.push(item.comment);
|
||||
break;
|
||||
default:
|
||||
comments = [];
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Colorize given number if <> 0
|
||||
|
|
@ -534,7 +518,7 @@ cases.forEach(function(item) {
|
|||
* @param count Integer: a number to colorize
|
||||
* @param color String: 'green' or 'red'
|
||||
*/
|
||||
function ColorizeCount( count, color ) {
|
||||
ParserTests.prototype.ColorizeCount = function ( count, color ) {
|
||||
if( count === 0 ) {
|
||||
return count;
|
||||
}
|
||||
|
|
@ -548,32 +532,87 @@ function ColorizeCount( count, color ) {
|
|||
|
||||
default: return count;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var failTotalTests = (failParseTests + failTreeTests + failOutputTests);
|
||||
ParserTests.prototype.reportSummary = function () {
|
||||
|
||||
console.log( "==========================================================");
|
||||
console.log( "SUMMARY: ");
|
||||
var failTotalTests = (this.failParseTests + this.failTreeTests +
|
||||
this.failOutputTests);
|
||||
|
||||
if( failTotalTests !== 0 ) {
|
||||
console.log( ColorizeCount( passedTests , 'green' ) + " passed");
|
||||
console.log( ColorizeCount( passedTestsManual , 'green' ) + " passed from whitelist");
|
||||
console.log( ColorizeCount( failParseTests , 'red' ) + " parse failures");
|
||||
console.log( ColorizeCount( failTreeTests , 'red' ) + " tree build failures");
|
||||
console.log( ColorizeCount( failOutputTests, 'red' ) + " output differences");
|
||||
console.log( "\n" );
|
||||
console.log( ColorizeCount( passedTests + passedTestsManual , 'green' ) +
|
||||
' total passed tests, ' +
|
||||
ColorizeCount( failTotalTests , 'red' ) + " total failures");
|
||||
console.log( "==========================================================");
|
||||
console.log( "SUMMARY: ");
|
||||
|
||||
if( failTotalTests !== 0 ) {
|
||||
console.log( this.ColorizeCount( this.passedTests , 'green' ) +
|
||||
" passed");
|
||||
console.log( this.ColorizeCount( this.passedTestsManual , 'green' ) +
|
||||
" passed from whitelist");
|
||||
console.log( this.ColorizeCount( this.failParseTests , 'red' ) +
|
||||
" parse failures");
|
||||
console.log( this.ColorizeCount( this.failTreeTests , 'red' ) +
|
||||
" tree build failures");
|
||||
console.log( this.ColorizeCount( this.failOutputTests, 'red' ) +
|
||||
" output differences");
|
||||
console.log( "\n" );
|
||||
console.log( this.ColorizeCount( this.passedTests + this.passedTestsManual , 'green' ) +
|
||||
' total passed tests, ' +
|
||||
this.ColorizeCount( failTotalTests , 'red' ) + " total failures");
|
||||
|
||||
} else {
|
||||
if( test_filter !== null ) {
|
||||
console.log( "Passed " + passedTests + passedTestsManual + " of " + passedTests + " tests matching " + test_filter + "... " + "ALL TESTS PASSED!".green );
|
||||
} else {
|
||||
// Should not happen if it does: Champagne!
|
||||
console.log( "Passed " + passedTests + " of " + passedTests + " tests... " + "ALL TESTS PASSED!".green );
|
||||
if( this.test_filter !== null ) {
|
||||
console.log( "Passed " + this.passedTests + pthis.assedTestsManual +
|
||||
" of " + passedTests + " tests matching " + this.test_filter +
|
||||
"... " + "ALL TESTS PASSED!".green );
|
||||
} else {
|
||||
// Should not happen if it does: Champagne!
|
||||
console.log( "Passed " + this.passedTests + " of " + this.passedTests +
|
||||
" tests... " + "ALL TESTS PASSED!".green );
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log( "==========================================================");
|
||||
console.log( "==========================================================");
|
||||
|
||||
};
|
||||
|
||||
ParserTests.prototype.main = function () {
|
||||
console.log( "Initialisation complete. Now launching tests." );
|
||||
|
||||
var comments = [],
|
||||
pt = this;
|
||||
this.cases.forEach(function(item) {
|
||||
if (typeof item == 'object') {
|
||||
switch(item.type) {
|
||||
case 'article':
|
||||
pt.processArticle(item);
|
||||
comments = [];
|
||||
break;
|
||||
case 'test':
|
||||
if( pt.test_filter &&
|
||||
-1 === item.title.search( pt.test_filter ) ) {
|
||||
// Skip test whose title does not match --filter
|
||||
break;
|
||||
}
|
||||
// Add comments to following test.
|
||||
item.comments = comments;
|
||||
comments = [];
|
||||
pt.processTest(item);
|
||||
break;
|
||||
case 'comment':
|
||||
comments.push(item.comment);
|
||||
break;
|
||||
default:
|
||||
comments = [];
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// print out the summary
|
||||
this.reportSummary();
|
||||
};
|
||||
|
||||
var pt = new ParserTests();
|
||||
console.log(pt.processArticle);
|
||||
pt.main();
|
||||
|
||||
|
||||
})();
|
||||
|
|
|
|||
Loading…
Reference in a new issue