wikimedia/mediawiki-extensions-VisualEditor
1
0
Fork 0
mirror of https://gerrit.wikimedia.org/r/mediawiki/extensions/VisualEditor synced 2024-11-28 00:00:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

create tokenizer without need to modify namespace with PEG source

Browse source
This commit is contained in:
Neil Kandalgaonkar 2011-12-28 01:36:36 +00:00
parent f7dd83e07b
commit 962d1262fc
Notes: Gabriel Wicke 2012-02-27 16:40:01 +00:00 
https://mediawiki.org/wiki/Special:Code/MediaWiki/107458
2 changed files with 17 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
modules/parser/mediawiki.tokenizer.peg.js
View file

@ -8,8 +8,9 @@
var PEG = require('pegjs');
function PegTokenizer(env) {
function PegTokenizer(env, src) {
this.env = env || {};
this.src = src;
}
PegTokenizer.src = false;
@ -17,7 +18,7 @@ PegTokenizer.src = false;
PegTokenizer.prototype.tokenize = function( text ) {
var out, err;
if ( !this.parser ) {
this.parser = PEG.buildParser(PegTokenizer.src);
this.parser = PEG.buildParser(this.src);
}
try {
out = this.parser.parse(text);

26
tests/parser/parserTests.js
View file

@ -7,11 +7,13 @@
*
* Needs smarter compare, as well as search-y helpers.
*
* 2011-07-20 <brion@pobox.com>
* @author Brion Vibber <brion@pobox.com>
* @author Gabriel Wicke <gwicke@wikimedia.org>
* @author Neil Kandalgaonkar <neilk@wikimedia.org>
*/
(function() {
//"use strict";
console.log( "Starting up JS parser tests" );
var fs = require('fs'),
@ -161,9 +163,15 @@ function ParserTests () {
this.cache_file = "parserTests.cache";
// Preload the grammar file...
PegTokenizer.src = fs.readFileSync(path.join(basePath, 'parser', 'pegTokenizer.pegjs.txt'), 'utf8');
this.wikiTokenizer = new PegTokenizer();
var peg = fs.readFileSync(path.join(basePath, 'parser', 'pegTokenizer.pegjs.txt'), 'utf8');
var parserEnv = {};
//var parserEnv = new MWParserEnvironment({
// tagHooks: {
// 'ref': MWRefTagHook,
// 'references': MWReferencesTagHook
// }
//});
this.wikiTokenizer = new PegTokenizer(parserEnv, peg);
this.testFileName = '../../../../phase3/tests/parser/parserTests.txt'; // default
this.testFileName2 = '../../../../tests/parser/parserTests.txt'; // Fallback. Not everyone fetch at phase3 level
@ -439,13 +447,7 @@ ParserTests.prototype.processTest = function (item) {
this.failParseTests++;
console.log('PARSE FAIL', res.err);
} else {
//var environment = new MWParserEnvironment({
// tagHooks: {
// 'ref': MWRefTagHook,
// 'references': MWReferencesTagHook
// }
//});
//var res = es.HtmlSerializer.stringify(tokens,environment);
//var res = es.HtmlSerializer.stringify(tokens,environment);
//Slightly better token output debugging:
//console.log( util.inspect( res.tokens, false, null ).yellow);