put add terminal token inside tokenize method (will pull it out again for streaming interface)

https://mediawiki.org/wiki/Special:Code/MediaWiki/107465

modules/parser/mediawiki.parser.js

@@ -8,7 +8,6 @@
 
 var fs = require('fs'),
 	path = require('path'),
-	$ = require('jquery'),
 	PegTokenizer                = require('./mediawiki.tokenizer.peg.js').PegTokenizer,
 	TokenTransformDispatcher    = require('./mediawiki.TokenTransformDispatcher.js').TokenTransformDispatcher,
 	DOMPostProcessor            = require('./mediawiki.DOMPostProcessor.js').DOMPostProcessor,

modules/parser/mediawiki.tokenizer.peg.js

@@ -32,6 +32,12 @@ PegTokenizer.prototype.tokenize = function( text ) {
 		err = e;
 		console.trace();
 	} finally {
+
+		// Append the end (for obvious reasons this should not
+		// be part of a stream, only when tokenizing complete
+		// texts)
+		out.push({type: 'END'});
+
 		return {tokens: out, err: err};
 	}
 }

modules/parser/parse.js

@@ -32,15 +32,12 @@
 	 * @param {String} text
 	 */
 	function getOutput( parser, input ) {
-		var res = parser.wikiTokenizer.tokenize(input + "\n");
+		var res = parser.wikiTokenizer.tokenize(input);
 		if (res.err) {
 			console.log('PARSE FAIL', res.err);
 			process.exit(1);
 		} 
 
-		// Append the end
-		res.tokens.push({type: 'END'});
-
 		parser.tokenDispatcher.transformTokens( res.tokens );
 
 		return parser.getWikiDom();

tests/parser/parserTests.js

@@ -399,9 +399,6 @@ ParserTests.prototype.processTest = function (item, pThingy) {
 		// TokenTransformDispatcher calls buildTree() and checkResult() with the
 		// transformed tokens.
 
-		// Append the end
-		res.tokens.push({type: 'END'});
-
 		//console.log(JSON.stringify(res.tokens, null, 2));
 		
 		pThingy.tokenDispatcher.transformTokens( res.tokens );