From 8fbf36e63e62322b6e994460293efd57214281fb Mon Sep 17 00:00:00 2001
From: Neil Kandalgaonkar <neilk@users.mediawiki.org>
Date: Wed, 28 Dec 2011 01:37:15 +0000
Subject: [PATCH] put add terminal token inside tokenize method (will pull it
 out again for streaming interface)

---
 modules/parser/mediawiki.parser.js        | 1 -
 modules/parser/mediawiki.tokenizer.peg.js | 6 ++++++
 modules/parser/parse.js                   | 5 +----
 tests/parser/parserTests.js               | 3 ---
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/modules/parser/mediawiki.parser.js b/modules/parser/mediawiki.parser.js
index 63fba0dfd0..a53b0f5c83 100644
--- a/modules/parser/mediawiki.parser.js
+++ b/modules/parser/mediawiki.parser.js
@@ -8,7 +8,6 @@
 
 var fs = require('fs'),
 	path = require('path'),
-	$ = require('jquery'),
 	PegTokenizer                = require('./mediawiki.tokenizer.peg.js').PegTokenizer,
 	TokenTransformDispatcher    = require('./mediawiki.TokenTransformDispatcher.js').TokenTransformDispatcher,
 	DOMPostProcessor            = require('./mediawiki.DOMPostProcessor.js').DOMPostProcessor,
diff --git a/modules/parser/mediawiki.tokenizer.peg.js b/modules/parser/mediawiki.tokenizer.peg.js
index 9b2ed7c61f..2ce5cee7ac 100644
--- a/modules/parser/mediawiki.tokenizer.peg.js
+++ b/modules/parser/mediawiki.tokenizer.peg.js
@@ -32,6 +32,12 @@ PegTokenizer.prototype.tokenize = function( text ) {
 		err = e;
 		console.trace();
 	} finally {
+
+		// Append the end (for obvious reasons this should not
+		// be part of a stream, only when tokenizing complete
+		// texts)
+		out.push({type: 'END'});
+
 		return {tokens: out, err: err};
 	}
 }
diff --git a/modules/parser/parse.js b/modules/parser/parse.js
index 56083f5288..a45d9c924f 100644
--- a/modules/parser/parse.js
+++ b/modules/parser/parse.js
@@ -32,15 +32,12 @@
 	 * @param {String} text
 	 */
 	function getOutput( parser, input ) {
-		var res = parser.wikiTokenizer.tokenize(input + "\n");
+		var res = parser.wikiTokenizer.tokenize(input);
 		if (res.err) {
 			console.log('PARSE FAIL', res.err);
 			process.exit(1);
 		} 
 
-		// Append the end
-		res.tokens.push({type: 'END'});
-
 		parser.tokenDispatcher.transformTokens( res.tokens );
 
 		return parser.getWikiDom();
diff --git a/tests/parser/parserTests.js b/tests/parser/parserTests.js
index 7c5a29fae1..b3f0b49204 100644
--- a/tests/parser/parserTests.js
+++ b/tests/parser/parserTests.js
@@ -399,9 +399,6 @@ ParserTests.prototype.processTest = function (item, pThingy) {
 		// TokenTransformDispatcher calls buildTree() and checkResult() with the
 		// transformed tokens.
 
-		// Append the end
-		res.tokens.push({type: 'END'});
-
 		//console.log(JSON.stringify(res.tokens, null, 2));
 		
 		pThingy.tokenDispatcher.transformTokens( res.tokens );