From b1bd0d73ec29e00183e2169c5420c6eac50cfeca Mon Sep 17 00:00:00 2001
From: Gabriel Wicke <gwicke@wikimedia.org>
Date: Thu, 10 May 2012 09:47:53 +0200
Subject: [PATCH] Don't eat end token in ListHandler, and lazier Quote handler
 registration

* Setting the rank on tokens is still used currently, but will be phased out
  in favor of setting it on chunks. Tokens will be immutable to allow sharing
  and caching without a need for cloning.
* Only register for newline and end tokens in QuoteTransformer when active.

Change-Id: I2c45bc7e4a105219a1404ab221eed7f242128f1e
---
 modules/parser/ext.core.ListHandler.js      |  3 ++-
 modules/parser/ext.core.QuoteTransformer.js | 16 +++++++++-------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/modules/parser/ext.core.ListHandler.js b/modules/parser/ext.core.ListHandler.js
index b0a65b2a1c..edeb843a1b 100644
--- a/modules/parser/ext.core.ListHandler.js
+++ b/modules/parser/ext.core.ListHandler.js
@@ -44,7 +44,8 @@ ListHandler.prototype.onNewline = function ( token, frame, prevToken ) {
 };
 
 ListHandler.prototype.onEnd = function( token, frame, prevToken ) {
-	return { tokens: this.end() };
+	token.rank = this.listRank + this.delta;
+	return { tokens: this.end().concat([token]) };
 };
 
 ListHandler.prototype.end = function( ) {
diff --git a/modules/parser/ext.core.QuoteTransformer.js b/modules/parser/ext.core.QuoteTransformer.js
index a81feebc98..b4884ca727 100644
--- a/modules/parser/ext.core.QuoteTransformer.js
+++ b/modules/parser/ext.core.QuoteTransformer.js
@@ -33,14 +33,9 @@ QuoteTransformer.prototype.reset = function ( ) {
 // Register this transformer with the TokenTransformer
 QuoteTransformer.prototype.register = function ( dispatcher ) {
 	this.dispatcher = dispatcher;
-	// Register for NEWLINE and QUOTE tag tokens
-	dispatcher.addTransform( this.onNewLine.bind(this), 
-			this.quoteAndNewlineRank, 'newline' );
+	// Register for QUOTE tag tokens
 	dispatcher.addTransform( this.onQuote.bind(this), 
 			this.quoteAndNewlineRank, 'tag', 'mw-quote' );
-	// Treat end-of-input just the same as a newline
-	dispatcher.addTransform( this.onNewLine.bind(this), 
-			this.quoteAndNewlineRank, 'end' );
 };
 
 // Make a copy of the token context
@@ -69,6 +64,11 @@ QuoteTransformer.prototype.onQuote = function ( token, frame, prevToken ) {
 	
 
 	if ( ! this.isActive ) {
+		this.dispatcher.addTransform( this.onNewLine.bind(this), 
+				this.quoteAndNewlineRank, 'newline' );
+		// Treat end-of-input just the same as a newline
+		this.dispatcher.addTransform( this.onNewLine.bind(this), 
+				this.quoteAndNewlineRank, 'end' );
 		// register for any token if not yet active
 		this.dispatcher.addTransform( this.onAny.bind(this), this.anyRank, 'any' );
 		this.isActive = true;
@@ -203,7 +203,9 @@ QuoteTransformer.prototype.onNewLine = function (  token, frame, prevToken ) {
 	// prepare for next line
 	this.reset();
 
-	// remove 'any' registration
+	// remove 'end', 'newline' and 'any' registrations
+	this.dispatcher.removeTransform( this.quoteAndNewlineRank, 'end' );
+	this.dispatcher.removeTransform( this.quoteAndNewlineRank, 'newline' );
 	this.dispatcher.removeTransform( this.anyRank, 'any' );
 
 	return res;