Don't eat end token in ListHandler, and lazier Quote handler registration

* Setting the rank on tokens is still used currently, but will be phased out
  in favor of setting it on chunks. Tokens will be immutable to allow sharing
  and caching without a need for cloning.
* Only register for newline and end tokens in QuoteTransformer when active.

Change-Id: I2c45bc7e4a105219a1404ab221eed7f242128f1e
This commit is contained in:
Gabriel Wicke 2012-05-10 09:47:53 +02:00
parent a909f1a10c
commit b1bd0d73ec
2 changed files with 11 additions and 8 deletions

View file

@ -44,7 +44,8 @@ ListHandler.prototype.onNewline = function ( token, frame, prevToken ) {
}; };
ListHandler.prototype.onEnd = function( token, frame, prevToken ) { ListHandler.prototype.onEnd = function( token, frame, prevToken ) {
return { tokens: this.end() }; token.rank = this.listRank + this.delta;
return { tokens: this.end().concat([token]) };
}; };
ListHandler.prototype.end = function( ) { ListHandler.prototype.end = function( ) {

View file

@ -33,14 +33,9 @@ QuoteTransformer.prototype.reset = function ( ) {
// Register this transformer with the TokenTransformer // Register this transformer with the TokenTransformer
QuoteTransformer.prototype.register = function ( dispatcher ) { QuoteTransformer.prototype.register = function ( dispatcher ) {
this.dispatcher = dispatcher; this.dispatcher = dispatcher;
// Register for NEWLINE and QUOTE tag tokens // Register for QUOTE tag tokens
dispatcher.addTransform( this.onNewLine.bind(this),
this.quoteAndNewlineRank, 'newline' );
dispatcher.addTransform( this.onQuote.bind(this), dispatcher.addTransform( this.onQuote.bind(this),
this.quoteAndNewlineRank, 'tag', 'mw-quote' ); this.quoteAndNewlineRank, 'tag', 'mw-quote' );
// Treat end-of-input just the same as a newline
dispatcher.addTransform( this.onNewLine.bind(this),
this.quoteAndNewlineRank, 'end' );
}; };
// Make a copy of the token context // Make a copy of the token context
@ -69,6 +64,11 @@ QuoteTransformer.prototype.onQuote = function ( token, frame, prevToken ) {
if ( ! this.isActive ) { if ( ! this.isActive ) {
this.dispatcher.addTransform( this.onNewLine.bind(this),
this.quoteAndNewlineRank, 'newline' );
// Treat end-of-input just the same as a newline
this.dispatcher.addTransform( this.onNewLine.bind(this),
this.quoteAndNewlineRank, 'end' );
// register for any token if not yet active // register for any token if not yet active
this.dispatcher.addTransform( this.onAny.bind(this), this.anyRank, 'any' ); this.dispatcher.addTransform( this.onAny.bind(this), this.anyRank, 'any' );
this.isActive = true; this.isActive = true;
@ -203,7 +203,9 @@ QuoteTransformer.prototype.onNewLine = function ( token, frame, prevToken ) {
// prepare for next line // prepare for next line
this.reset(); this.reset();
// remove 'any' registration // remove 'end', 'newline' and 'any' registrations
this.dispatcher.removeTransform( this.quoteAndNewlineRank, 'end' );
this.dispatcher.removeTransform( this.quoteAndNewlineRank, 'newline' );
this.dispatcher.removeTransform( this.anyRank, 'any' ); this.dispatcher.removeTransform( this.anyRank, 'any' );
return res; return res;