From e5a1116817f1cefd0fce4fe1d8b7217e50b60f58 Mon Sep 17 00:00:00 2001
From: Gabriel Wicke <gwicke@users.mediawiki.org>
Date: Wed, 7 Mar 2012 16:29:06 +0000
Subject: [PATCH] Start re-transformation as soon as possible in
 TokenAccumulator._returnTokens to maximize IO concurrency. Signal that all
 tokens are fully transformed to callbacks called from
 TokenAccumulator._returnTokens. The result should be a single
 re-transformation when entering the callback chain, and only if the transform
 does not signal that it took care of full transformation itself. Template
 expansion would set this flag, as the nested transform pipeline processes all
 tokens to the end of phase async12.

---
 .../parser/mediawiki.TokenTransformManager.js | 34 ++++++++++++-------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/modules/parser/mediawiki.TokenTransformManager.js b/modules/parser/mediawiki.TokenTransformManager.js
index 3fa7e407e5..e38a17fb9d 100644
--- a/modules/parser/mediawiki.TokenTransformManager.js
+++ b/modules/parser/mediawiki.TokenTransformManager.js
@@ -458,11 +458,6 @@ AsyncTokenTransformManager.prototype.onChunk = function ( tokens ) {
 		this.tailAccumulator = res.async;
 		this.tokenCB = res.async.getParentCB ( 'sibling' );
 	}
-
-	// The next processed chunk should call back as a sibling to last
-	// accumulator, if any.
-	//if ( res.async ) {
-	//}
 };
 
 /**
@@ -952,21 +947,36 @@ TokenAccumulator.prototype.getParentCB = function ( reference ) {
  */
 TokenAccumulator.prototype._returnTokens = 
 	function ( reference, tokens, notYetDone, allTokensProcessed ) {
-	var res,
-		cb,
+	var cb,
 		returnTokens = [];
 
+
 	if ( ! notYetDone ) {
 		this.outstanding--;
 	}
 
 	//console.warn( 'TokenAccumulator._returnTokens' );
 	if ( reference === 'child' ) {
-		tokens = tokens.concat( this.accum );
+		var res = {};
+		if( !allTokensProcessed ) {
+			// There might be transformations missing on the returned tokens,
+			// re-transform to make sure those are applied too.
+			res = this.manager.transformTokens( tokens, this.parentCB );
+			tokens = res.tokens;
+		}
+
+		if ( !notYetDone ) {
+			// empty accum too
+			tokens = tokens.concat( this.accum );
+			this.accum = [];
+		}
 		this.manager.env.dp( 'TokenAccumulator._returnTokens child: ',
 				tokens, ' outstanding: ', this.outstanding );
-		this.accum = [];
-		this.parentCB( tokens, this.outstanding );
+		this.parentCB( tokens, this.outstanding, true );
+
+		if ( res.async ) {
+			this.parentCB = res.async.getParentCB( 'sibling' );
+		}
 		return null;
 	} else {
 		// sibling
@@ -977,7 +987,7 @@ TokenAccumulator.prototype._returnTokens =
 			this.manager.env.dp( 'TokenAccumulator._returnTokens: ',
 					'sibling done and parentCB ',
 					tokens );
-			this.parentCB( tokens, false );
+			this.parentCB( tokens, false, true );
 			return null;
 		} else if ( this.outstanding === 1 && notYetDone ) {
 			this.manager.env.dp( 'TokenAccumulator._returnTokens: ',
@@ -987,7 +997,7 @@ TokenAccumulator.prototype._returnTokens =
 			// allow the sibling to go direct, and call back parent with
 			// tokens. The internal accumulator is empty at this stage, as its
 			// tokens are passed to the parent when the child is done.
-			return this.parentCB( tokens, true);
+			return this.parentCB( tokens, true, true);
 		} else {
 			this.accum  = this.accum.concat( tokens );
 			this.manager.env.dp( 'TokenAccumulator._returnTokens: sibling done, but not overall. notYetDone=',