* NoInclude and IncludeOnly improvements

* Tokenizer support for templates and template args in template arguments and titles
* Async attribute expansion fixes

modules/parser/ext.core.NoIncludeOnly.js

@@ -7,12 +7,21 @@
 
 var TokenCollector = require( './ext.util.TokenCollector.js' ).TokenCollector;
 
-function NoInclude( manager ) {
+function NoInclude( manager, isInclude ) {
 	new TokenCollector( 
 			manager,
 			function ( tokens ) { 
-				manager.env.dp( 'noinclude stripping', tokens );
-				return {};
+				if ( isInclude ) {
+					manager.env.dp( 'noinclude stripping', tokens );
+					return {};
+				} else {
+					tokens.shift();
+					if ( tokens.length &&
+						tokens[tokens.length - 1].type !== 'END' ) {
+						tokens.pop();
+					}
+					return { tokens: tokens };
+				}
 			}, // just strip it all..
 			true, // match the end-of-input if </noinclude> is missing
 			0.01, // very early in stage 1, to avoid any further processing.
@@ -21,19 +30,30 @@ function NoInclude( manager ) {
 			);
 }
 
-function OnlyInclude( manager ) {
+function IncludeOnly( manager, isInclude ) {
 	new TokenCollector( 
 			manager,
-			function ( ) { return {} }, // just strip it all..
+			function ( tokens ) { 
+				if ( isInclude ) {
+					tokens.shift();
+					if ( tokens.length &&
+						tokens[tokens.length - 1].type !== 'END' ) {
+							tokens.pop();
+					}
+					return { tokens: tokens };
+				} else {
+					return {};
+				}
+			},
 			true, // match the end-of-input if </noinclude> is missing
 			0.01, // very early in stage 1, to avoid any further processing.
 			'tag',
-			'onlyinclude'
+			'includeonly'
 			);
 }
 
 
 if (typeof module == "object") {
 	module.exports.NoInclude = NoInclude;
-	module.exports.OnlyInclude = OnlyInclude;
+	module.exports.IncludeOnly = IncludeOnly;
 }

modules/parser/ext.core.TemplateHandler.js

@@ -75,8 +75,8 @@ TemplateHandler.prototype.onTemplate = function ( token, cb ) {
 	var attributes = [[[{ type: 'TEXT', value: '' }] , token.target ]]
 			.concat( this._nameArgs( token.orderedArgs ) );
 
-	//console.log( 'before AttributeTransformManager: ' + 
-	//					JSON.stringify( attributes, null, 2 ) );
+	this.manager.env.dp( 'before AttributeTransformManager: ' + 
+						JSON.stringify( attributes, null, 2 ) );
 	new AttributeTransformManager( 
 				this.manager, 
 				this._returnAttributes.bind( this, tplExpandData ) 
@@ -85,7 +85,10 @@ TemplateHandler.prototype.onTemplate = function ( token, cb ) {
 	// Unblock finish
 	if ( ! tplExpandData.attribsAsync ) {
 		// Attributes were transformed synchronously
-		this.manager.env.dp( 'sync attribs for ' + JSON.stringify( token ));
+		this.manager.env.dp ( 
+				'sync attribs for ' + JSON.stringify( tplExpandData.target ),
+				tplExpandData.expandedArgs
+		);
 		// All attributes are fully expanded synchronously (no IO was needed)
 		return this._expandTemplate ( tplExpandData );
 	} else {
@@ -110,7 +113,7 @@ TemplateHandler.prototype._nameArgs = function ( orderedArgs ) {
 			out.push( orderedArgs[i] );
 		}
 	}
-	//console.log( '_nameArgs: ' + JSON.stringify( out ) );
+	this.manager.env.dp( '_nameArgs: ' + JSON.stringify( out ) );
 	return out;
 };
 
@@ -190,6 +193,7 @@ TemplateHandler.prototype._expandTemplate = function ( tplExpandData ) {
 				this.manager.env.KVtoHash( tplExpandData.expandedArgs ),
 				tplExpandData.target
 			);
+	this.manager.env.dp( 'argHash:', this.manager.env.KVtoHash( tplExpandData.expandedArgs ) );
 
 	// Hook up the inputPipeline output events to our handlers
 	inputPipeline.addListener( 'chunk', this._onChunk.bind ( this, tplExpandData ) );
@@ -359,8 +363,8 @@ TemplateHandler.prototype._returnArgAttributes = function ( token, cb, frame, at
 		//		' vs. ' + JSON.stringify( this.manager.args ) ); 
 		res = this.manager.args[argName];
 	} else {
-		//console.log( 'templateArg not found: ' + argName + 
-		//		' vs. ' + JSON.stringify( this.manager.args ) );
+		console.log( 'templateArg not found: ' + argName + 
+				' vs. ' + JSON.stringify( this.manager.args ) );
 		if ( token.attribs.length > 1 ) {
 			res = defaultValue;
 		} else {

modules/parser/mediawiki.TokenTransformManager.js

@@ -664,7 +664,7 @@ SyncTokenTransformManager.prototype.onChunk = function ( tokens ) {
 		}
 	}
 	this.env.dp( 'SyncTokenTransformManager.onChunk: emitting ' + 
-			JSON.stringify( localAccum ) );
+			JSON.stringify( localAccum, null, 2 ) );
 	this.emit( 'chunk', localAccum );
 };
 
@@ -698,7 +698,7 @@ SyncTokenTransformManager.prototype.onEndEvent = function () {
 function AttributeTransformManager ( manager, callback ) {
 	this.manager = manager;
 	this.callback = callback;
-	this.outstanding = 0;
+	this.outstanding = 1;
 	this.kvs = [];
 	//this.pipe = manager.getAttributePipeline( manager.args );
 }
@@ -726,7 +726,7 @@ AttributeTransformManager.prototype.process = function ( attributes ) {
 		pipe.addListener( 'end', 
 				this.onEnd.bind( this, this._returnAttributeKey.bind( this, i ) ) 
 				);
-		pipe.process( attributes[i][0] );
+		pipe.process( attributes[i][0].concat([{type:'END'}]) );
 
 		// transform the value
 		pipe = this.manager.getAttributePipeline( this.manager.args );
@@ -737,7 +737,11 @@ AttributeTransformManager.prototype.process = function ( attributes ) {
 				this.onEnd.bind( this, this._returnAttributeValue.bind( this, i ) ) 
 				);
 		//console.log('starting attribute transform of ' + JSON.stringify( attributes[i][1] ) );
-		pipe.process( attributes[i][1] );
+		pipe.process( attributes[i][1].concat([{type:'END'}]) );
+	}
+	this.outstanding--;
+	if ( this.outstanding == 0 ) {
+		this._returnAttributes();
 	}
 };
 
@@ -758,6 +762,9 @@ AttributeTransformManager.prototype._returnAttributes = function ( ) {
  * Collect chunks returned from the pipeline
  */
 AttributeTransformManager.prototype.onChunk = function ( cb, chunk ) {
+	if ( chunk.length && chunk[chunk.length - 1].type === 'END' ) {
+		chunk.pop();
+	}
 	cb( chunk, true );
 };
 
@@ -773,7 +780,8 @@ AttributeTransformManager.prototype.onEnd = function ( cb ) {
  * Callback for async argument value expansions
  */
 AttributeTransformManager.prototype._returnAttributeValue = function ( ref, tokens, notYetDone ) {
-	//console.log( 'check _returnAttributeValue: ' + JSON.stringify( tokens ) );
+	//console.log( 'check _returnAttributeValue: ' + JSON.stringify( tokens ) + 
+	//		' notYetDone:' + notYetDone );
 	this.kvs[ref].value = this.kvs[ref].value.concat( tokens );
 	if ( ! notYetDone ) {
 		this.outstanding--;
@@ -788,6 +796,8 @@ AttributeTransformManager.prototype._returnAttributeValue = function ( ref, toke
  * Callback for async argument key expansions
  */
 AttributeTransformManager.prototype._returnAttributeKey = function ( ref, tokens, notYetDone ) {
+	//console.log( 'check _returnAttributeKey: ' + JSON.stringify( tokens ) + 
+	//		' notYetDone:' + notYetDone );
 	this.kvs[ref].key = this.kvs[ref].key.concat( tokens );
 	if ( ! notYetDone ) {
 		this.outstanding--;

modules/parser/mediawiki.parser.js

@@ -16,9 +16,9 @@ var fs = require('fs'),
 	PegTokenizer                = require('./mediawiki.tokenizer.peg.js').PegTokenizer,
 	TokenTransformManager       = require('./mediawiki.TokenTransformManager.js'),
 
-	NoOnlyInclude				= require('./ext.core.NoOnlyInclude.js'),
-	OnlyInclude					= NoOnlyInclude.OnlyInclude,
-	NoInclude					= NoOnlyInclude.NoInclude,
+	NoIncludeOnly				= require('./ext.core.NoIncludeOnly.js'),
+	IncludeOnly					= NoIncludeOnly.IncludeOnly,
+	NoInclude					= NoIncludeOnly.NoInclude,
 	QuoteTransformer            = require('./ext.core.QuoteTransformer.js').QuoteTransformer,
 	PostExpandParagraphHandler  = require('./ext.core.PostExpandParagraphHandler.js')
 																.PostExpandParagraphHandler,
@@ -65,7 +65,7 @@ function ParserPipeline( env, inputType ) {
 	};
 
 	// Create an input pipeline for the given input type.
-	this.inputPipeline = this.makeInputPipeline ( inputType );
+	this.inputPipeline = this.makeInputPipeline ( inputType, {}, true );
 	this.inputPipeline.atTopLevel = true;
 
 
@@ -137,7 +137,7 @@ ParserPipeline.prototype.constructor = ParserPipeline;
  * accepts the wiki text this way. The last stage of the input pipeline is
  * always an AsyncTokenTransformManager, which emits its output in events.
  */
-ParserPipeline.prototype.makeInputPipeline = function ( inputType, args ) {
+ParserPipeline.prototype.makeInputPipeline = function ( inputType, args, isNoInclude ) {
 	switch ( inputType ) {
 		case 'text/wiki':
 			//console.log( 'makeInputPipeline ' + JSON.stringify( args ) );
@@ -158,8 +158,9 @@ ParserPipeline.prototype.makeInputPipeline = function ( inputType, args ) {
 				var tokenPreProcessor = new TokenTransformManager.SyncTokenTransformManager ( this.env );
 				tokenPreProcessor.listenForTokensFrom ( wikiTokenizer );
 
+				new IncludeOnly( tokenPreProcessor, ! isNoInclude );
 				// Add noinclude transform for now
-				new NoInclude( tokenPreProcessor );
+				new NoInclude( tokenPreProcessor, ! isNoInclude );
 
 				var tokenExpander = new TokenTransformManager.AsyncTokenTransformManager (
 							{

modules/parser/pegTokenizer.pegjs.txt

@@ -480,7 +480,11 @@ inline_breaks
   / & { return syntaxFlags['extlink']; } "]" { return true; }
   / & { return syntaxFlags['linkdesc']; } link_end { return true; }
   / & { return syntaxFlags['h']; } '='+ space* newline { return true; }
-  / & { return syntaxFlags['template']; } ('|' / '}}') { return true; }
+  / & { return syntaxFlags['template']; } ('|' / '}}' ) { return true; }
+  / & { return syntaxFlags['equal']; } '=' { 
+                    //console.log( 'equal stop!' );
+                    return true; 
+                }
 
 inline
   = c:(urltext / (! inline_breaks (inline_element / . )))+ {
@@ -539,6 +543,7 @@ inline_element
   = //& { dp('inline_element enter' + input.substr(pos, 10)); return true; }
       & '<' ( comment / xmlish_tag )
     / & '{' ( & '{{{{{' template / tplarg / template )
+    /// & '{' ( tplarg / template )
      // Eat three opening brackets as text.
     / '[[[' { return { type: 'TEXT', value: '[[[' } }
     / & '[' ( wikilink / extlink )
@@ -711,25 +716,26 @@ template
             name: 'template', 
             attribs: [['data-target', JSON.stringify(target)]],
             orderedArgs: params,
-            args: {},
+            //args: {},
             target: target
       };
-      if (params && params.length) {
-          var position = 1;
-          for ( var i = 0, l = params.length; i < l; i++ ) {
-              var param = params[i];
-              if ( param[0] === null ) {
-                  obj.args[position] = param[1];
-                  position++;
-              } else {
-                  // Last value wins for duplicates.
-                  obj.args[param[0]] = param[1];
-              }
-          }
-          // HACK: temporarily also push the args into an attribute
-          // (just for debugging)
-          obj.attribs.push(['data-json-args', JSON.stringify(obj.args)]);
-      }
+      // XXX: this is kind of broken, as arg keys need to be expanded
+      //if (params && params.length) {
+      //    var position = 1;
+      //    for ( var i = 0, l = params.length; i < l; i++ ) {
+      //        var param = params[i];
+      //        if ( param[0] === null ) {
+      //            obj.args[position] = param[1];
+      //            position++;
+      //        } else {
+      //            // Last value wins for duplicates.
+      //            obj.args[param[0]] = param[1];
+      //        }
+      //    }
+      //    // HACK: temporarily also push the args into an attribute
+      //    // (just for debugging)
+      //    obj.attribs.push(['data-json-args', JSON.stringify(obj.args)]);
+      //}
       // Should actually use a self-closing tag here, but the Node HTML5
       // parser only recognizes known self-closing tags for now, so use an
       // explicit end tag for now.
@@ -738,18 +744,14 @@ template
   }
 
 // XXX: support template and args in target!
-template_target
-  = h:( !"}}" x:([^|\n]) { return x } )* { return { type: 'TEXT', value: h.join('') } }
-
-template_param
-  = name:template_param_name space* "=" space* c:template_param_text {
-      return [[{ type: 'TEXT', value: name }], flatten( c )];
-  } / c:template_param_text {
-      return [[], flatten( c ) ];
-  }
+//template_target
+//  = h:( !"}}" x:([^|\n]) { return x } )* { return { type: 'TEXT', value: h.join('') } }
 
 tplarg 
-  = "{{{" name:template_param_text params:("|" p:template_param { return p })* "}}}" {
+  = "{{{" 
+    name:template_param_text 
+    params:( newline? "|" newline? p:template_param { return p })* 
+    "}}}" {
       name = flatten( name );
       var obj = { 
             type: 'SELFCLOSINGTAG', 
@@ -764,21 +766,39 @@ tplarg
           obj.attribs.push(['data-json-args', JSON.stringify(params)]);
           obj.defaultvalue = params[0][1];
       }
-      //console.log( 'tokenizer templatearg ' + JSON.stringify( obj ));
+      console.log( 'tokenizer tplarg ' + JSON.stringify( obj ));
       return obj;
   }
 
+template_param
+  = name:template_param_name space* "=" space* c:template_param_text {
+      //console.log( 'named template_param matched' + pp([name, flatten( c )]) );
+      return [name, flatten( c )];
+  } / c:template_param_text {
+      return [[], flatten( c ) ];
+  }
+
+
 // FIXME: handle template args and templates in key! (or even parser functions?)
 template_param_name
-  = h:( !"}}" x:([^=|\n]) { return x } )* { return h.join(''); }
+  = & { return setFlag( 'equal' ) }
+    tpt:template_param_text
+    & { clearFlag( 'equal' ); return true; }
+    {
+        //console.log( 'template param name matched: ' + pp( tpt ) );
+        return tpt;
+    }
+
+  / & { return clearFlag( 'equal' ) }
+  //= h:( !"}}" x:([^=|\n]) { return x } )* { return h.join(''); }
 
 template_param_text
   = & { return setFlag('template') }
-    il:inline+ {
+    il:inline {
         clearFlag('template');
         return il;
     }
-  / & { clearFlag('template'); return false; }
+  / & { return clearFlag('template'); }
 
 // TODO: handle link prefixes as in al[[Razi]]
 wikilink