Add ability to pass through JSON data to WikiDom in data-json-* attributes,

and fix parser to actually parse the Barack Obama article except for one table
with nested templates at the start-of-line.

modules/parser/mediawiki.DOMConverter.js

@@ -395,7 +395,10 @@ DOMConverter.prototype._HTMLPropertiesToWikiAttributes = function ( elem ) {
 		var attrib = attribs.item(i),
 			key = attrib.name;
 		console.log('key: ' + key);
-		if ( key.match( /^data-/ ) ) {
+		if ( key.match( /^data-json-/ ) ) {
+			// strip data- prefix from data-*
+			out[key.replace( /^data-json-/, '' )] = JSON.parse(attrib.value);
+		} else if ( key.match( /^data-/ ) ) {
 			// strip data- prefix from data-*
 			out[key.replace( /^data-/, '' )] = attrib.value;
 		} else {
@@ -412,7 +415,10 @@ DOMConverter.prototype._HTMLPropertiesToWikiData = function ( elem ) {
 	for ( var i = 0, l = attribs.length; i < l; i++ ) {
 		var attrib = attribs.item(i),
 			key = attrib.name;
-		if ( key.match( /^data-/ ) ) {
+		if ( key.match( /^data-json-/ ) ) {
+			// strip data- prefix from data-*
+			out[key.replace( /^data-json-/, '' )] = JSON.parse(attrib.value);
+		} else if ( key.match( /^data-/ ) ) {
 			// strip data- prefix from data-*
 			out[key.replace( /^data-/, '' )] = attrib.value;
 		} else {

modules/parser/mediawiki.HTML5TreeBuilder.node.js

@@ -25,7 +25,7 @@ FauxHTML5.TreeBuilder.prototype = new events.EventEmitter();
 // html tree builder by emitting the token.
 FauxHTML5.TreeBuilder.prototype.processToken = function (token) {
 	var att = function (maybeAttribs) {
-		if ( $.isArray(maybeAttribs) ) {
+		if ( $.isArray( maybeAttribs ) ) {
 			var atts = [];
 			for(var i = 0, length = maybeAttribs.length; i < length; i++) {
 				var att = maybeAttribs[i];
@@ -62,12 +62,16 @@ FauxHTML5.TreeBuilder.prototype.processToken = function (token) {
 			break;
 		case "END":
 			this.emit('end');
+			console.log("at end..");
 			this.document = this.parser.document;
-			// HACK: This should not be needed really.
-			this.document.body = this.document.getElementsByTagName('body')[0];
+			if ( ! this.document.body ) {
+				// HACK: This should not be needed really.
+				this.document.body = this.parser.document.getElementsByTagName('body')[0];
+			}
 			break;
 		case "NEWLINE":
 			//this.emit('end');
+			this.emit('token', {type: 'Characters', data: "\n"});
 			break;
 		default:
 			console.log("Unhandled token: " + JSON.stringify(token));

modules/parser/pegTokenizer.pegjs.txt

@@ -644,7 +644,7 @@ template
     params:(newline? "|" newline? p:template_param { return p })* 
     "}}" {
       var obj = { type: 'TAG', name: 'template', 
-                  attribs: [['target', target]],
+                  attribs: [['data-target', target]],
                   args: {}}
       if (params && params.length) {
           var position = 1;
@@ -659,7 +659,7 @@ template
           }
           // HACK: temporarily also push the args into an attribute
           // (just for debugging)
-          obj.attribs.push(['data-args', JSON.stringify(obj.args)]);
+          obj.attribs.push(['data-json-args', JSON.stringify(obj.args)]);
       }
       // Should actually use a self-closing tag here, but the Node HTML5
       // parser only recognizes known self-closing tags for now, so use an
@@ -687,7 +687,7 @@ tplarg
       };
       if (params && params.length) {
           // HACK, not final.
-          obj.attribs.push(['data-args', JSON.stringify(params)]);
+          obj.attribs.push(['data-json-args', JSON.stringify(params)]);
       }
       return obj;
   }

tests/parser/parserTests.js

@@ -203,7 +203,7 @@ function ParserTests () {
 
 		// Build a DOM tree from tokens using the HTML tree builder/parser.
 		pt.buildTree( tokens, treeBuilder );
-
+		
 		// Perform post-processing on DOM.
 		pt.postProcessor.doPostProcess(treeBuilder.document);
 
@@ -447,9 +447,8 @@ ParserTests.prototype.processTest = function (item) {
 		//});
 		//var res = es.HtmlSerializer.stringify(tokens,environment);
 
-		//console.log(JSON.stringify(tokens));
 		//Slightly better token output debugging:
-		//console.log( util.inspect( tokens, false, null ).yellow);	
+		//console.log( util.inspect( res.tokens, false, null ).yellow);	
 
 		// Transform tokens using the TokenTransformDispatcher. When done, the
 		// TokenTransformDispatcher calls buildTree() and checkResult() with the
@@ -457,6 +456,9 @@ ParserTests.prototype.processTest = function (item) {
 
 		// Append the end
 		res.tokens.push({type: 'END'});
+
+		//console.log(JSON.stringify(res.tokens, null, 2));
+		
 		this.tokenDispatcher.transformTokens( res.tokens );
 	}
 };
@@ -554,6 +556,12 @@ ParserTests.prototype.buildTree = function ( tokens, treeBuilder ) {
 	for (var i = 0, length = tokens.length; i < length; i++) {
 		treeBuilder.processToken(tokens[i]);
 	}
+	
+	// FIXME HACK: For some reason the end token is not processed sometimes,
+	// which normally fixes the body reference up.
+	treeBuilder.document.body = treeBuilder.parser
+		.document.getElementsByTagName('body')[0];
+
 };
 
 /**