Merge "Compress HTML data with deflate before POSTing"

ApiVisualEditor.php

@@ -301,6 +301,11 @@ class ApiVisualEditor extends ApiBase {
 			$parserParams['oldid'] = $params['oldid'];
 		}
 
+		$html = $params['html'];
+		if ( substr( $html, 0, 11 ) === 'rawdeflate,' ) {
+			$html = gzinflate( base64_decode( substr( $html, 11 ) ) );
+		}
+
 		switch ( $params['paction'] ) {
 			case 'parse':
 				$parsed = $this->getHTML( $page, $parserParams );
@@ -476,7 +481,6 @@ class ApiVisualEditor extends ApiBase {
 					if ( $params['html'] === null ) {
 						$this->dieUsageMsg( 'missingparam', 'html' );
 					}
-					$html = $params['html'];
 					$content = $this->postHTML( $page, $html, $parserParams );
 					if ( $content === false ) {
 						$this->dieUsage( 'Error contacting the Parsoid server', 'parsoidserver' );
@@ -492,7 +496,7 @@ class ApiVisualEditor extends ApiBase {
 						$this->dieUsage( 'No cached serialization found with that key', 'badcachekey' );
 					}
 				} else {
-					$wikitext = $this->postHTML( $page, $params['html'], $parserParams );
+					$wikitext = $this->postHTML( $page, $html, $parserParams );
 					if ( $wikitext === false ) {
 						$this->dieUsage( 'Error contacting the Parsoid server', 'parsoidserver' );
 					}
@@ -507,7 +511,7 @@ class ApiVisualEditor extends ApiBase {
 				break;
 
 			case 'serializeforcache':
-				$key = $this->storeInSerializationCache( $page, $parserParams['oldid'], $params['html'] );
+				$key = $this->storeInSerializationCache( $page, $parserParams['oldid'], $html );
 				$result = array( 'result' => 'success', 'cachekey' => $key );
 				break;
 

ApiVisualEditorEdit.php

@@ -71,13 +71,18 @@ class ApiVisualEditorEdit extends ApiVisualEditor {
 			$parserParams['oldid'] = $params['oldid'];
 		}
 
+		$html = $params['html'];
+		if ( substr( $html, 0, 11 ) === 'rawdeflate,' ) {
+			$html = gzinflate( base64_decode( substr( $html, 11 ) ) );
+		}
+
 		if ( $params['cachekey'] !== null ) {
 			$wikitext = $this->trySerializationCache( $params['cachekey'] );
 			if ( !is_string( $wikitext ) ) {
 				$this->dieUsage( 'No cached serialization found with that key', 'badcachekey' );
 			}
 		} else {
-			$wikitext = $this->postHTML( $page, $params['html'], $parserParams );
+			$wikitext = $this->postHTML( $page, $html, $parserParams );
 			if ( $wikitext === false ) {
 				$this->dieUsage( 'Error contacting the Parsoid server', 'parsoidserver' );
 			}

VisualEditor.php

@@ -94,6 +94,37 @@ $wgResourceModules += array(
 		'targets' => array( 'desktop', 'mobile' ),
 	),
 
+	'Base64.js' => $wgVisualEditorResourceTemplate + array(
+		'scripts' => array(
+			'lib/Base64.js/base64.js',
+		),
+		'targets' => array( 'desktop', 'mobile' ),
+	),
+
+	// This includes typed arrays and base64 polyfills for IE9
+	// All the other polyfills Easy-Deflate provides are
+	// already VE requirements.
+	'easy-deflate.core' => $wgVisualEditorResourceTemplate + array(
+		'scripts' => array(
+			'lib/Easy-Deflate/easydeflate.js',
+			'lib/Easy-Deflate/typedarrays.js',
+		),
+		'dependencies' => array(
+			'Base64.js'
+		),
+		'targets' => array( 'desktop', 'mobile' ),
+	),
+
+	'easy-deflate.deflate' => $wgVisualEditorResourceTemplate + array(
+		'scripts' => array(
+			'lib/Easy-Deflate/deflate.js',
+		),
+		'dependencies' => array(
+			'easy-deflate.core'
+		),
+		'targets' => array( 'desktop', 'mobile' ),
+	),
+
 	'unicodejs.wordbreak' => $wgVisualEditorResourceTemplate + array(
 		'scripts' => array(
 			'lib/ve/modules/unicodejs/unicodejs.js',
@@ -260,6 +291,7 @@ $wgResourceModules += array(
 			'mediawiki.Uri',
 			'mediawiki.user',
 			'mediawiki.util',
+			'easy-deflate.deflate',
 			'user.options',
 			'user.tokens',
 			'ext.visualEditor.base',

lib/Base64.js/LICENSE

@@ -0,0 +1,14 @@
+
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                    Version 2, December 2004
+
+ Copyright (c) 2011..2012 David Chambers <dc@hashify.me>
+
+ Everyone is permitted to copy and distribute verbatim or modified
+ copies of this license document, and changing it is allowed as long
+ as the name is changed.
+
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. You just DO WHAT THE FUCK YOU WANT TO.

lib/Base64.js/README.md

@@ -0,0 +1,34 @@
+# Base64.js
+
+≈ 500 byte* polyfill for browsers which don't provide [`window.btoa`][1] and
+[`window.atob`][2].
+
+Although the script does no harm in browsers which do provide these functions,
+a conditional script loader such as [yepnope][3] can prevent unnecessary HTTP
+requests.
+
+```javascript
+yepnope({
+  test: window.btoa && window.atob,
+  nope: 'base64.js',
+  callback: function () {
+    // `btoa` and `atob` are now safe to use
+  }
+})
+```
+
+Base64.js stems from a [gist][4] by [yahiko][5].
+
+### Running the test suite
+
+    make setup
+    make test
+
+\* Minified and gzipped. Run `make bytes` to verify.
+
+
+[1]: https://developer.mozilla.org/en/DOM/window.btoa
+[2]: https://developer.mozilla.org/en/DOM/window.atob
+[3]: http://yepnopejs.com/
+[4]: https://gist.github.com/229984
+[5]: https://github.com/yahiko

lib/Base64.js/base64.js

@@ -0,0 +1,61 @@
+;(function () {
+
+  var object = typeof exports != 'undefined' ? exports : this; // #8: web workers
+  var chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';
+
+  function InvalidCharacterError(message) {
+    this.message = message;
+  }
+  InvalidCharacterError.prototype = new Error;
+  InvalidCharacterError.prototype.name = 'InvalidCharacterError';
+
+  // encoder
+  // [https://gist.github.com/999166] by [https://github.com/nignag]
+  object.btoa || (
+  object.btoa = function (input) {
+    var str = String(input);
+    for (
+      // initialize result and counter
+      var block, charCode, idx = 0, map = chars, output = '';
+      // if the next str index does not exist:
+      //   change the mapping table to "="
+      //   check if d has no fractional digits
+      str.charAt(idx | 0) || (map = '=', idx % 1);
+      // "8 - idx % 1 * 8" generates the sequence 2, 4, 6, 8
+      output += map.charAt(63 & block >> 8 - idx % 1 * 8)
+    ) {
+      charCode = str.charCodeAt(idx += 3/4);
+      if (charCode > 0xFF) {
+        throw new InvalidCharacterError("'btoa' failed: The string to be encoded contains characters outside of the Latin1 range.");
+      }
+      block = block << 8 | charCode;
+    }
+    return output;
+  });
+
+  // decoder
+  // [https://gist.github.com/1020396] by [https://github.com/atk]
+  object.atob || (
+  object.atob = function (input) {
+    var str = String(input).replace(/=+$/, '');
+    if (str.length % 4 == 1) {
+      throw new InvalidCharacterError("'atob' failed: The string to be decoded is not correctly encoded.");
+    }
+    for (
+      // initialize result and counters
+      var bc = 0, bs, buffer, idx = 0, output = '';
+      // get next character
+      buffer = str.charAt(idx++);
+      // character found in table? initialize bit storage and add its ascii value;
+      ~buffer && (bs = bc % 4 ? bs * 64 + buffer : buffer,
+        // and if not first of each 4 characters,
+        // convert the first 8 bits to one ascii character
+        bc++ % 4) ? output += String.fromCharCode(255 & bs >> (-2 * bc & 6)) : 0
+    ) {
+      // try to find character in table (0-63, not found => -1)
+      buffer = chars.indexOf(buffer);
+    }
+    return output;
+  });
+
+}());

lib/Easy-Deflate/README.md

@@ -0,0 +1,36 @@
+Modified version of Easy-Deflate https://github.com/Jacob-Christian-Munch-Andersen/Easy-Deflate
+
+* Added semi-colons to easydeflate.js so it can be minified
+* Namespaced functions inside global EasyDeflate object
+* Base64 lib replaced with one with detailed license info
+
+Modifications by Ed Sanders, Public Domain.
+
+Easy-Deflate
+============
+
+Library for compressing and decompressing strings in JavaScript, feature full Unicode support and is compatible with most browsers.
+
+Use:
+====
+Copy the script inclusion from demo.html.<br>
+Call deflate(foo) in order to compress a string.<br>
+Call inflate(bar) in order to decompress a string compressed in this manner.<br>
+Both functions return a string, or null in case of illegal input.
+
+The compression works by first UTF-8 encoding the input, then compressing it to a raw deflate stream. The stream is then base64 encoded, and finally the identifier "rawdeflate," is prepended.
+
+Credits:
+========
+Gildas Lormeau made the JavaScript conversion of a Deflate utility: https://github.com/gildas-lormeau/zip.js<br>
+Jacob Christian Munch-Andersen made this package in order to make simple use easier and compatible with older browsers.
+
+The following shims are included:<br>
+es5-shim by Kristopher Michael Kowal https://github.com/kriskowal/es5-shim<br>
+JSON 3 by Kit Cambridge http://bestiejs.github.com/json3/<br>
+Typed arrays light shim by Jacob Christian Munch-Andersen https://github.com/Jacob-Christian-Munch-Andersen/Typed-arrays-light-shim<br>
+<s>base64 by Yaffle https://gist.github.com/1284012</s>
+
+License:
+========
+Main packages come with a BSD licence, the shims, except for base64 that include no license text, each has a permissive license.

lib/Easy-Deflate/easydeflate.js

@@ -0,0 +1,219 @@
+/**
+Copyright (c) 2013, Specialisterne.
+http://specialisterne.com/dk/
+All rights reserved.
+Authors:
+Jacob Christian Munch-Andersen
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**/
+// For information and latest version see: https://github.com/Jacob-Christian-Munch-Andersen/Easy-Deflate
+(function(){
+
+var zip={};
+function UTF8encode(str){
+	var out=[];
+	var a;
+	var c,c2;
+	for(a=0;a<str.length;a++){
+		c=str.charCodeAt(a);
+		if(c<128){
+			out.push(c);
+		}
+		else if(c<2048){
+			out.push((c >> 6)+192);
+			out.push((c & 63)+128);
+		}
+		else if(c<65536){
+			if(c>=0xD800 && c<0xDC00){
+				a++;
+				if(a>=str.length){
+					return null;
+				}
+				c2=str.charCodeAt(a);
+				if(c2>=0xDC00 && c2<0xE000){
+					c=65536+(c-0xD800)*1024+c2-0xDC00;
+					out.push((c >> 18)+240);
+					out.push(((c >> 12) & 63)+128);
+					out.push(((c >> 6) & 63)+128);
+					out.push((c & 63)+128);
+				}
+				else{
+					return null;
+				}
+			}
+			else if(c>=0xDC00 && c<0xE000){
+				return null;
+			}
+			else{
+				out.push((c >> 12)+224);
+				out.push(((c >> 6) & 63)+128);
+				out.push((c & 63)+128);
+			}
+		}
+		else{
+			return null;
+		}
+	}
+	return new Uint8Array(out);
+}
+function UTF8decodeA(arrarr){
+	var result="";
+	var intermediate;
+	var minvalue;
+	var missing=0;
+	var a,b;
+	var arr;
+	var c;
+	var lower,upper;
+	for(a=0;a<arrarr.length;a++){
+		arr=arrarr[a];
+		for(b=0;b<arr.length;b++){
+			c=arr[b];
+			if(missing){
+				if(c>127 && c<192){
+					intermediate=intermediate*64+c-128;
+					missing--;
+					if(!missing){
+						if(intermediate>=minvalue){
+							if(intermediate>=65536){
+								if(intermediate>0x10FFFF){
+									return null;
+								}
+								upper=(intermediate-65536)>>10;
+								lower=intermediate%1024;
+								result+=String.fromCharCode(upper+0xD800,lower+0xDC00);
+							}
+							else{
+								result+=String.fromCharCode(intermediate);
+							}
+						}
+						else{
+							return null;
+						}
+					}
+				}
+				else{
+					return null;
+				}
+			}
+			else if(c<128){
+				result+=String.fromCharCode(c);
+			}
+			else if(c>191 && c<248){
+				if(c<224){
+					intermediate=c-192;
+					minvalue=128;
+					missing=1;
+				}
+				else if(c<240){
+					intermediate=c-224;
+					minvalue=2048;
+					missing=2;
+				}
+				else{
+					intermediate=c-240;
+					minvalue=65536;
+					missing=3;
+				}
+			}
+			else{
+				return null;
+			}
+		}
+	}
+	if(missing){
+		return null;
+	}
+	return result;
+}
+function deflate(str){
+	var a,c;
+	var readlen=50000;
+	var resulta=[];
+	var results="";
+	var b,d;
+	var zipper=new zip.Deflater(9);
+	for(a=0;a<str.length;a+=readlen){
+		d=UTF8encode(str.substr(a,readlen));
+		if(d===null){ //This error may be due to a 4 byte charachter being split, retry with a string that is 1 longer to fix it.
+			d=UTF8encode(str.substr(a,readlen+1));
+			a+=1;
+			if(d===null){
+				return null;
+			}
+		}
+		b=zipper.append(d);
+		if(b.length!==0){
+			resulta.push(b);
+		}
+	}
+	b=zipper.flush();
+	if(b.length!==0){
+		resulta.push(b);
+	}
+	for(a=0;a<resulta.length;a++){
+		for(c=0;c<resulta[a].length;c++){
+			results+=String.fromCharCode(resulta[a][c]);
+		}
+	}
+	return "rawdeflate,"+btoa(results);
+}
+function inflate(dfl){
+	var unzipper=new zip.Inflater();
+	var resulta=[];
+	var dfls;
+	var a,c;
+	var b,d;
+	if(dfl.slice(0,11)!="rawdeflate,"){
+		return null;
+	}
+	try{
+		dfls=atob(dfl.slice(11));
+	}
+	catch(e){
+		return null;
+	}
+	try{
+		for(a=0;a<dfls.length;a+=50000){
+			b=new Uint8Array(Math.min(50000,dfls.length-a));
+			for(c=0;c<b.length;c++){
+				b[c]=dfls.charCodeAt(c+a);
+			}
+			d=unzipper.append(b);
+			if(d.length){
+				resulta.push(d);
+			}
+		}
+		return UTF8decodeA(resulta);
+	}
+	catch(e){
+		return null;
+	}
+}
+
+window.EasyDeflate = {
+	'zip': zip,
+	'inflate': inflate,
+	'deflate': deflate
+};
+
+})();

lib/Easy-Deflate/typedarrays.js

@@ -0,0 +1,111 @@
+/**
+Copyright (c) 2013, Specialisterne.
+All rights reserved.
+Authors:
+Jacob Christian Munch-Andersen
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met: 
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer. 
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution. 
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**/
+;(function(){
+	/**
+	 * Light shim for JavaScript typed arrays.
+	 *
+	 * IMPORTANT: This code is not intended to replicate the behaviour of typed
+	 * arrays in JavaScript exacly, several features are left out or implemented
+	 * dirreferently in order to acheive high performance and browser
+	 * compatibility. Code should be tested thorougly both with this shim active
+	 * and with a native implementation.
+	 * 
+	 * For more information and newest version go to:
+	 * https://github.com/Jacob-Christian-Munch-Andersen/Typed-arrays-light-shim
+	**/
+	function Typedarray(length,elementlength,begin,end){
+		var obj=[]
+		var a
+		if(typeof length=="number"){
+			for(a=0;a<length;a++){
+				obj.push(0)
+			}
+		}
+		else{
+			if(end==null){
+				begin=0
+				end=length.length
+			}
+			for(a=begin;a<end;a++){
+				obj.push(length[a])
+			}
+		}
+		obj.subarray=subarray
+		obj.set=set
+		obj.byteLength=obj.length*elementlength
+		obj.byteOffset=0
+		function subarray(begin,end){
+			return Typedarray(obj,elementlength,begin,end)
+		}
+		function set(arr,w){
+			w=w||0
+			var a
+			var target=obj
+			var len=arr.length
+			for(a=0;a<len;a++,w++){
+				target[w]=arr[a]
+			}
+		}
+		return obj
+	}
+	function array1(length){
+		return Typedarray(length,1)
+	}
+	function array2(length){
+		return Typedarray(length,2)
+	}
+	function array4(length){
+		return Typedarray(length,4)
+	}
+	function array8(length){
+		return Typedarray(length,8)
+	}
+	if(!window.Uint8Array){
+		window.Uint8Array=array1
+	}
+	if(!window.Int8Array){
+		window.Int8Array=array1
+	}
+	if(!window.Uint16Array){
+		window.Uint16Array=array2
+	}
+	if(!window.Int16Array){
+		window.Int16Array=array2
+	}
+	if(!window.Uint32Array){
+		window.Uint32Array=array4
+	}
+	if(!window.Int32Array){
+		window.Int32Array=array4
+	}
+	if(!window.Float32Array){
+		window.Float32Array=array4
+	}
+	if(!window.Float64Array){
+		window.Float64Array=array8
+	}
+}())

modules/ve-mw/init/ve.init.mw.Target.js

@@ -5,7 +5,7 @@
  * @license The MIT License (MIT); see LICENSE.txt
  */
 
-/*global mw */
+/*global mw,EasyDeflate */
 
 /**
  * Initialization MediaWiki target.
@@ -1022,7 +1022,8 @@ ve.init.mw.Target.prototype.prepareCacheKey = function ( doc ) {
 	}
 	this.clearPreparedCacheKey();
 
-	html = this.getHtml( doc );
+	html = EasyDeflate.deflate( this.getHtml( doc ) );
+
 	xhr = this.constructor.static.apiRequest( {
 		'action': 'visualeditor',
 		'paction': 'serializeforcache',
@@ -1102,7 +1103,7 @@ ve.init.mw.Target.prototype.tryWithPreparedCacheKey = function ( doc, options, e
 			data.cachekey = cachekey;
 		} else {
 			// Getting a cache key failed, fall back to sending the HTML
-			data.html = preparedCacheKey && preparedCacheKey.html || target.getHtml( doc );
+			data.html = preparedCacheKey && preparedCacheKey.html || EasyDeflate.deflate( target.getHtml( doc ) );
 			// If using the cache key fails, we'll come back here with cachekey still set
 			delete data.cachekey;
 		}