Implement html/rest.js gateway which handles HTML Restbase responses

Refactor existing Restbase gateway and extract shared logic into
shared Restbase provider. Also introduced new createNullModel()
which defines an empty preview model.

Additionally improve naming in new gateways/formatter so function
names are more explicity.
 * Htmlize() became formatPlainTextExtract() as it should be used
   only with plain text extracts
 * removeEllipsis() became  removeTrailingEllipsis() as it removes
   only trailing ellipsis.
 * src/gateway/index.js defines gateways by configuration name stored
   in extension.json

Bug: T165018
Change-Id: Ibe54dddfc1080e94814d1562d41e85cb6b43bfc1
Depends-On: I4f42c61b155a37c5dd42bc40034583865abd5d7a
This commit is contained in:
Piotr Miazga 2017-06-08 15:29:57 +02:00
parent 9c2a4b143f
commit f2fbef6ec7
16 changed files with 165 additions and 79 deletions

View file

@ -55,8 +55,8 @@
"PopupsOptInDefaultState": "0",
"@PopupsConflictingNavPopupsGadgetName": "@var string: Navigation popups gadget name",
"PopupsConflictingNavPopupsGadgetName": "Navigation_popups",
"@PopupsAPIUseRESTBase": "Whether to use RESTBase rather than the MediaWiki API for fetching Popups data.",
"PopupsAPIUseRESTBase": false,
"@PopupsGateway": "Which gateway to use for fetching Popups data. Available options: [mwApiPlain|restbasePlain|restbaseHTML]. Full and always up to date list is available in src/gateway/index.js",
"PopupsGateway": "mwApiPlain",
"@PopupsAnonsEnabledSamplingRate": "Sampling rate for showing popups to anonymous users.",
"PopupsAnonsEnabledSamplingRate": 0.9,
"@PopupsStatsvSamplingRate": "Sampling rate for logging performance data to statsv.",

View file

@ -120,7 +120,7 @@ class PopupsHooks {
$conf = PopupsContext::getInstance()->getConfig();
$vars['wgPopupsSchemaSamplingRate'] = $conf->get( 'PopupsSchemaSamplingRate' );
$vars['wgPopupsBetaFeature'] = $conf->get( 'PopupsBetaFeature' );
$vars['wgPopupsAPIUseRESTBase'] = $conf->get( 'PopupsAPIUseRESTBase' );
$vars['wgPopupsGateway'] = $conf->get( 'PopupsGateway' );
$vars['wgPopupsAnonsEnabledSamplingRate'] = $conf->get( 'PopupsAnonsEnabledSamplingRate' );
$vars['wgPopupsStatsvSamplingRate'] = $conf->get( 'PopupsStatsvSamplingRate' );
}

Binary file not shown.

Binary file not shown.

View file

@ -7,14 +7,14 @@ var $ = jQuery,
* @param {String} title
* @returns {Array}
*/
function htmlize( plainTextExtract, title ) {
function formatPlainTextExtract( plainTextExtract, title ) {
var extract = plainTextExtract;
if ( plainTextExtract === undefined ) {
return [];
}
extract = removeParentheticals( extract );
extract = removeEllipsis( extract );
extract = removeTrailingEllipsis( extract );
// After cleaning the extract it may have been blanked
if ( extract.length === 0 ) {
@ -78,7 +78,7 @@ function makeTitleInExtractBold( extract, title ) {
* @param {String} extract
* @return {String}
*/
function removeEllipsis( extract ) {
function removeTrailingEllipsis( extract ) {
return extract.replace( /\.\.\.$/, '' );
}
@ -127,5 +127,7 @@ function removeParentheticals( extract ) {
}
module.exports = {
htmlize: htmlize
formatPlainTextExtract: formatPlainTextExtract,
removeTrailingEllipsis: removeTrailingEllipsis,
removeParentheticals: removeParentheticals
};

35
src/gateway/html/rest.js Normal file
View file

@ -0,0 +1,35 @@
var formatter = require( '../../formatter' ),
restbaseProvider = require( '../restProvider' ),
$ = jQuery;
/**
* Creates an instance of the RESTBase gateway.
*
* This gateway differs from the {@link MediaWikiGateway MediaWiki gateway} in
* that it fetches page data from [the RESTBase page summary endpoint][0].
*
* [0]: https://en.wikipedia.org/api/rest_v1/#!/Page_content/get_page_summary_title
*
* @param {Function} ajax A function with the same signature as `jQuery.ajax`
* @param {Object} config Configuration that affects the major behavior of the
* gateway.
* @param {Number} config.THUMBNAIL_SIZE The length of the major dimension of
* the thumbnail.
* @returns {RESTBaseGateway}
*/
module.exports = function createRESTHTMLBaseGateway( ajax, config ) {
return restbaseProvider( ajax, config, parseHTMLResponse );
};
/**
* Prepare extract
* @param {Object} page Rest response
* @returns {Array} An array of DOM Elements
*/
function parseHTMLResponse( page ) {
var extract = page.extract_html;
extract = formatter.removeTrailingEllipsis( extract );
extract = formatter.removeParentheticals( extract );
return extract.length === 0 ? [] : $.parseHTML( extract );
}

View file

@ -18,6 +18,7 @@
*/
module.exports = {
createMediaWikiApiGateway: require( './mediawiki' ),
createRESTBaseGateway: require( './rest' )
mwApiPlain: require( './plain/mediawiki' ),
restbasePlain: require( './plain/rest' ),
restbaseHTML: require( './html/rest' )
};

View file

@ -13,8 +13,8 @@
//
// FIXME: Move this to src/constants.js.
var CACHE_LIFETIME = 300,
createModel = require( '../preview/model' ).createModel,
plainTextHTMLizer = require( '../formatter' ).htmlize,
modelBuilder = require( '../../preview/model' ),
formatter = require( '../../formatter' ),
$ = jQuery;
/**
@ -71,7 +71,7 @@ module.exports = function createMediaWikiApiGateway( api, config ) {
function getPageSummary( title ) {
return fetch( title )
.then( extractPageFromResponse )
.then( htmlize )
.then( formatPlainTextExtract )
.then( convertPageToModel );
}
@ -80,7 +80,7 @@ module.exports = function createMediaWikiApiGateway( api, config ) {
extractPageFromResponse: extractPageFromResponse,
convertPageToModel: convertPageToModel,
getPageSummary: getPageSummary,
htmlize: htmlize
formatPlainTextExtract: formatPlainTextExtract
};
};
@ -107,16 +107,16 @@ function extractPageFromResponse( data ) {
}
/**
* HTMLize plain text response
* Make plain text nicer by applying formatter.
*
* @function
* @name MediaWikiGateway#htmlize
* @name MediaWikiGateway#formatPlainTextExtract
* @param {Object} data The response
* @returns {Object}
*/
function htmlize( data ) {
function formatPlainTextExtract( data ) {
var result = $.extend( {}, data );
result.extract = plainTextHTMLizer( data.extract, data.title );
result.extract = formatter.formatPlainTextExtract( data.extract, data.title );
return result;
}
@ -129,7 +129,7 @@ function htmlize( data ) {
* @returns {PreviewModel}
*/
function convertPageToModel( page ) {
return createModel(
return modelBuilder.createModel(
page.title,
page.canonicalurl,
page.pagelanguagehtmlcode,

30
src/gateway/plain/rest.js Normal file
View file

@ -0,0 +1,30 @@
var formatter = require( '../../formatter' ),
restbaseProvider = require( '../restProvider' );
/**
* Creates an instance of the RESTBase gateway that returns plain text
*
* This gateway differs from the {@link MediaWikiGateway MediaWiki gateway} in
* that it fetches page data from [the RESTBase page summary endpoint][0].
*
* [0]: https://en.wikipedia.org/api/rest_v1/#!/Page_content/get_page_summary_title
*
* @param {Function} ajax A function with the same signature as `jQuery.ajax`
* @param {Object} config Configuration that affects the major behavior of the
* gateway.
* @param {Number} config.THUMBNAIL_SIZE The length of the major dimension of
* the thumbnail.
* @returns {RESTBaseGateway}
*/
module.exports = function createRESTBaseGateway( ajax, config ) {
return restbaseProvider( ajax, config, parsePlainTextResponse );
};
/**
* Prepare extract
* @param {Object} page Rest response
* @returns {Array} An array of DOM Elements
*/
function parsePlainTextResponse( page ) {
return formatter.formatPlainTextExtract( page.extract, page.title );
}

View file

@ -4,11 +4,9 @@
var RESTBASE_ENDPOINT = '/api/rest_v1/page/summary/',
RESTBASE_PROFILE = 'https://www.mediawiki.org/wiki/Specs/Summary/1.2.0',
createModel = require( '../preview/model' ).createModel,
plainTextHTMLizer = require( '../formatter' ).htmlize,
modelBuilder = require( '../preview/model' ),
mw = window.mediaWiki,
$ = jQuery;
/**
* @interface RESTBaseGateway
* @extends Gateway
@ -29,9 +27,10 @@ var RESTBASE_ENDPOINT = '/api/rest_v1/page/summary/',
* gateway.
* @param {Number} config.THUMBNAIL_SIZE The length of the major dimension of
* the thumbnail.
* @param {Function} extractParser A function that takes response and returns parsed extract
* @returns {RESTBaseGateway}
*/
module.exports = function createRESTBaseGateway( ajax, config ) {
module.exports = function createRESTBaseGateway( ajax, config, extractParser ) {
/**
* Fetches page data from [the RESTBase page summary endpoint][0].
@ -60,17 +59,13 @@ module.exports = function createRESTBaseGateway( ajax, config ) {
.then(
function ( page ) {
result.resolve(
convertPageToModel( page, config.THUMBNAIL_SIZE ) );
convertPageToModel( page, config.THUMBNAIL_SIZE, extractParser ) );
},
function ( jqXHR ) {
if ( jqXHR.status === 404 ) {
result.resolve(
convertPageToModel( {
title: title,
lang: '',
dir: '',
extract: ''
}, 0 )
modelBuilder.createNullModel( title )
);
} else {
result.reject();
@ -117,7 +112,7 @@ function generateThumbnailData( thumbnail, original, thumbSize ) {
// where the thumbnail's extension is .svg.png.
filename = lastPart.substr( lastPart.indexOf( 'px-' ) + 3 );
// Scale the thumbnail's largest dimension.
// Scale the thumbnail's largest dimension.
if ( thumbnail.width > thumbnail.height ) {
width = thumbSize;
height = Math.floor( ( thumbSize / thumbnail.width ) * thumbnail.height );
@ -148,15 +143,16 @@ function generateThumbnailData( thumbnail, original, thumbSize ) {
* @name RESTBaseGateway#convertPageToModel
* @param {Object} page
* @param {Number} thumbSize
* @param {Function} extractParser
* @returns {PreviewModel}
*/
function convertPageToModel( page, thumbSize ) {
return createModel(
function convertPageToModel( page, thumbSize, extractParser ) {
return modelBuilder.createModel(
page.title,
new mw.Title( page.title ).getUrl(),
page.lang,
page.dir,
plainTextHTMLizer( page.extract, page.title ),
extractParser( page ),
page.thumbnail ? generateThumbnailData( page.thumbnail, page.originalimage, thumbSize ) : undefined
);
}

View file

@ -8,8 +8,7 @@ var mw = mediaWiki,
ReduxThunk = require( 'redux-thunk' ),
constants = require( './constants' ),
createRESTBaseGateway = require( './gateway/rest' ),
createMediaWikiApiGateway = require( './gateway/mediawiki' ),
gatewayBuilder = require( './gateway/index' ),
createUserSettings = require( './userSettings' ),
createPreviewBehavior = require( './previewBehavior' ),
createSchema = require( './schema' ),
@ -47,10 +46,16 @@ var mw = mediaWiki,
* @return {ext.popups.Gateway}
*/
function createGateway( config ) {
if ( config.get( 'wgPopupsAPIUseRESTBase' ) ) {
return createRESTBaseGateway( $.ajax, constants );
switch ( config.get( 'wgPopupsGateway' ) ) {
case 'mwApiPlain':
return gatewayBuilder.mwApiPlain( new mw.Api(), constants );
case 'restbasePlain':
return gatewayBuilder.restbasePlain( $.ajax, constants );
case 'restbaseHTML':
return gatewayBuilder.restbaseHTML( $.ajax, constants );
default:
throw new Error( 'Unknown gateway' );
}
return createMediaWikiApiGateway( new mw.Api(), constants );
}
/**

View file

@ -2,19 +2,15 @@
* @module preview/model
*/
/**
* @constant {String}
*/
var TYPE_GENERIC = 'generic',
/**
* @constant {String}
*/
TYPE_PAGE = 'page';
/**
* @constant {String}
*/
exports.TYPE_GENERIC = TYPE_GENERIC;
/**
* @constant {String}
*/
exports.TYPE_PAGE = TYPE_PAGE;
/**
* @typedef {Object} PreviewModel
* @property {String} title
@ -24,11 +20,17 @@ exports.TYPE_PAGE = TYPE_PAGE;
* @property {?Array} extract `undefined` if the extract isn't
* viable, e.g. if it's empty after having ellipsis and parentheticals
* removed
* @property {String} type Either "EXTRACT" or "GENERIC"
* @property {String} type Either "extract" or "generic"
* @property {?Object} thumbnail
*
* @global
*/
module.exports = {
TYPE_GENERIC: TYPE_GENERIC,
TYPE_PAGE: TYPE_PAGE,
createModel: createModel,
createNullModel: createNullModel
};
/**
* Creates a preview model.
@ -41,7 +43,7 @@ exports.TYPE_PAGE = TYPE_PAGE;
* @param {?Object} thumbnail
* @return {PreviewModel}
*/
exports.createModel = function createModel(
function createModel(
title,
url,
languageCode,
@ -60,7 +62,17 @@ exports.createModel = function createModel(
type: processedExtract === undefined ? TYPE_GENERIC : TYPE_PAGE,
thumbnail: thumbnail
};
};
}
/**
* Creates an empty preview model.
*
* @param {String} title
* @return {PreviewModel}
*/
function createNullModel( title ) {
return createModel( title, '', '', '', [], '' );
}
/**
* Processes the extract returned by the TextExtracts MediaWiki API query
@ -69,11 +81,11 @@ exports.createModel = function createModel(
* If the extract is `undefined`, `null`, or empty, then `undefined` is
* returned.
*
* @param {?Array} extract
* @return {?String}
* @param {Array|undefined|null} extract
* @return {Array|undefined} Array when extract is an not empty array, undefined otherwise
*/
function processExtract( extract ) {
if ( extract === undefined || extract.length === 0 ) {
if ( extract === undefined || extract === null || extract.length === 0 ) {
return undefined;
}
return extract;

View file

@ -55,7 +55,7 @@ QUnit.test( 'Title is bold', function ( assert ) {
function test( extract, title, expected, msg ) {
var $div = $( '<div>' ).append(
formatter.htmlize( extract, title )
formatter.formatPlainTextExtract( extract, title )
);
assert.equal( $div.html(), expected, msg );
}
@ -86,7 +86,7 @@ QUnit.test( 'it strips ellipsis and parentheticals', function ( assert ) {
testCase = cases[ i ];
$div = $( '<div>' ).append(
formatter.htmlize( testCase[ 0 ], 'Test' )
formatter.formatPlainTextExtract( testCase[ 0 ], 'Test' )
);
assert.equal( $div.html(), testCase[ 1 ] );

View file

@ -1,5 +1,5 @@
var createModel = require( '../../../src/preview/model' ).createModel,
createMediaWikiApiGateway = require( '../../../src/gateway/mediawiki' ),
createMediaWikiApiGateway = require( '../../../src/gateway/plain/mediawiki' ),
DEFAULT_CONSTANTS = {
THUMBNAIL_SIZE: 300,
EXTRACT_LENGTH: 525
@ -146,7 +146,7 @@ QUnit.test( 'MediaWiki API gateway is correctly converting the page data to a mo
page = gateway.extractPageFromResponse( MEDIAWIKI_API_RESPONSE );
assert.deepEqual(
gateway.convertPageToModel( gateway.htmlize( page ) ),
gateway.convertPageToModel( gateway.formatPlainTextExtract( page ) ),
MEDIAWIKI_API_RESPONSE_PREVIEW_MODEL
);
} );

View file

@ -1,5 +1,5 @@
var createModel = require( '../../../src/preview/model' ).createModel,
createRESTBaseGateway = require( '../../../src/gateway/rest' ),
createRESTBaseGateway = require( '../../../src/gateway/restProvider' ),
DEFAULT_CONSTANTS = {
THUMBNAIL_SIZE: 512
},
@ -97,7 +97,7 @@ var createModel = require( '../../../src/preview/model' ).createModel,
'url/Barack Obama', // Generated in the stub below
'en',
'ltr',
[ document.createTextNode( 'Barack Hussein Obama II born August 4, 1961) ' ) ],
'!Barack Hussein Obama II born August 4, 1961) ...!',
{
source: 'https://upload.wikimedia.org/wikipedia/commons/thumb/8/8d/President_Barack_Obama.jpg/409px-President_Barack_Obama.jpg',
width: 409,
@ -105,19 +105,17 @@ var createModel = require( '../../../src/preview/model' ).createModel,
}
);
function provideParsedExtract( page ) {
return '!' + page.extract + '!';
}
QUnit.module( 'gateway/rest', {
beforeEach: function () {
window.mediaWiki.RegExp = {
escape: this.sandbox.spy( function ( str ) {
return str.replace( /([\\{}()|.?*+\-\^$\[\]])/g, '\\$1' );
} )
};
window.mediaWiki.Title = function ( title ) {
this.getUrl = function () { return 'url/' + title; };
};
},
afterEach: function () {
window.mediaWiki.RegExp = null;
window.mediaWiki.Title = null;
}
} );
@ -134,15 +132,22 @@ QUnit.test( 'RESTBase gateway is called with correct arguments', function ( asse
};
gateway.fetch( 'Test Title' );
assert.deepEqual( getSpy.getCall( 0 ).args[ 0 ], expectedOptions, 'options' );
} );
QUnit.test( 'RESTBase provider uses extract parser', function ( assert ) {
var getSpy = this.sandbox.spy(),
gateway = createRESTBaseGateway();
gateway.convertPageToModel( RESTBASE_RESPONSE, 512, getSpy );
assert.deepEqual( getSpy.getCall( 0 ).args[ 0 ], RESTBASE_RESPONSE );
} );
QUnit.test( 'RESTBase gateway is correctly converting the page data to a model ', function ( assert ) {
var gateway = createRESTBaseGateway();
assert.deepEqual(
gateway.convertPageToModel( RESTBASE_RESPONSE, 512 ),
gateway.convertPageToModel( RESTBASE_RESPONSE, 512, provideParsedExtract ),
RESTBASE_RESPONSE_PREVIEW_MODEL
);
} );
@ -151,7 +156,7 @@ QUnit.test( 'RESTBase gateway doesn\'t stretch thumbnails', function ( assert )
var model,
gateway = createRESTBaseGateway();
model = gateway.convertPageToModel( RESTBASE_RESPONSE, 2000 );
model = gateway.convertPageToModel( RESTBASE_RESPONSE, 2000, provideParsedExtract );
assert.deepEqual(
model.thumbnail,
@ -160,7 +165,7 @@ QUnit.test( 'RESTBase gateway doesn\'t stretch thumbnails', function ( assert )
);
// ---
model = gateway.convertPageToModel( RESTBASE_RESPONSE, RESTBASE_RESPONSE.originalimage.height );
model = gateway.convertPageToModel( RESTBASE_RESPONSE, RESTBASE_RESPONSE.originalimage.height, provideParsedExtract );
assert.deepEqual(
model.thumbnail,
@ -169,7 +174,7 @@ QUnit.test( 'RESTBase gateway doesn\'t stretch thumbnails', function ( assert )
);
// ---
model = gateway.convertPageToModel( RESTBASE_RESPONSE_WITH_SMALL_IMAGE, 320 );
model = gateway.convertPageToModel( RESTBASE_RESPONSE_WITH_SMALL_IMAGE, 320, provideParsedExtract );
assert.deepEqual(
model.thumbnail,
@ -178,7 +183,7 @@ QUnit.test( 'RESTBase gateway doesn\'t stretch thumbnails', function ( assert )
);
// ---
model = gateway.convertPageToModel( RESTBASE_RESPONSE_WITH_LANDSCAPE_IMAGE, 640 );
model = gateway.convertPageToModel( RESTBASE_RESPONSE_WITH_LANDSCAPE_IMAGE, 640, provideParsedExtract );
assert.deepEqual(
model.thumbnail,
@ -200,7 +205,7 @@ QUnit.test( 'RESTBase gateway handles awkwardly thumbnails', function ( assert )
response.thumbnail = Object.assign( {}, RESTBASE_RESPONSE.thumbnail );
response.thumbnail.source = 'http://foo.bar/baz/Qux-320px-Quux.png/800px-Qux-320px-Quux.png';
model = gateway.convertPageToModel( response, 500 );
model = gateway.convertPageToModel( response, 500, provideParsedExtract );
assert.deepEqual(
model.thumbnail.source,
@ -213,7 +218,7 @@ QUnit.test( 'RESTBase gateway stretches SVGs', function ( assert ) {
var model,
gateway = createRESTBaseGateway();
model = gateway.convertPageToModel( SVG_RESTBASE_RESPONSE, 2000 );
model = gateway.convertPageToModel( SVG_RESTBASE_RESPONSE, 2000, provideParsedExtract );
assert.equal(
model.thumbnail.source,
@ -238,7 +243,7 @@ QUnit.test( 'RESTBase gateway handles the API failure', function ( assert ) {
QUnit.test( 'RESTBase gateway does not treat a 404 as a failure', function ( assert ) {
var deferred = $.Deferred(),
api = this.sandbox.stub().returns( deferred.reject( { status: 404 } ).promise() ),
gateway = createRESTBaseGateway( api ),
gateway = createRESTBaseGateway( api, { THUMBNAIL_SIZE: 200 }, provideParsedExtract ),
done = assert.async( 1 );
gateway.getPageSummary( 'Test Title' ).done( function () {
@ -251,7 +256,7 @@ QUnit.test( 'RESTBase gateway returns the correct data ', function ( assert ) {
var api = this.sandbox.stub().returns(
$.Deferred().resolve( RESTBASE_RESPONSE ).promise()
),
gateway = createRESTBaseGateway( api, DEFAULT_CONSTANTS ),
gateway = createRESTBaseGateway( api, DEFAULT_CONSTANTS, provideParsedExtract ),
done = assert.async( 1 );
gateway.getPageSummary( 'Test Title' ).done( function ( result ) {
@ -263,7 +268,7 @@ QUnit.test( 'RESTBase gateway returns the correct data ', function ( assert ) {
QUnit.test( 'RESTBase gateway handles missing images ', function ( assert ) {
var model,
gateway = createRESTBaseGateway();
model = gateway.convertPageToModel( RESTBASE_RESPONSE_WITHOUT_IMAGE, 300 );
model = gateway.convertPageToModel( RESTBASE_RESPONSE_WITHOUT_IMAGE, 300, provideParsedExtract );
assert.equal(
model.originalimage,
@ -283,7 +288,7 @@ QUnit.test( 'RESTBase gateway handles missing pages ', function ( assert ) {
api = this.sandbox.stub().returns(
$.Deferred().reject( response ).promise()
),
gateway = createRESTBaseGateway( api, DEFAULT_CONSTANTS ),
gateway = createRESTBaseGateway( api, DEFAULT_CONSTANTS, provideParsedExtract ),
done = assert.async( 1 );
gateway.getPageSummary( 'Missing Page' ).fail( function () {

View file

@ -178,7 +178,7 @@ class PopupsHooksTest extends MediaWikiTestCase {
$config = [
'wgPopupsSchemaSamplingRate' => 10,
'wgPopupsBetaFeature' => true,
'wgPopupsAPIUseRESTBase' => false,
'wgPopupsGateway' => 'mwApiPlain',
'wgPopupsAnonsEnabledSamplingRate' => 0.9,
'wgPopupsStatsvSamplingRate' => 0
];