AllInfoboxes and PIDataService improvements

extension.json

@@ -13,6 +13,7 @@
 		"MediaWiki": ">= 1.27.0"
 	},
 	"config": {
+		"AllInfoboxesCompatibleMode": false,
 		"AllInfoboxesMiserMode": true,
 		"AllInfoboxesSubpagesBlacklist": [ "doc", "draft", "test" ],
 		"PortableInfoboxCustomImageWidth": 300,

includes/controllers/PortableInfoboxParserTagController.php

@@ -58,6 +58,31 @@ class PortableInfoboxParserTagController {
 	 * @throws InvalidInfoboxParamsException when unsupported attributes exist in params array
 	 */
 	public function render( $markup, Parser $parser, PPFrame $frame, $params = null ) {
+		$data = $this->prepareInfobox( $markup, $parser, $frame, $params );
+
+		$themeList = $this->getThemes( $params, $frame );
+		$layout = $this->getLayout( $params );
+		$accentColor = $this->getColor( self::ACCENT_COLOR, $params, $frame );
+		$accentColorText = $this->getColor( self::ACCENT_COLOR_TEXT, $params, $frame );
+
+		$renderService = new PortableInfoboxRenderService();
+		return $renderService->renderInfobox(
+			$data, implode( ' ', $themeList ), $layout, $accentColor, $accentColorText
+		);
+	}
+
+	/**
+	 * @param string $markup
+	 * @param Parser $parser
+	 * @param PPFrame $frame
+	 * @param array|null $params
+	 *
+	 * @return array
+	 * @throws UnimplementedNodeException when node used in markup does not exists
+	 * @throws XmlMarkupParseErrorException xml not well formatted
+	 * @throws InvalidInfoboxParamsException when unsupported attributes exist in params array
+	 */
+	public function prepareInfobox( $markup, Parser $parser, PPFrame $frame, $params = null ) {
 		$frameArguments = $frame->getArguments();
 		$infoboxNode = Nodes\NodeFactory::newFromXML( $markup, $frameArguments ? $frameArguments : [] );
 		$infoboxNode->setExternalParser(
@@ -75,15 +100,7 @@ class PortableInfoboxParserTagController {
 		// save for later api usage
 		$this->saveToParserOutput( $parser->getOutput(), $infoboxNode );
 
-		$themeList = $this->getThemes( $params, $frame );
-		$layout = $this->getLayout( $params );
-		$accentColor = $this->getColor( self::ACCENT_COLOR, $params, $frame );
-		$accentColorText = $this->getColor( self::ACCENT_COLOR_TEXT, $params, $frame );
-
-		$renderService = new PortableInfoboxRenderService();
-		return $renderService->renderInfobox(
-			$data, implode( ' ', $themeList ), $layout, $accentColor, $accentColorText
-		);
+		return $data;
 	}
 
 	/**

includes/querypage/AllinfoboxesQueryPage.php

@@ -1,17 +1,24 @@
 <?php
 
+use PortableInfobox\Helpers\PortableInfoboxParsingHelper;
+
 class AllinfoboxesQueryPage extends PageQueryPage {
 
 	const ALL_INFOBOXES_TYPE = 'AllInfoboxes';
-	private static $subpagesBlacklist = [];
+	private $compatibleMode;
+	private $subpagesBlacklist = [];
+	private $parsingHelper;
 
 	function __construct() {
 		parent::__construct( self::ALL_INFOBOXES_TYPE );
 
 		$blacklist = $this->getConfig( 'AllInfoboxesSubpagesBlacklist' );
 		if ( is_array( $blacklist ) ) {
-			self::$subpagesBlacklist = $blacklist;
+			$this->subpagesBlacklist = $blacklist;
 		}
+
+		$this->compatibleMode = $this->getConfig()->get( 'AllInfoboxesCompatibleMode' );
+		$this->parsingHelper = new PortableInfoboxParsingHelper();
 	}
 
 	function getGroupName() {
@@ -42,18 +49,33 @@ class AllinfoboxesQueryPage extends PageQueryPage {
 	}
 
 	function getQueryInfo() {
-		return [
+		$query = [
 			'tables' => [ 'page' ],
 			'fields' => [
-				'namespace' => 'page_namespace',
-				'title' => 'page_title',
-				'value' => 'page_id'
+				'namespace' => 'page.page_namespace',
+				'title' => 'page.page_title',
+				'value' => 'page.page_id'
 			],
 			'conds' => [
-				'page_is_redirect' => 0,
-				'page_namespace' => NS_TEMPLATE
+				'page.page_is_redirect' => 0,
+				'page.page_namespace' => NS_TEMPLATE
 			]
 		];
+
+		if ( !$this->compatibleMode ) {
+			$query = array_merge_recursive( $query, [
+				'tables' => [ 'revision', 'text' ],
+				'fields' => [
+					'text' => 'text.old_text'
+				],
+				'join_conds' => [
+					'revision' => [ 'LEFT JOIN', 'page.page_latest = revision.rev_id' ],
+					'text' => [ 'LEFT JOIN', 'revision.rev_text_id = text.old_id AND text.old_flags = "utf-8"' ]
+				]
+			] );
+		}
+
+		return $query;
 	}
 
 	/**
@@ -104,20 +126,19 @@ class AllinfoboxesQueryPage extends PageQueryPage {
 		return new FakeResultWrapper( $out );
 	}
 
-	private function hasInfobox( Title $title ) {
-		// omit subages from blacklist
-		return !(
-				$title->isSubpage() &&
-				in_array( mb_strtolower( $title->getSubpageText() ), self::$subpagesBlacklist )
-			) &&
-			!empty( PortableInfoboxDataService::newFromTitle( $title )->getData() );
-	}
-
 	private function filterInfoboxes( $tmpl ) {
 		$title = Title::newFromID( $tmpl->value );
 
 		return $title &&
 			$title->exists() &&
-			$this->hasInfobox( $title );
+			!(
+				$title->isSubpage() &&
+				in_array( mb_strtolower( $title->getSubpageText() ), $this->subpagesBlacklist )
+			) &&
+			(
+				$this->compatibleMode ?
+				!empty( PortableInfoboxDataService::newFromTitle( $title )->getData() ) :
+				$this->parsingHelper->hasInfobox( is_null( $tmpl->text ) ? $title : $tmpl->text )
+			);
 	}
 }

includes/services/Helpers/PagePropsProxy.php

@@ -4,6 +4,13 @@ namespace PortableInfobox\Helpers;
 
 class PagePropsProxy {
 
+	protected $atomicStarted;
+	protected $manualWrite;
+
+	public function __construct( $manualWrite = false ) {
+		$this->manualWrite = $manualWrite;
+	}
+
 	public function get( $id, $property ) {
 		$dbr = wfGetDB( DB_REPLICA );
 		$propValue = $dbr->selectField(
@@ -20,23 +27,38 @@ class PagePropsProxy {
 
 	public function set( $id, array $props ) {
 		$dbw = wfGetDB( DB_MASTER );
-		$dbw->startAtomic( __METHOD__ );
+
+		if ( !$this->atomicStarted ) {
+			$dbw->startAtomic( __METHOD__ );
+			$this->atomicStarted = true;
+		}
+
 		foreach ( $props as $sPropName => $sPropValue ) {
 			$dbw->replace(
-				"page_props",
+				'page_props',
 				[
-					"pp_page",
-					"pp_propname"
+					'pp_page',
+					'pp_propname'
 				],
 				[
-					"pp_page" => $id,
-					"pp_propname" => $sPropName,
-					"pp_value" => $sPropValue
+					'pp_page' => $id,
+					'pp_propname' => $sPropName,
+					'pp_value' => $sPropValue
 				],
 				__METHOD__
 			);
 		}
-		$dbw->endAtomic( __METHOD__ );
+
+		if ( !$this->manualWrite ) {
+			$dbw->endAtomic( __METHOD__ );
+			$this->atomicStarted = false;
+		}
 	}
 
+	public function write() {
+		if ( $this->atomicStarted && $this->manualWrite ) {
+			wfGetDB( DB_MASTER )->endAtomic( __CLASS__ . '::set' );
+			$this->atomicStarted = false;
+		}
+	}
 }

includes/services/Helpers/PortableInfoboxParsingHelper.php

@@ -3,6 +3,7 @@
 namespace PortableInfobox\Helpers;
 
 use MediaWiki\Logger\LoggerFactory;
+use PortableInfobox\Parser\Nodes\NodeFactory;
 
 class PortableInfoboxParsingHelper {
 
@@ -23,7 +24,7 @@ class PortableInfoboxParsingHelper {
 	 */
 	public function parseIncludeonlyInfoboxes( $title ) {
 		// for templates we need to check for include tags
-		$templateText = $this->removeNowikiPre( $this->fetchArticleContent( $title ) );
+		$templateText = $this->fetchArticleContent( $title );
 
 		if ( $templateText ) {
 			$parser = new \Parser();
@@ -31,15 +32,14 @@ class PortableInfoboxParsingHelper {
 			$frame = $parser->getPreprocessor()->newFrame();
 
 			$includeonlyText = $parser->getPreloadText( $templateText, $title, $parserOptions );
-			$infoboxes = $this->getInfoboxes( $includeonlyText );
+			$infoboxes = $this->getInfoboxes( $this->removeNowikiPre( $includeonlyText ) );
 
 			if ( $infoboxes ) {
-				// clear up cache before parsing
 				foreach ( $infoboxes as $infobox ) {
 					try {
-						$this->parserTagController->render( $infobox, $parser, $frame );
+						$this->parserTagController->prepareInfobox( $infobox, $parser, $frame );
 					} catch ( \Exception $e ) {
-						$this->logger->info( 'Invalid infobox syntax in includeonly tag' );
+						$this->logger->info( 'Invalid infobox syntax' );
 					}
 				}
 
@@ -49,11 +49,10 @@ class PortableInfoboxParsingHelper {
 				);
 			}
 		}
-
 		return false;
 	}
 
-	public function reparseArticle( $title ) {
+	public function reparseArticle( \Title $title ) {
 		$parser = new \Parser();
 		$parserOptions = new \ParserOptions();
 		$parser->parse( $this->fetchArticleContent( $title ), $title, $parserOptions );
@@ -64,6 +63,33 @@ class PortableInfoboxParsingHelper {
 		);
 	}
 
+	public function hasInfobox( $template ) {
+		$parser = new \Parser();
+		$parserOptions = new \ParserOptions();
+
+		if ( $template instanceof \Title ) {
+			$text = $this->fetchArticleContent( $template );
+			$title = $template;
+		} else {
+			$text = $template;
+			$title = new \Title();
+		}
+
+		$includeonlyText = $parser->getPreloadText( $text, $title, $parserOptions );
+		$infoboxes = $this->getInfoboxes( $this->removeNowikiPre( $includeonlyText ) );
+
+		if ( $infoboxes ) {
+			try {
+				NodeFactory::newFromXML( $infoboxes[0] );
+				return true;
+			} catch ( \Exception $e ) {
+				$this->logger->info( 'Invalid infobox syntax' );
+			}
+		}
+
+		return false;
+	}
+
 	/**
 	 * @param \Title $title
 	 *
@@ -71,13 +97,9 @@ class PortableInfoboxParsingHelper {
 	 */
 	protected function fetchArticleContent( \Title $title ) {
 		if ( $title && $title->exists() ) {
-			$wikipage = \WikiPage::factory( $title );
-
-			if ( $wikipage && $wikipage->exists() ) {
-				$content = \ContentHandler::getContentText(
-					$wikipage->getRevision()->getContent( \Revision::RAW )
-				);
-			}
+			$content = \WikiPage::factory( $title )
+				->getContent( \Revision::FOR_PUBLIC )
+				->getNativeData();
 		}
 
 		return isset( $content ) && $content ? $content : '';
@@ -100,8 +122,7 @@ class PortableInfoboxParsingHelper {
 	 * @return string
 	 */
 	protected function removeNowikiPre( $text ) {
-		$text = preg_replace( "/<nowiki>.+<\/nowiki>/sU", '', $text );
-		$text = preg_replace( "/<pre>.+<\/pre>/sU", '', $text );
+		$text = preg_replace( '/<(nowiki|pre)>.+<\/\g1>/sU', '', $text );
 
 		return $text;
 	}
@@ -115,9 +136,7 @@ class PortableInfoboxParsingHelper {
 	 * @return array of striped infoboxes ready to parse
 	 */
 	protected function getInfoboxes( $text ) {
-		preg_match_all( "/<infobox[^>]*\\/>/sU", $text, $empty );
-		preg_match_all( "/<infobox.+<\/infobox>/sU", $text, $result );
-
-		return array_merge( $empty[0], $result[0] );
+		preg_match_all( '/<infobox(?:[^>]*\/>|.+<\/infobox>)/sU', $text, $result );
+		return $result[0];
 	}
 }

includes/services/PortableInfoboxDataService.php

@@ -62,14 +62,13 @@ class PortableInfoboxDataService {
 	 * @return array in format [ [ 'data' => [], 'metadata' => [] ] or [] will be returned
 	 */
 	public function getData() {
-		if ( $this->title && $this->title->exists() && $this->title->inNamespace( NS_TEMPLATE ) ) {
-			$incOnlyTemplates = $this->parsingHelper->parseIncludeonlyInfoboxes( $this->title );
-			$this->delete();
-			$this->set( $incOnlyTemplates );
+		if ( $this->title->exists() && $this->title->inNamespace( NS_TEMPLATE ) ) {
+			$result = $this->reparseArticle();
+		} else {
+			$result = $this->get();
 		}
-		$result = $this->get();
 
-		return $result !== null ? $result : [];
+		return $result ? $result : [];
 	}
 
 	/**
@@ -198,6 +197,22 @@ class PortableInfoboxDataService {
 		return [];
 	}
 
+	protected function reparseArticle() {
+		if ( $this->title->inNamespace( NS_TEMPLATE ) ) {
+			$result = $this->parsingHelper->parseIncludeonlyInfoboxes( $this->title );
+		} else {
+			$result = $this->parsingHelper->reparseArticle( $this->title );
+		}
+
+		if ( $result ) {
+			$this->set( $result );
+		} else {
+			$this->delete();
+		}
+
+		return $result;
+	}
+
 	/**
 	 * If PageProps has an old version of infobox data/metadata then reparse the page
 	 * and store fresh data. If it doesn't have infoboxes property,
@@ -215,8 +230,7 @@ class PortableInfoboxDataService {
 					!isset( $infobox['parser_tag_version'] ) ||
 					$infobox['parser_tag_version'] !== PortableInfoboxParserTagController::PARSER_TAG_VERSION
 				) {
-					$infoboxes = $this->parsingHelper->reparseArticle( $this->title );
-					$this->set( $infoboxes );
+					$infoboxes = $this->reparseArticle();
 				}
 			}
 		}