Merge "Trigger Parsoid run when page metadata is being updated"

extension.json

@@ -30,9 +30,12 @@
 			"services": [
 				"LinkRenderer",
 				"JobQueueGroup",
+				"ParsoidParserFactory",
+				"WikiPageFactory",
 				"Linter.CategoryManager",
 				"Linter.TotalsLookup",
-				"Linter.Database"
+				"Linter.Database",
+				"MainConfig"
 			]
 		},
 		"schema": {
@@ -46,7 +49,8 @@
 		"InfoAction": "main",
 		"WikiPageDeletionUpdates": "main",
 		"RevisionFromEditComplete": "main",
-		"ParserLogLinterData": "main"
+		"ParserLogLinterData": "main",
+		"RevisionDataUpdates": "main"
 	},
 	"APIListModules": {
 		"linterrors": {
@@ -247,6 +251,9 @@
 		},
 		"LinterUserInterfaceTagAndTemplateStage": {
 			"value": false
+		},
+		"LinterParseOnDerivedDataUpdate": {
+			"value": true
 		}
 	},
 	"ServiceWiringFiles": [

includes/Hooks.php

@@ -21,11 +21,13 @@
 namespace MediaWiki\Linter;
 
 use ApiQuerySiteinfo;
+use Config;
 use Content;
 use IContextSource;
 use JobQueueError;
 use JobQueueGroup;
 use MediaWiki\Api\Hook\APIQuerySiteInfoGeneralInfoHook;
+use MediaWiki\Deferred\DeferrableUpdate;
 use MediaWiki\Deferred\MWCallableUpdate;
 use MediaWiki\Hook\BeforePageDisplayHook;
 use MediaWiki\Hook\InfoActionHook;
@@ -35,8 +37,12 @@ use MediaWiki\Logger\LoggerFactory;
 use MediaWiki\Output\OutputPage;
 use MediaWiki\Page\Hook\RevisionFromEditCompleteHook;
 use MediaWiki\Page\Hook\WikiPageDeletionUpdatesHook;
+use MediaWiki\Page\WikiPageFactory;
+use MediaWiki\Parser\Parsoid\ParsoidParserFactory;
+use MediaWiki\Revision\RenderedRevision;
 use MediaWiki\Revision\RevisionRecord;
 use MediaWiki\SpecialPage\SpecialPage;
+use MediaWiki\Storage\Hook\RevisionDataUpdatesHook;
 use MediaWiki\Title\Title;
 use MediaWiki\User\UserIdentity;
 use Skin;
@@ -48,17 +54,23 @@ class Hooks implements
 	InfoActionHook,
 	ParserLogLinterDataHook,
 	RevisionFromEditCompleteHook,
-	WikiPageDeletionUpdatesHook
+	WikiPageDeletionUpdatesHook,
+	RevisionDataUpdatesHook
 {
 	private LinkRenderer $linkRenderer;
 	private JobQueueGroup $jobQueueGroup;
+	private ParsoidParserFactory $parsoidParserFactory;
+	private WikiPageFactory $wikiPageFactory;
 	private CategoryManager $categoryManager;
 	private TotalsLookup $totalsLookup;
 	private Database $database;
+	private bool $parseOnDerivedDataUpdates;
 
 	/**
 	 * @param LinkRenderer $linkRenderer
 	 * @param JobQueueGroup $jobQueueGroup
+	 * @param ParsoidParserFactory $parsoidParserFactory
+	 * @param WikiPageFactory $wikiPageFactory
 	 * @param CategoryManager $categoryManager
 	 * @param TotalsLookup $totalsLookup
 	 * @param Database $database
@@ -66,15 +78,21 @@ class Hooks implements
 	public function __construct(
 		LinkRenderer $linkRenderer,
 		JobQueueGroup $jobQueueGroup,
+		ParsoidParserFactory $parsoidParserFactory,
+		WikiPageFactory $wikiPageFactory,
 		CategoryManager $categoryManager,
 		TotalsLookup $totalsLookup,
-		Database $database
+		Database $database,
+		Config $config
 	) {
 		$this->linkRenderer = $linkRenderer;
 		$this->jobQueueGroup = $jobQueueGroup;
+		$this->parsoidParserFactory = $parsoidParserFactory;
+		$this->wikiPageFactory = $wikiPageFactory;
 		$this->categoryManager = $categoryManager;
 		$this->totalsLookup = $totalsLookup;
 		$this->database = $database;
+		$this->parseOnDerivedDataUpdates = $config->get( 'LinterParseOnDerivedDataUpdate' );
 	}
 
 	/**
@@ -240,6 +258,7 @@ class Hooks implements
 	): bool {
 		$errors = [];
 		$title = Title::newFromText( $page );
+
 		if (
 			!$title || !$title->getArticleID() ||
 			$title->getLatestRevID() != $revision
@@ -310,4 +329,35 @@ class Hooks implements
 
 		return true;
 	}
+
+	/**
+	 * @param Title $title
+	 * @param RenderedRevision $renderedRevision
+	 * @param DeferrableUpdate[] &$updates
+	 */
+	public function onRevisionDataUpdates( $title, $renderedRevision, &$updates ) {
+		if ( !$this->parseOnDerivedDataUpdates ) {
+			return;
+		}
+
+		if ( $renderedRevision->getOptions()->getUseParsoid() ) {
+			// Parsoid was already used for the canonical parse, nothing to do:
+			// onParserLogLinterData was already called.
+			// This will be the case when parsoid page views are enabled.
+			// Eventually, ParserLogLinterData will probably go away and we'll
+			// have the lint data in the ParserOutput. We'll then just use
+			// that data to create a RecordLintJob.
+			return;
+		}
+
+		if ( !in_array( $title->getContentModel(), self::LINTABLE_CONTENT_MODELS ) ) {
+			return;
+		}
+
+		$updates[] = new LintUpdate(
+			$this->parsoidParserFactory->create(),
+			$this->wikiPageFactory,
+			$renderedRevision
+		);
+	}
 }

includes/LintUpdate.php

@@ -0,0 +1,92 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+namespace MediaWiki\Linter;
+
+use MediaWiki\Content\TextContent;
+use MediaWiki\Deferred\DataUpdate;
+use MediaWiki\Logger\LoggerFactory;
+use MediaWiki\Page\WikiPageFactory;
+use MediaWiki\Parser\Parsoid\ParsoidParser;
+use MediaWiki\Revision\RenderedRevision;
+use MediaWiki\Revision\RevisionRecord;
+use MediaWiki\Revision\SlotRecord;
+
+class LintUpdate extends DataUpdate {
+
+	private ParsoidParser $parsoid;
+	private WikiPageFactory $wikiPageFactory;
+	private RenderedRevision $renderedRevision;
+
+	public function __construct(
+		ParsoidParser $parsoid,
+		WikiPageFactory $wikiPageFactory,
+		RenderedRevision $renderedRevision
+	) {
+		parent::__construct();
+		$this->parsoid = $parsoid;
+		$this->wikiPageFactory = $wikiPageFactory;
+		$this->renderedRevision = $renderedRevision;
+	}
+
+	public function doUpdate() {
+		$rev = $this->renderedRevision->getRevision();
+		$mainSlot = $rev->getSlot( SlotRecord::MAIN, RevisionRecord::RAW );
+
+		$page = $this->wikiPageFactory->newFromTitle( $rev->getPage() );
+
+		if ( $page->getLatest() !== $rev->getId() ) {
+			// The given revision is no longer the latest revision.
+			return;
+		}
+
+		$content = $mainSlot->getContent();
+		if ( !$content instanceof TextContent ) {
+			// Linting is only defined for text
+			return;
+		}
+
+		$pOptions = $page->makeParserOptions( 'canonical' );
+		$pOptions->setUseParsoid();
+
+		LoggerFactory::getInstance( 'Linter' )->debug(
+			'{method}: Parsing {page}',
+			[
+				'method' => __METHOD__,
+				'page' => $page->getTitle()->getPrefixedDBkey(),
+				'touched' => $page->getTouched()
+			]
+		);
+
+		// Don't update the parser cache, to avoid flooding it.
+		// This matches the behavior of RefreshLinksJob.
+		// However, unlike RefreshLinksJob, we don't parse if we already
+		// have the output in the cache. This avoids duplicating the effort
+		// of ParsoidCachePrewarmJob.
+		$this->parsoid->parse(
+			$content->getText(),
+			$rev->getPage(),
+			$pOptions,
+			true,
+			true,
+			$rev->getId()
+		);
+	}
+}

tests/phpunit/LintUpdateTest.php

@@ -0,0 +1,196 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+namespace MediaWiki\Linter\Test;
+
+use MediaWiki\Content\JavaScriptContent;
+use MediaWiki\Linter\LintUpdate;
+use MediaWiki\MainConfigNames;
+use MediaWiki\Parser\ParserOutput;
+use MediaWiki\Parser\Parsoid\ParsoidParser;
+use MediaWiki\Revision\MutableRevisionRecord;
+use MediaWiki\Revision\RenderedRevision;
+use MediaWiki\Revision\RevisionRecord;
+use MediaWiki\Revision\SlotRecord;
+use MediaWikiIntegrationTestCase;
+use RefreshLinksJob;
+use WikiPage;
+use WikitextContent;
+
+/**
+ * @group Database
+ * @covers \MediaWiki\Linter\LintUpdate
+ */
+class LintUpdateTest extends MediaWikiIntegrationTestCase {
+
+	protected function setUp(): void {
+		parent::setUp();
+
+		$this->overrideConfigValue( MainConfigNames::ParsoidSettings, [
+			'linting' => true
+		] );
+		$this->overrideConfigValue( 'LinterParseOnDerivedDataUpdate', true );
+	}
+
+	/**
+	 * Assert that we trigger a parse, and that parse triggers onParserLogLinterData,
+	 * and in turn triggers the ParserLogLinterData hook via Parsoid.
+	 */
+	public function testUpdate() {
+		$parsoid = $this->getServiceContainer()->getParsoidParserFactory()->create();
+
+		// NOTE: This performs an edit, so do it before installing the temp hook below!
+		$rrev = $this->newRenderedRevision();
+
+		$hookCalled = false;
+		$this->setTemporaryHook( 'ParserLogLinterData', static function () use ( &$hookCalled ) {
+			$hookCalled = true;
+		}, false );
+
+		$update = $this->newLintUpdate( $parsoid, $rrev );
+		$update->doUpdate();
+
+		$this->assertTrue( $hookCalled );
+	}
+
+	/**
+	 * Assert that we don't parse if the content model is not supported.
+	 */
+	public function testSkipModel() {
+		$parsoid = $this->createNoOpMock(
+			ParsoidParser::class,
+			[ 'parse' ]
+		);
+
+		$parsoid->expects( $this->never() )->method( 'parse' );
+
+		$page = $this->getExistingTestPage();
+		$rev = new MutableRevisionRecord( $page );
+		$rev->setSlot(
+			SlotRecord::newUnsaved(
+				SlotRecord::MAIN,
+				new JavaScriptContent( '{}' )
+			)
+		);
+
+		$update = $this->newLintUpdate( $parsoid, $this->newRenderedRevision( $page, $rev ) );
+		$update->doUpdate();
+	}
+
+	/**
+	 * Assert that we don't parse if the given revision is no longer the
+	 * latest revision.
+	 */
+	public function testSkipOld() {
+		$parsoid = $this->createNoOpMock(
+			ParsoidParser::class,
+			[ 'parse' ]
+		);
+
+		$parsoid->expects( $this->never() )->method( 'parse' );
+
+		$page = $this->getExistingTestPage();
+		$rev = new MutableRevisionRecord( $page );
+		$rev->setSlot(
+			SlotRecord::newUnsaved(
+				SlotRecord::MAIN,
+				new WikitextContent( 'bla bla' )
+			)
+		);
+
+		// make it not the current revision
+		$rev->setId( $page->getLatest() - 1 );
+
+		$update = $this->newLintUpdate( $parsoid, $this->newRenderedRevision( $page, $rev ) );
+		$update->doUpdate();
+	}
+
+	/**
+	 * Assert that a LintUpdate is triggered on edit through the RevisionDataUpdates
+	 * hook, and in turn triggers the ParserLogLinterData hook via Parsoid.
+	 *
+	 * @covers \MediaWiki\Linter\Hooks::onRevisionDataUpdates
+	 */
+	public function testPageEditIntegration() {
+		$hookCalled = 0;
+		$this->setTemporaryHook( 'ParserLogLinterData', static function () use ( &$hookCalled ) {
+			$hookCalled++;
+		}, false );
+
+		$status = $this->editPage( 'JustSomePage', new WikitextContent( 'hello world' ) );
+
+		$this->assertSame( 1, $hookCalled );
+
+		$page = $this->getServiceContainer()->getWikiPageFactory()
+			->newFromTitle( $status->getNewRevision()->getPage() );
+	}
+
+	/**
+	 * Assert that a LintUpdate is triggered from RefreshLinksJob through the
+	 * RevisionDataUpdates hook, and in turn triggers the ParserLogLinterData
+	 * hook via Parsoid.
+	 *
+	 * @covers \MediaWiki\Linter\Hooks::onRevisionDataUpdates
+	 */
+	public function testRefreshLinksJobIntegration() {
+		// NOTE: This performs an edit, so do it before installing the temp hook below!
+		$page = $this->getExistingTestPage();
+
+		$hookCalled = 0;
+		$this->setTemporaryHook( 'ParserLogLinterData', static function () use ( &$hookCalled ) {
+			$hookCalled++;
+		}, false );
+
+		$job = new RefreshLinksJob( $page, [] );
+		$job->run();
+		$this->assertSame( 1, $hookCalled );
+	}
+
+	private function newRenderedRevision( WikiPage $page = null, RevisionRecord $rev = null ) {
+		$page = $this->getExistingTestPage();
+		$title = $page->getTitle();
+
+		$rev ??= $this->getServiceContainer()->getRevisionLookup()->getRevisionByTitle( $title );
+		$pOpt = $page->makeParserOptions( 'canonical' );
+
+		$rrev = new RenderedRevision(
+			$rev,
+			$pOpt,
+			$this->getServiceContainer()->getContentRenderer(),
+			static function () {
+			}
+		);
+
+		$rrev->setRevisionParserOutput( new ParserOutput( 'testing' ) );
+
+		return $rrev;
+	}
+
+	private function newLintUpdate( ParsoidParser $parsoid, RenderedRevision $renderedRevision ) {
+		$wikiPageFactory = $this->getServiceContainer()->getWikiPageFactory();
+
+		return new LintUpdate(
+			$parsoid,
+			$wikiPageFactory,
+			$renderedRevision
+		);
+	}
+
+}