mirror of
https://gerrit.wikimedia.org/r/mediawiki/extensions/DiscussionTools
synced 2024-11-24 16:34:21 +00:00
219339551c
It was useful when I was debugging those parts of the code, but now it's usually annoying. The warnings can still sometimes be useful for understanding how the tool parses some discussion, though. To keep that functionality, add displaying warnings for each comment in the debug mode. Change-Id: I2d218a8a394f179bcc0990ff988a0567c275ccf2
1100 lines
36 KiB
PHP
1100 lines
36 KiB
PHP
<?php
|
|
|
|
namespace MediaWiki\Extension\DiscussionTools;
|
|
|
|
use Config;
|
|
use DateInterval;
|
|
use DateTime;
|
|
use DateTimeImmutable;
|
|
use DateTimeZone;
|
|
use DOMElement;
|
|
use DOMNode;
|
|
use DOMText;
|
|
use DOMXPath;
|
|
use IP;
|
|
use Language;
|
|
use MediaWiki\MediaWikiServices;
|
|
use MWException;
|
|
use stdClass;
|
|
use Title;
|
|
|
|
// TODO maybe make a class for ranges?
|
|
// TODO make a class for comments
|
|
// TODO clean up static vs non-static
|
|
|
|
// TODO consider rewriting as single traversal, without XPath
|
|
// TODO consider making timestamp parsing not a returned function
|
|
|
|
class CommentParser {
|
|
private const SIGNATURE_SCAN_LIMIT = 100;
|
|
|
|
/**
|
|
* @param Language $language Content language
|
|
* @param Config $config
|
|
* @param array $data
|
|
*/
|
|
public function __construct( Language $language, Config $config, array $data = [] ) {
|
|
$this->language = $language;
|
|
$this->config = $config;
|
|
$this->dateFormat = $this->language->getDateFormatString(
|
|
'both',
|
|
$this->language->dateFormat( false )
|
|
);
|
|
// TODO: We probably shouldn't assume that each digit can be represented by a single BMP
|
|
// codepoint in every language (although it seems to be true right now).
|
|
$this->digits = $this->config->get( 'TranslateNumerals' ) ?
|
|
$this->language->formatNum( '0123456789', true ) :
|
|
null;
|
|
$this->digitsRegexp = $this->config->get( 'TranslateNumerals' ) ?
|
|
'[' . $this->language->formatNum( '0123456789', true ) . ']' :
|
|
'\\d';
|
|
// TODO: Instead of passing data used for mocking, mock the methods that fetch the data.
|
|
$this->data = $data;
|
|
$this->localTimezone = $this->config->get( 'Localtimezone' );
|
|
$this->timezoneAbbrs = $this->computeTimezoneAbbrs();
|
|
}
|
|
|
|
public static function newFromGlobalState() : CommentParser {
|
|
return new static(
|
|
MediaWikiServices::getInstance()->getContentLanguage(),
|
|
MediaWikiServices::getInstance()->getMainConfig()
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Build the timezone abbreviations map for the local timezone.
|
|
* @return array Associative array mapping localized timezone abbreviations to IANA abbreviations
|
|
*/
|
|
private function computeTimezoneAbbrs() : array {
|
|
// Return only timezone abbreviations for the local timezone (there will often be two, for
|
|
// non-DST and DST timestamps, and sometimes more due to historical data, but that's okay).
|
|
$timezoneAbbrs = array_keys( array_filter(
|
|
DateTimeZone::listAbbreviations(),
|
|
function ( array $timezones ) {
|
|
foreach ( $timezones as $tz ) {
|
|
if ( $tz['timezone_id'] === $this->localTimezone ) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
) );
|
|
return array_combine(
|
|
array_map( function ( string $tzMsg ) {
|
|
// MWTimestamp::getTimezoneMessage()
|
|
// Parser::pstPass2()
|
|
// Messages used here: 'timezone-utc' and so on
|
|
$key = 'timezone-' . strtolower( trim( $tzMsg ) );
|
|
$msg = wfMessage( $key )->inLanguage( $this->language );
|
|
// TODO: This probably causes a similar issue to https://phabricator.wikimedia.org/T221294,
|
|
// but we *must* check the message existence in the database, because the messages are not
|
|
// actually defined by MediaWiki core for any timezone other than UTC...
|
|
if ( $msg->exists() ) {
|
|
return $this->getMessages( [ $key ] )[0];
|
|
}
|
|
return strtoupper( $tzMsg );
|
|
}, $timezoneAbbrs ),
|
|
array_map( 'strtoupper', $timezoneAbbrs )
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Get a MediaWiki page title from a URL
|
|
* @param string $url
|
|
* @return Title|null
|
|
*/
|
|
private function getTitleFromUrl( string $url ) : ?Title {
|
|
// TODO: Set the correct base in the document?
|
|
if ( strpos( $url, './' ) === 0 ) {
|
|
$url = 'https://local/wiki/' . substr( $url, 2 );
|
|
} elseif ( strpos( $url, '://' ) === false ) {
|
|
$url = 'https://local' . $url;
|
|
}
|
|
$bits = wfParseUrl( $url );
|
|
$query = wfCgiToArray( $bits['query'] ?? '' );
|
|
if ( isset( $query['title'] ) ) {
|
|
return Title::newFromText( $query['title'] );
|
|
}
|
|
|
|
$articlePathRegexp = '/' . str_replace(
|
|
preg_quote( '$1', '/' ),
|
|
'(.*)',
|
|
preg_quote( $this->config->get( 'ArticlePath' ), '/' )
|
|
) . '/';
|
|
$matches = null;
|
|
if ( preg_match( $articlePathRegexp, $url, $matches ) ) {
|
|
return Title::newFromText( urldecode( $matches[1] ) );
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Return the next leaf node in the tree order that is not an empty or whitespace-only text node.
|
|
*
|
|
* In other words, this returns a text node with content other than whitespace, or an element node
|
|
* with no children, that follows the given node.
|
|
*
|
|
* @param DOMNode $node Node to start searching at. This node's children are ignored.
|
|
* @param DOMElement $rootNode Node to stop searching at
|
|
* @return DOMNode|null
|
|
*/
|
|
private function nextInterestingLeafNode( DOMNode $node, DOMElement $rootNode ) : ?DOMNode {
|
|
$n = $node;
|
|
do {
|
|
if ( $n->firstChild && ( $node === $rootNode || $n !== $node ) ) {
|
|
$n = $n->firstChild;
|
|
} elseif ( $n->nextSibling ) {
|
|
$n = $n->nextSibling;
|
|
} else {
|
|
while ( $n && $n !== $rootNode && !$n->nextSibling ) {
|
|
$n = $n->parentNode;
|
|
}
|
|
$n = $n->nextSibling;
|
|
}
|
|
|
|
if (
|
|
$n && (
|
|
(
|
|
$n->nodeType === XML_TEXT_NODE &&
|
|
CommentUtils::htmlTrim( $n->nodeValue ) !== ''
|
|
) ||
|
|
(
|
|
$n->nodeType === XML_CDATA_SECTION_NODE &&
|
|
CommentUtils::htmlTrim( $n->nodeValue ) !== ''
|
|
) ||
|
|
( $n->nodeType === XML_ELEMENT_NODE && !$n->firstChild )
|
|
)
|
|
) {
|
|
return $n;
|
|
}
|
|
} while ( $n && $n !== $rootNode );
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* @param string[] $values Values to match
|
|
* @return string Regular expression
|
|
*/
|
|
private static function regexpAlternateGroup( array $values ) : string {
|
|
return '(' . implode( '|', array_map( function ( string $x ) {
|
|
return preg_quote( $x, '/' );
|
|
}, $values ) ) . ')';
|
|
}
|
|
|
|
/**
|
|
* @param string[] $messageKeys Message keys
|
|
* @return string[] Message values
|
|
*/
|
|
private function getMessages( array $messageKeys ) : array {
|
|
return array_map( function ( string $key ) {
|
|
return isset( $this->data['contLangMessages'][$key] ) ?
|
|
$this->data['contLangMessages'][$key] :
|
|
wfMessage( $key )->inLanguage( $this->language )->text();
|
|
}, $messageKeys );
|
|
}
|
|
|
|
/**
|
|
* Get a regexp that matches timestamps generated using the given date format.
|
|
*
|
|
* This only supports format characters that are used by the default date format in any of
|
|
* MediaWiki's languages, namely: D, d, F, G, H, i, j, l, M, n, Y, xg, xkY (and escape characters),
|
|
* and only dates when MediaWiki existed, let's say 2000 onwards (Thai dates before 1941 are
|
|
* complicated).
|
|
*
|
|
* @param string $format Date format
|
|
* @param string $digitsRegexp Regular expression matching a single localized digit, e.g. '[0-9]'
|
|
* @param array $tzAbbrs Associative array mapping localized timezone abbreviations to
|
|
* IANA abbrevations, for the local timezone, e.g. [ 'EDT' => 'EDT', 'EST' => 'EST' ]
|
|
* @return string Regular expression
|
|
*/
|
|
private function getTimestampRegexp(
|
|
string $format, string $digitsRegexp, array $tzAbbrs
|
|
) : string {
|
|
$formatLength = strlen( $format );
|
|
$s = '';
|
|
// Adapted from Language::sprintfDate()
|
|
for ( $p = 0; $p < $formatLength; $p++ ) {
|
|
$num = false;
|
|
$code = $format[ $p ];
|
|
if ( $code === 'x' && $p < $formatLength - 1 ) {
|
|
$code .= $format[++$p];
|
|
}
|
|
if ( $code === 'xk' && $p < $formatLength - 1 ) {
|
|
$code .= $format[++$p];
|
|
}
|
|
|
|
switch ( $code ) {
|
|
case 'xx' :
|
|
$s .= 'x';
|
|
break;
|
|
case 'xg':
|
|
$s .= self::regexpAlternateGroup(
|
|
$this->getMessages( Language::MONTH_GENITIVE_MESSAGES )
|
|
);
|
|
break;
|
|
case 'd':
|
|
$num = '2';
|
|
break;
|
|
case 'D':
|
|
$s .= self::regexpAlternateGroup(
|
|
$this->getMessages( Language::WEEKDAY_ABBREVIATED_MESSAGES )
|
|
);
|
|
break;
|
|
case 'j':
|
|
$num = '1,2';
|
|
break;
|
|
case 'l':
|
|
$s .= self::regexpAlternateGroup(
|
|
$this->getMessages( Language::WEEKDAY_MESSAGES )
|
|
);
|
|
break;
|
|
case 'F':
|
|
$s .= self::regexpAlternateGroup(
|
|
$this->getMessages( Language::MONTH_MESSAGES )
|
|
);
|
|
break;
|
|
case 'M':
|
|
$s .= self::regexpAlternateGroup(
|
|
$this->getMessages( Language::MONTH_ABBREVIATED_MESSAGES )
|
|
);
|
|
break;
|
|
case 'n':
|
|
$num = '1,2';
|
|
break;
|
|
case 'Y':
|
|
$num = '4';
|
|
break;
|
|
case 'xkY':
|
|
$num = '4';
|
|
break;
|
|
case 'G':
|
|
$num = '1,2';
|
|
break;
|
|
case 'H':
|
|
$num = '2';
|
|
break;
|
|
case 'i':
|
|
$num = '2';
|
|
break;
|
|
case '\\':
|
|
// Backslash escaping
|
|
if ( $p < $formatLength - 1 ) {
|
|
$s .= preg_quote( $format[++$p], '/' );
|
|
} else {
|
|
$s .= preg_quote( '\\', '/' );
|
|
}
|
|
break;
|
|
case '"':
|
|
// Quoted literal
|
|
if ( $p < $formatLength - 1 ) {
|
|
$endQuote = strpos( $format, '"', $p + 1 );
|
|
if ( $endQuote === false ) {
|
|
// No terminating quote, assume literal "
|
|
$s .= '"';
|
|
} else {
|
|
$s .= preg_quote( substr( $format, $p + 1, $endQuote - $p - 1 ), '/' );
|
|
$p = $endQuote;
|
|
}
|
|
} else {
|
|
// Quote at end of string, assume literal "
|
|
$s .= '"';
|
|
}
|
|
break;
|
|
default:
|
|
$s .= preg_quote( $format[$p], '/' );
|
|
}
|
|
if ( $num !== false ) {
|
|
$s .= '(' . $digitsRegexp . '{' . $num . '})';
|
|
}
|
|
}
|
|
|
|
$tzRegexp = self::regexpAlternateGroup( array_keys( $tzAbbrs ) );
|
|
|
|
// Hardcoded parentheses and space like in Parser::pstPass2
|
|
// Ignore some invisible Unicode characters that often sneak into copypasted timestamps (T245784)
|
|
// \uNNNN syntax can only be used from PHP 7.3
|
|
return '/' . $s . '[\\x{200E}\\x{200F}]? [\\x{200E}\\x{200F}]?\\(' . $tzRegexp . '\\)/u';
|
|
}
|
|
|
|
/**
|
|
* Get a function that parses timestamps generated using the given date format, based on the result
|
|
* of matching the regexp returned by getTimestampRegexp()
|
|
*
|
|
* @param string $format Date format, as used by MediaWiki
|
|
* @param string|null $digits Localised digits from 0 to 9, e.g. `0123456789`
|
|
* @param string $localTimezone Local timezone IANA name, e.g. `America/New_York`
|
|
* @param array $tzAbbrs Map of localised timezone abbreviations to IANA abbreviations
|
|
* for the local timezone, e.g. [ 'EDT' => 'EDT', 'EST' => 'EST' ]
|
|
* @return callable Parser function
|
|
*/
|
|
private function getTimestampParser(
|
|
string $format, ?string $digits, string $localTimezone, array $tzAbbrs
|
|
) : callable {
|
|
$untransformDigits = function ( string $text ) use ( $digits ) {
|
|
if ( !$digits ) {
|
|
return $text;
|
|
}
|
|
return preg_replace_callback(
|
|
'/[' . $digits . ']/',
|
|
function ( array $m ) use ( $digits ) {
|
|
return strpos( $digits, $m[0] );
|
|
},
|
|
$text
|
|
);
|
|
};
|
|
|
|
$formatLength = strlen( $format );
|
|
$matchingGroups = [];
|
|
for ( $p = 0; $p < $formatLength; $p++ ) {
|
|
$code = $format[$p];
|
|
if ( $code === 'x' && $p < $formatLength - 1 ) {
|
|
$code .= $format[++$p];
|
|
}
|
|
if ( $code === 'xk' && $p < $formatLength - 1 ) {
|
|
$code .= $format[++$p];
|
|
}
|
|
|
|
switch ( $code ) {
|
|
case 'xx':
|
|
break;
|
|
case 'xg':
|
|
case 'd':
|
|
case 'j':
|
|
case 'D':
|
|
case 'l':
|
|
case 'F':
|
|
case 'M':
|
|
case 'n':
|
|
case 'Y':
|
|
case 'xkY':
|
|
case 'G':
|
|
case 'H':
|
|
case 'i':
|
|
$matchingGroups[] = $code;
|
|
break;
|
|
case '\\':
|
|
// Backslash escaping
|
|
if ( $p < $formatLength - 1 ) {
|
|
$p++;
|
|
}
|
|
break;
|
|
case '"':
|
|
// Quoted literal
|
|
if ( $p < $formatLength - 1 ) {
|
|
$endQuote = strpos( $format, '"', $p + 1 );
|
|
if ( $endQuote !== false ) {
|
|
$p = $endQuote;
|
|
}
|
|
}
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
return function ( array $match ) use (
|
|
$matchingGroups, $untransformDigits, $localTimezone, $tzAbbrs
|
|
) {
|
|
if ( is_array( $match[0] ) ) {
|
|
// Strip PREG_OFFSET_CAPTURE data
|
|
$match = array_map( function ( array $tuple ) {
|
|
return $tuple[0];
|
|
}, $match );
|
|
}
|
|
$year = 0;
|
|
$monthIdx = 0;
|
|
$day = 0;
|
|
$hour = 0;
|
|
$minute = 0;
|
|
foreach ( $matchingGroups as $i => $code ) {
|
|
$text = $match[$i + 1];
|
|
switch ( $code ) {
|
|
case 'xg':
|
|
$monthIdx = array_search( $text, $this->getMessages( Language::MONTH_GENITIVE_MESSAGES ) );
|
|
break;
|
|
case 'd':
|
|
case 'j':
|
|
$day = intval( $untransformDigits( $text ) );
|
|
break;
|
|
case 'D':
|
|
case 'l':
|
|
// Day of the week - unused
|
|
break;
|
|
case 'F':
|
|
$monthIdx = array_search( $text, $this->getMessages( Language::MONTH_MESSAGES ) );
|
|
break;
|
|
case 'M':
|
|
$monthIdx = array_search( $text, $this->getMessages( Language::MONTH_ABBREVIATED_MESSAGES ) );
|
|
break;
|
|
case 'n':
|
|
$monthIdx = intval( $untransformDigits( $text ) ) - 1;
|
|
break;
|
|
case 'Y':
|
|
$year = intval( $untransformDigits( $text ) );
|
|
break;
|
|
case 'xkY':
|
|
// Thai year
|
|
$year = intval( $untransformDigits( $text ) ) - 543;
|
|
break;
|
|
case 'G':
|
|
case 'H':
|
|
$hour = intval( $untransformDigits( $text ) );
|
|
break;
|
|
case 'i':
|
|
$minute = intval( $untransformDigits( $text ) );
|
|
break;
|
|
default:
|
|
// TODO throw NotImplementedException or whatever it's called
|
|
throw new MWException( 'Not implemented' );
|
|
}
|
|
}
|
|
|
|
// The last matching group is the timezone abbreviation
|
|
$tzAbbr = $tzAbbrs[ end( $match ) ];
|
|
|
|
// Most of the time, the timezone abbreviation is not necessary to parse the date, since we
|
|
// can assume all times are in the wiki's local timezone.
|
|
$date = new DateTime();
|
|
// setTimezone must be called before setDate/setTime
|
|
$date->setTimezone( new DateTimeZone( $localTimezone ) );
|
|
$date->setDate( $year, $monthIdx + 1, $day );
|
|
$date->setTime( $hour, $minute, 0 );
|
|
|
|
// But during the "fall back" at the end of DST, some times will happen twice.
|
|
// Since the timezone abbreviation disambiguates the DST/non-DST times, we can detect
|
|
// when PHP chose the wrong one, and then try the other one. It appears that PHP always
|
|
// uses the later (non-DST) hour, but that behavior isn't documented, so we account for both.
|
|
if ( $date->format( 'T' ) !== $tzAbbr ) {
|
|
$altDate = clone $date;
|
|
if ( $date->format( 'I' ) ) {
|
|
// Parsed time is DST, try non-DST by advancing one hour
|
|
$altDate->add( new DateInterval( 'PT1H' ) );
|
|
} else {
|
|
// Parsed time is non-DST, try DST by going back one hour
|
|
$altDate->sub( new DateInterval( 'PT1H' ) );
|
|
}
|
|
if ( $altDate->format( 'T' ) === $tzAbbr ) {
|
|
$date = $altDate;
|
|
$discussionToolsWarning = 'Timestamp has timezone abbreviation for the wrong time';
|
|
} else {
|
|
$discussionToolsWarning = 'Ambiguous time at DST switchover was parsed';
|
|
}
|
|
}
|
|
|
|
// Now set the timezone back to UTC for formatting
|
|
$date->setTimezone( new DateTimeZone( 'UTC' ) );
|
|
$date = DateTimeImmutable::createFromMutable( $date );
|
|
if ( isset( $discussionToolsWarning ) ) {
|
|
$date->discussionToolsWarning = $discussionToolsWarning;
|
|
}
|
|
|
|
return $date;
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Get a regular expression that matches timestamps in the local date format.
|
|
*
|
|
* This calls getTimestampRegexp() with predefined data for the current wiki.
|
|
*
|
|
* @return string Regular expression
|
|
*/
|
|
public function getLocalTimestampRegexp() : string {
|
|
return $this->getTimestampRegexp(
|
|
$this->dateFormat,
|
|
$this->digitsRegexp,
|
|
$this->timezoneAbbrs
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Get a function that parses timestamps in the local date format, based on the result
|
|
* of matching the regexp returned by getLocalTimestampRegexp().
|
|
*
|
|
* This calls getTimestampParser() with predefined data for the current wiki.
|
|
*
|
|
* @return callable Parser function
|
|
*/
|
|
private function getLocalTimestampParser() : callable {
|
|
return $this->getTimestampParser(
|
|
$this->dateFormat,
|
|
$this->digits,
|
|
$this->localTimezone,
|
|
$this->timezoneAbbrs
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Get the indent level of $node, relative to $rootNode.
|
|
*
|
|
* The indent level is the number of lists inside of which it is nested.
|
|
*
|
|
* @param DOMNode $node
|
|
* @param DOMElement $rootNode
|
|
* @return int
|
|
*/
|
|
private function getIndentLevel( DOMNode $node, DOMElement $rootNode ) : int {
|
|
$indent = 0;
|
|
while ( $node ) {
|
|
if ( $node === $rootNode ) {
|
|
break;
|
|
}
|
|
$nodeName = strtolower( $node->nodeName );
|
|
if ( $nodeName === 'li' || $nodeName === 'dd' ) {
|
|
$indent++;
|
|
}
|
|
$node = $node->parentNode;
|
|
}
|
|
return $indent;
|
|
}
|
|
|
|
/**
|
|
* Find a user signature preceding a timestamp.
|
|
*
|
|
* The signature includes the timestamp node.
|
|
*
|
|
* A signature must contain at least one link to the user's userpage, discussion page or
|
|
* contributions (and may contain other links). The link may be nested in other elements.
|
|
*
|
|
* This function returns a two-element array. The first element is an array of sibling nodes
|
|
* comprising the signature, with $timestampNode as the last element. The second element
|
|
* is the username (null for unsigned comments).
|
|
*
|
|
* @param DOMText $timestampNode Text node
|
|
* @param DOMNode|null $until Node to stop searching at
|
|
* @return array [ nodes, username ]
|
|
*/
|
|
private function findSignature( DOMText $timestampNode, ?DOMNode $until = null ) : array {
|
|
$node = $timestampNode;
|
|
$sigNodes = [ $node ];
|
|
$sigUsername = null;
|
|
$length = 0;
|
|
$lastLinkNode = $timestampNode;
|
|
while (
|
|
( $node = $node->previousSibling ) && $length < self::SIGNATURE_SCAN_LIMIT && $node !== $until
|
|
) {
|
|
$sigNodes[] = $node;
|
|
$length += $node->textContent ? strlen( $node->textContent ) : 0;
|
|
if ( $node->nodeType === XML_TEXT_NODE ) {
|
|
// FIXME use proper constant, or proper isText check
|
|
continue;
|
|
}
|
|
|
|
$links = [];
|
|
if ( strtolower( $node->nodeName ) === 'a' ) {
|
|
$links = [ $node ];
|
|
} elseif ( $node->nodeType === XML_ELEMENT_NODE ) {
|
|
// Handle links nested in formatting elements.
|
|
// Helpful accidental feature: users whose signature is not detected in full (due to
|
|
// text formatting) can just wrap it in a <span> to fix that.
|
|
// "Ten Pound Hammer • (What did I screw up now?)"
|
|
// "« Saper // dyskusja »"
|
|
$links = $node->getElementsByTagName( 'a' );
|
|
}
|
|
if ( !count( $links ) ) {
|
|
continue;
|
|
}
|
|
|
|
// Find the earliest link that links to the user's user page
|
|
foreach ( $links as $link ) {
|
|
$username = null;
|
|
$title = $this->getTitleFromUrl( $link->getAttribute( 'href' ) );
|
|
if ( !$title ) {
|
|
continue;
|
|
}
|
|
if ( $title->getNamespace() === NS_USER || $title->getNamespace() === NS_USER_TALK ) {
|
|
$username = $title->getText();
|
|
if ( strpos( $username, '/' ) !== false ) {
|
|
continue;
|
|
}
|
|
} elseif ( $title->isSpecial( 'Contributions' ) ) {
|
|
$parts = explode( '/', $title->getText(), 2 );
|
|
if ( !isset( $parts[1] ) ) {
|
|
continue;
|
|
}
|
|
// Normalize the username: users may link to their contributions with an unnormalized name
|
|
$userpage = Title::makeTitleSafe( NS_USER, $parts[1] );
|
|
if ( !$userpage ) {
|
|
continue;
|
|
}
|
|
$username = $userpage->getText();
|
|
}
|
|
if ( !$username ) {
|
|
continue;
|
|
}
|
|
if ( IP::isIPv6( $username ) ) {
|
|
// Bot-generated links "Preceding unsigned comment added by" have non-standard case
|
|
$username = strtoupper( $username );
|
|
}
|
|
|
|
// Accept the first link to the user namespace, then only accept links to that user
|
|
if ( $sigUsername === null ) {
|
|
$sigUsername = $username;
|
|
}
|
|
if ( $username === $sigUsername ) {
|
|
$lastLinkNode = $node;
|
|
break;
|
|
}
|
|
}
|
|
// Keep looking if a node with links wasn't a link to a user page
|
|
// "Doc James (talk · contribs · email)"
|
|
}
|
|
// Pop excess text nodes
|
|
while ( end( $sigNodes ) !== $lastLinkNode ) {
|
|
array_pop( $sigNodes );
|
|
}
|
|
return [ $sigNodes, $sigUsername ];
|
|
}
|
|
|
|
/**
|
|
* Find all timestamps within a DOM subtree.
|
|
*
|
|
* @param DOMElement $rootNode
|
|
* @return array Array of [node, matchData] pairs
|
|
*/
|
|
public function findTimestamps( DOMElement $rootNode ) : array {
|
|
$xpath = new DOMXPath( $rootNode->ownerDocument );
|
|
$textNodes = $xpath->query( '//text()', $rootNode );
|
|
$matches = [];
|
|
$timestampRegex = $this->getLocalTimestampRegexp();
|
|
foreach ( $textNodes as $node ) {
|
|
$startNode = $node;
|
|
$nodeText = '';
|
|
|
|
while ( $node ) {
|
|
$nodeText .= $node->nodeValue;
|
|
|
|
// In Parsoid HTML, entities are represented as a 'mw:Entity' node, rather than normal HTML
|
|
// entities. On Arabic Wikipedia, the "UTC" timezone name contains some non-breaking spaces,
|
|
// which apparently are often turned into entities by buggy editing tools. To handle
|
|
// this, we must piece together the text, so that our regexp can match those timestamps.
|
|
if (
|
|
$node->nextSibling &&
|
|
$node->nextSibling->nodeType === XML_ELEMENT_NODE &&
|
|
$node->nextSibling->getAttribute( 'typeof' ) === 'mw:Entity'
|
|
) {
|
|
$nodeText .= $node->nextSibling->firstChild->nodeValue;
|
|
|
|
// If the entity is followed by more text, do this again
|
|
if (
|
|
$node->nextSibling->nextSibling &&
|
|
$node->nextSibling->nextSibling->nodeType === XML_TEXT_NODE
|
|
) {
|
|
$node = $node->nextSibling->nextSibling;
|
|
} else {
|
|
$node = null;
|
|
}
|
|
} else {
|
|
$node = null;
|
|
}
|
|
}
|
|
|
|
$matchData = null;
|
|
// Allows us to mimic match.index in #getComments
|
|
if ( preg_match( $timestampRegex, $nodeText, $matchData, PREG_OFFSET_CAPTURE ) ) {
|
|
$matches[] = [ $startNode, $matchData ];
|
|
}
|
|
}
|
|
return $matches;
|
|
}
|
|
|
|
/**
|
|
* Get all discussion comments (and headings) within a DOM subtree.
|
|
*
|
|
* This returns a flat list, use groupThreads() to associate replies to original messages and
|
|
* get a tree structure starting at section headings.
|
|
*
|
|
* For example, for a MediaWiki discussion like this (we're dealing with HTML DOM here,
|
|
* the wikitext syntax is just for illustration):
|
|
*
|
|
* == A ==
|
|
* B. ~~~~
|
|
* : C.
|
|
* : C. ~~~~
|
|
* :: D. ~~~~
|
|
* ::: E. ~~~~
|
|
* ::: F. ~~~~
|
|
* : G. ~~~~
|
|
* H. ~~~~
|
|
* : I. ~~~~
|
|
*
|
|
* This function would return a structure like:
|
|
*
|
|
* [
|
|
* [ 'type' => 'heading', 'level' => 0, 'range' => (h2: A) },
|
|
* [ 'type' => 'comment', 'level' => 1, 'range' => (p: B) },
|
|
* [ 'type' => 'comment', 'level' => 2, 'range' => (li: C, li: C) },
|
|
* [ 'type' => 'comment', 'level' => 3, 'range' => (li: D) },
|
|
* [ 'type' => 'comment', 'level' => 4, 'range' => (li: E) },
|
|
* [ 'type' => 'comment', 'level' => 4, 'range' => (li: F) },
|
|
* [ 'type' => 'comment', 'level' => 2, 'range' => (li: G) },
|
|
* [ 'type' => 'comment', 'level' => 1, 'range' => (p: H) },
|
|
* [ 'type' => 'comment', 'level' => 2, 'range' => (li: I) }
|
|
* ]
|
|
*
|
|
* The elements of the array are stdClass objects with the following fields:
|
|
* - 'type' (string): 'heading' or 'comment'
|
|
* - 'range' (array): The extent of the comment, including the signature and timestamp.
|
|
* Comments can start or end in the middle of a DOM node.
|
|
* Keys: 'startContainer', 'startOffset', 'endContainer' and 'endOffset'
|
|
* - 'signatureRanges' (array): The extents of the comment's signatures (plus timestamps).
|
|
* There is always at least one signature, but there may be multiple.
|
|
* The author and timestamp of the comment is determined from the
|
|
* first signature. The last node in every signature range is the
|
|
* text node containing the timestamp.
|
|
* - 'level' (int): Indentation level of the comment. Headings are 0, comments start at 1.
|
|
* - 'timestamp' (string): ISO 8601 timestamp in UTC (ending in 'Z'). Not set for headings.
|
|
* - 'author' (string|null): Comment author's username, null for unsigned comments.
|
|
* Not set for headings.
|
|
*
|
|
* @param DOMElement $rootNode
|
|
* @return stdClass[] Results. Each result is an object.
|
|
*/
|
|
public function getComments( DOMElement $rootNode ) : array {
|
|
$timestamps = $this->findTimestamps( $rootNode );
|
|
|
|
$xpath = new DOMXPath( $rootNode->ownerDocument );
|
|
$allNodes = $xpath->query( '//text()|//node()', $rootNode );
|
|
$tocNode = $rootNode->ownerDocument->getElementById( 'toc' );
|
|
$comments = [];
|
|
$dfParser = $this->getLocalTimestampParser();
|
|
|
|
// Placeholder heading in case there are comments in the 0th section
|
|
$range = (object)[
|
|
'startContainer' => $rootNode,
|
|
'startOffset' => 0,
|
|
'endContainer' => $rootNode,
|
|
'endOffset' => 0
|
|
];
|
|
$fakeHeading = (object)[
|
|
'placeholderHeading' => true,
|
|
'type' => 'heading',
|
|
'range' => $range,
|
|
'level' => 0
|
|
];
|
|
|
|
$curComment = $fakeHeading;
|
|
|
|
$nextTimestamp = 0;
|
|
foreach ( $allNodes as $node ) {
|
|
// Skip nodes inside <div id="toc">
|
|
if ( $tocNode && CommentUtils::contains( $tocNode, $node ) ) {
|
|
continue;
|
|
}
|
|
|
|
if ( $node->nodeType === XML_ELEMENT_NODE && preg_match( '/^h[1-6]$/i', $node->nodeName ) ) {
|
|
$range = (object)[
|
|
'startContainer' => $node,
|
|
'startOffset' => 0,
|
|
'endContainer' => $node,
|
|
'endOffset' => $node->childNodes->length
|
|
];
|
|
$curComment = (object)[
|
|
'type' => 'heading',
|
|
'range' => $range,
|
|
'level' => 0
|
|
];
|
|
$comments[] = $curComment;
|
|
} elseif ( isset( $timestamps[$nextTimestamp] ) && $node === $timestamps[$nextTimestamp][0] ) {
|
|
$warnings = [];
|
|
$foundSignature = $this->findSignature( $node, $curComment->range->endContainer );
|
|
$author = $foundSignature[1];
|
|
$firstSigNode = end( $foundSignature[0] );
|
|
|
|
if ( !$author ) {
|
|
// Ignore timestamps for which we couldn't find a signature. It's probably not a real
|
|
// comment, but just a false match due to a copypasted timestamp.
|
|
$nextTimestamp++;
|
|
continue;
|
|
}
|
|
|
|
// Everything from the last comment up to here is the next comment
|
|
$startNode = $this->nextInterestingLeafNode( $curComment->range->endContainer, $rootNode );
|
|
$match = $timestamps[$nextTimestamp][1];
|
|
$range = (object)[
|
|
'startContainer' => $startNode->parentNode,
|
|
'startOffset' => CommentUtils::childIndexOf( $startNode ),
|
|
'endContainer' => $node,
|
|
'endOffset' => $match[0][1] + strlen( $match[0][0] )
|
|
];
|
|
$sigRange = (object)[
|
|
'startContainer' => $firstSigNode->parentNode,
|
|
'startOffset' => CommentUtils::childIndexOf( $firstSigNode ),
|
|
'endContainer' => $node,
|
|
'endOffset' => $match[0][1] + strlen( $match[0][0] )
|
|
];
|
|
|
|
$startLevel = $this->getIndentLevel( $startNode, $rootNode ) + 1;
|
|
$endLevel = $this->getIndentLevel( $node, $rootNode ) + 1;
|
|
if ( $startLevel !== $endLevel ) {
|
|
$warnings[] = 'Comment starts and ends with different indentation';
|
|
}
|
|
|
|
// Avoid generating multiple comments when there is more than one signature on a single "line".
|
|
// Often this is done when someone edits their comment later and wants to add a note about that.
|
|
// (Or when another person corrects a typo, or strikes out a comment, etc.) Multiple comments
|
|
// within one paragraph/listitem result in a confusing double "Reply" button, and we also have
|
|
// no way to indicate which one you're replying to (this might matter in the future for
|
|
// notifications or something).
|
|
if (
|
|
$curComment->type === 'comment' &&
|
|
(
|
|
CommentUtils::closestElement(
|
|
$node, [ 'li', 'dd', 'p' ]
|
|
) ?? $node->parentNode
|
|
) ===
|
|
(
|
|
CommentUtils::closestElement(
|
|
$curComment->range->endContainer, [ 'li', 'dd', 'p' ]
|
|
) ?? $curComment->range->endContainer->parentNode
|
|
)
|
|
) {
|
|
// Merge this with the previous comment. Use that comment's author and timestamp.
|
|
$curComment->range->endContainer = $range->endContainer;
|
|
$curComment->range->endOffset = $range->endOffset;
|
|
$curComment->signatureRanges[] = $sigRange;
|
|
$curComment->level = min( min( $startLevel, $endLevel ), $curComment->level );
|
|
|
|
$nextTimestamp++;
|
|
continue;
|
|
}
|
|
|
|
$dateTime = $dfParser( $match );
|
|
if ( isset( $dateTime->discussionToolsWarning ) ) {
|
|
$warnings[] = $dateTime->discussionToolsWarning;
|
|
}
|
|
|
|
$curComment = (object)[
|
|
'type' => 'comment',
|
|
// ISO 8601 date. Almost DateTimeInterface::RFC3339_EXTENDED, but ending with 'Z' instead
|
|
// of '+00:00', like Date#toISOString in JavaScript.
|
|
'timestamp' => $dateTime->format( 'Y-m-d\TH:i:s.v\Z' ),
|
|
'author' => $author,
|
|
'range' => $range,
|
|
'signatureRanges' => [ $sigRange ],
|
|
// Should this use the indent level of $startNode or $node?
|
|
'level' => min( $startLevel, $endLevel )
|
|
];
|
|
if ( $warnings ) {
|
|
$curComment->warnings = $warnings;
|
|
}
|
|
$comments[] = $curComment;
|
|
$nextTimestamp++;
|
|
}
|
|
}
|
|
|
|
// Insert the fake placeholder heading if there are any comments in the 0th section
|
|
// (before the first real heading)
|
|
if ( count( $comments ) && $comments[ 0 ]->type !== 'heading' ) {
|
|
array_unshift( $comments, $fakeHeading );
|
|
}
|
|
|
|
return $comments;
|
|
}
|
|
|
|
/**
|
|
* Group discussion comments into threads and associate replies to original messages.
|
|
*
|
|
* Each thread must begin with a heading. Original messages in the thread are treated as replies to
|
|
* its heading. Other replies are associated based on the order and indentation level.
|
|
*
|
|
* Note that the objects in `comments` are extended in-place with the additional data.
|
|
*
|
|
* For example, for a MediaWiki discussion like this (we're dealing with HTML DOM here,
|
|
* the wikitext syntax is just for illustration):
|
|
*
|
|
* == A ==
|
|
* B. ~~~~
|
|
* : C.
|
|
* : C. ~~~~
|
|
* :: D. ~~~~
|
|
* ::: E. ~~~~
|
|
* ::: F. ~~~~
|
|
* : G. ~~~~
|
|
* H. ~~~~
|
|
* : I. ~~~~
|
|
*
|
|
* This function would return a structure like:
|
|
*
|
|
* [
|
|
* [ 'type' => 'heading', 'level' => 0, 'range' => (h2: A), 'replies' => [
|
|
* [ 'type' => 'comment', 'level' => 1, 'range' => (p: B), 'replies' => [
|
|
* [ 'type' => 'comment', 'level' => 2, 'range' => (li: C, li: C), 'replies' => [
|
|
* [ 'type' => 'comment', 'level' => 3, 'range' => (li: D), 'replies' => [
|
|
* [ 'type' => 'comment', 'level' => 4, 'range' => (li: E), 'replies' => [] ],
|
|
* [ 'type' => 'comment', 'level' => 4, 'range' => (li: F), 'replies': [] ],
|
|
* ] ],
|
|
* ] ],
|
|
* [ 'type' => 'comment', 'level' => 2, 'range' => (li: G), 'replies' => [] ],
|
|
* ] ],
|
|
* [ 'type' => 'comment', 'level' => 1, 'range' => (p: H), 'replies' => [
|
|
* [ 'type' => 'comment', 'level' => 2, 'range' => (li: I), 'replies' => [] ],
|
|
* ] ],
|
|
* ] ],
|
|
* ]
|
|
*
|
|
* @param stdClass[] &$comments Result of #getComments, will be modified to add more properties
|
|
* @return stdClass[] Tree structure of comments, using the same objects as `comments`. Top-level
|
|
* items are the headings. The following properties are added:
|
|
* - id: Unique ID (within the page) for this comment, intended to be used to
|
|
* find this comment in other revisions of the same page
|
|
* - replies: Comment objects which are replies to this comment
|
|
* - parent: Comment object which this is a reply to (null for headings)
|
|
*/
|
|
public function groupThreads( array &$comments ) : array {
|
|
$threads = [];
|
|
$replies = [];
|
|
$commentsById = [];
|
|
|
|
foreach ( $comments as &$comment ) {
|
|
if ( $comment->level === 0 ) {
|
|
// We don't need ids for section headings right now, but we might in the future
|
|
// e.g. if we allow replying directly to sections (adding top-level comments)
|
|
$id = null;
|
|
} else {
|
|
$id = ( $comment->author ?? '' ) . '|' . $comment->timestamp;
|
|
|
|
// If there would be multiple comments with the same ID (i.e. the user left multiple comments
|
|
// in one edit, or within a minute), append sequential numbers
|
|
$number = 0;
|
|
while ( isset( $commentsById["$id|$number"] ) ) {
|
|
$number++;
|
|
}
|
|
$id = "$id|$number";
|
|
}
|
|
|
|
if ( $id !== null ) {
|
|
$commentsById[$id] = $comment;
|
|
}
|
|
|
|
// This modifies the original objects in $comments!
|
|
$comment->id = $id;
|
|
$comment->replies = [];
|
|
$comment->parent = null;
|
|
|
|
if ( count( $replies ) < $comment->level ) {
|
|
// Someone skipped an indentation level (or several). Pretend that the previous reply
|
|
// covers multiple indentation levels, so that following comments get connected to it.
|
|
$comment->warnings[] = 'Comment skips indentation level';
|
|
while ( count( $replies ) < $comment->level ) {
|
|
// FIXME this will clone the reply, not just set a reference
|
|
$replies[] = end( $replies );
|
|
}
|
|
}
|
|
|
|
if ( $comment->level === 0 ) {
|
|
// New root (thread)
|
|
$threads[] = $comment;
|
|
} elseif ( isset( $replies[ $comment->level - 1 ] ) ) {
|
|
// Add as a reply to the closest less-nested comment
|
|
$comment->parent = $replies[ $comment->level - 1 ];
|
|
$comment->parent->replies[] = $comment;
|
|
} else {
|
|
$comment->warnings[] = 'Comment could not be connected to a thread';
|
|
}
|
|
|
|
$replies[ $comment->level ] = $comment;
|
|
// Cut off more deeply nested replies
|
|
array_splice( $replies, $comment->level + 1 );
|
|
}
|
|
|
|
return $threads;
|
|
}
|
|
|
|
/**
|
|
* Get the list of authors involved in a comment and its replies.
|
|
*
|
|
* You probably want to pass a thread root here (a heading).
|
|
*
|
|
* @param stdClass $comment Comment object, as returned by #groupThreads
|
|
* @return string[] Author usernames
|
|
*/
|
|
public function getAuthors( stdClass $comment ) : array {
|
|
$authors = [];
|
|
$getAuthorSet = function ( stdClass $comment ) use ( &$authors, &$getAuthorSet ) {
|
|
if ( $comment->author ?? false ) {
|
|
$authors[ $comment->author ] = true;
|
|
}
|
|
// Get the set of authors in the same format from each reply
|
|
array_map( $getAuthorSet, $comment->replies );
|
|
};
|
|
|
|
$getAuthorSet( $comment );
|
|
|
|
ksort( $authors );
|
|
return array_keys( $authors );
|
|
}
|
|
|
|
/**
|
|
* Get the name of the page from which this comment is transcluded (if any).
|
|
*
|
|
* @param stdClass $comment Comment object, as returned by #groupThreads
|
|
* @return string|bool `false` if this comment is not transcluded. A string if it's transcluded
|
|
* from a single page (the page title, in text form with spaces). `true` if it's transcluded, but
|
|
* we can't determine the source.
|
|
*/
|
|
public function getTranscludedFrom( stdClass $comment ) {
|
|
// If some template is used within the comment (e.g. {{ping|…}} or {{tl|…}}), that *does not* mean
|
|
// the comment is transcluded. We only want to consider comments to be transcluded if the wrapper
|
|
// element (usually <li> or <p>) is marked as part of a transclusion.
|
|
// TODO: This seems to work fine but I'm having a hard time explaining why it is correct...
|
|
$node = $comment->range->endContainer;
|
|
|
|
// Find the node containing information about the transclusion:
|
|
// 1. Find the closest ancestor with an 'about' attribute
|
|
// 2. Find the main node of the about-group (first sibling with the same 'about' attribute)
|
|
// 3. If this is an mw:Transclusion node, return it; otherwise, go to step 1
|
|
while ( $node ) {
|
|
// 1.
|
|
if (
|
|
$node->nodeType === XML_ELEMENT_NODE &&
|
|
$node->getAttribute( 'about' ) &&
|
|
preg_match( '/^#mwt\d+$/', $node->getAttribute( 'about' ) )
|
|
) {
|
|
$about = $node->getAttribute( 'about' );
|
|
|
|
// 2.
|
|
while (
|
|
$node->previousSibling &&
|
|
$node->previousSibling->nodeType === XML_ELEMENT_NODE &&
|
|
$node->previousSibling->getAttribute( 'about' ) === $about
|
|
) {
|
|
$node = $node->previousSibling;
|
|
}
|
|
|
|
// 3.
|
|
if (
|
|
$node->getAttribute( 'typeof' ) &&
|
|
in_array( 'mw:Transclusion', explode( ' ', $node->getAttribute( 'typeof' ) ) )
|
|
) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
$node = $node->parentNode;
|
|
}
|
|
|
|
if ( !$node ) {
|
|
// No mw:Transclusion node found, this comment is not transcluded
|
|
return false;
|
|
}
|
|
|
|
$dataMw = json_decode( $node->getAttribute( 'data-mw' ), true );
|
|
|
|
// Only return a page name if this is a simple single-template transclusion.
|
|
if (
|
|
$dataMw['parts'] &&
|
|
count( $dataMw['parts'] ) === 1 &&
|
|
$dataMw['parts'][0]['template'] &&
|
|
$dataMw['parts'][0]['template']['target']['href']
|
|
) {
|
|
$title = self::getTitleFromUrl( $dataMw['parts'][0]['template']['target']['href'] );
|
|
return $title->getPrefixedText();
|
|
}
|
|
|
|
// Multi-template transclusion, or a parser function call, or template-affected wikitext outside
|
|
// of a template call, or a mix of the above
|
|
return true;
|
|
}
|
|
}
|