mirror of
https://gerrit.wikimedia.org/r/mediawiki/extensions/DiscussionTools
synced 2024-11-16 20:58:28 +00:00
0fc71f60cd
We've recently decided that we want to "extend" comments until the end of the paragraph (e36dc8e78a,d0ae6c4e44). However, we still had this special case that did the opposite: it ensured that if a comment ended in the middle of a text node, the comment would not be extended to the end of the node. Remove it. Note the change in the test file signatures-funny-formattedreply.html, which actually covered this case specifically. Change-Id: Id1384bb0c6e1a5f0c70f55efcb4caa240f230f07
1152 lines
37 KiB
PHP
1152 lines
37 KiB
PHP
<?php
|
|
|
|
namespace MediaWiki\Extension\DiscussionTools;
|
|
|
|
use Config;
|
|
use DateInterval;
|
|
use DateTime;
|
|
use DateTimeImmutable;
|
|
use DateTimeZone;
|
|
use DOMElement;
|
|
use DOMNode;
|
|
use DOMText;
|
|
use IP;
|
|
use Language;
|
|
use MediaWiki\MediaWikiServices;
|
|
use MWException;
|
|
use Title;
|
|
|
|
// TODO clean up static vs non-static
|
|
// TODO consider making timestamp parsing not a returned function
|
|
|
|
class CommentParser {
|
|
private const SIGNATURE_SCAN_LIMIT = 100;
|
|
|
|
/** @var DOMElement */
|
|
private $rootNode;
|
|
|
|
/** @var ThreadItem[] */
|
|
private $threadItems;
|
|
/** @var CommentItem[] */
|
|
private $commentItems;
|
|
/** @var ThreadItem[] */
|
|
private $threadItemsById;
|
|
/** @var HeadingItem[] */
|
|
private $threads;
|
|
|
|
/** @var Config */
|
|
private $config;
|
|
|
|
/** @var Language */
|
|
private $language;
|
|
|
|
private $dateFormat;
|
|
private $digits;
|
|
/** @var string[][] */
|
|
private $contLangMessages;
|
|
private $localTimezone;
|
|
private $timezones;
|
|
|
|
/**
|
|
* @param DomElement $rootNode Root node of content to parse
|
|
* @param Language $language Content language
|
|
* @param Config $config
|
|
* @param array $data
|
|
*/
|
|
public function __construct( DOMElement $rootNode, Language $language, Config $config, array $data = [] ) {
|
|
$this->rootNode = $rootNode;
|
|
$this->config = $config;
|
|
$this->language = $language;
|
|
|
|
if ( !$data ) {
|
|
// TODO: Instead of passing data used for mocking, mock the methods that fetch the data.
|
|
$data = Data::getLocalData( null, $config, $language );
|
|
}
|
|
|
|
$this->dateFormat = $data['dateFormat'];
|
|
$this->digits = $data['digits'];
|
|
$this->contLangMessages = $data['contLangMessages'];
|
|
$this->localTimezone = $data['localTimezone'];
|
|
$this->timezones = $data['timezones'];
|
|
}
|
|
|
|
/**
|
|
* @param DomElement $rootNode
|
|
* @return CommentParser
|
|
*/
|
|
public static function newFromGlobalState( DOMElement $rootNode ) : CommentParser {
|
|
return new static(
|
|
$rootNode,
|
|
MediaWikiServices::getInstance()->getContentLanguage(),
|
|
MediaWikiServices::getInstance()->getMainConfig()
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Return the next leaf node in the tree order that is likely a part of a discussion comment,
|
|
* rather than some boring "separator" element.
|
|
*
|
|
* Currently, this can return a Text node with content other than whitespace, or an `<img>` node.
|
|
*
|
|
* @param DOMNode $node Node to start searching at. This node's children are ignored.
|
|
* @return DOMNode
|
|
*/
|
|
private function nextInterestingLeafNode( DOMNode $node ) : DOMNode {
|
|
$rootNode = $this->rootNode;
|
|
$treeWalker = new TreeWalker(
|
|
$rootNode,
|
|
NodeFilter::SHOW_ELEMENT | NodeFilter::SHOW_TEXT,
|
|
function ( $n ) use ( $node, $rootNode ) {
|
|
// Ignore this node and its descendants
|
|
// (unless it's the root node, this is a special case for "fakeHeading" handling)
|
|
if ( $node !== $rootNode && ( $n === $node || $n->parentNode === $node ) ) {
|
|
return NodeFilter::FILTER_REJECT;
|
|
}
|
|
if (
|
|
(
|
|
$n->nodeType === XML_TEXT_NODE &&
|
|
CommentUtils::htmlTrim( $n->nodeValue ) !== ''
|
|
) ||
|
|
(
|
|
$n->nodeType === XML_CDATA_SECTION_NODE &&
|
|
CommentUtils::htmlTrim( $n->nodeValue ) !== ''
|
|
) ||
|
|
(
|
|
$n->nodeType === XML_ELEMENT_NODE &&
|
|
strtolower( $n->nodeName ) === 'img'
|
|
)
|
|
) {
|
|
return NodeFilter::FILTER_ACCEPT;
|
|
}
|
|
return NodeFilter::FILTER_SKIP;
|
|
}
|
|
);
|
|
$treeWalker->currentNode = $node;
|
|
$treeWalker->nextNode();
|
|
if ( !$treeWalker->currentNode ) {
|
|
throw new MWException( 'nextInterestingLeafNode not found' );
|
|
}
|
|
return $treeWalker->currentNode;
|
|
}
|
|
|
|
/**
|
|
* @param string[] $values Values to match
|
|
* @return string Regular expression
|
|
*/
|
|
private static function regexpAlternateGroup( array $values ) : string {
|
|
return '(' . implode( '|', array_map( function ( string $x ) {
|
|
return preg_quote( $x, '/' );
|
|
}, $values ) ) . ')';
|
|
}
|
|
|
|
/**
|
|
* Get text of localisation messages in content language.
|
|
*
|
|
* @param string $contLangVariant Content language variant
|
|
* @param string[] $messages Message keys
|
|
* @return string[] Message values
|
|
*/
|
|
private function getMessages( string $contLangVariant, array $messages ) : array {
|
|
return array_map( function ( string $key ) use ( $contLangVariant ) {
|
|
return $this->contLangMessages[$contLangVariant][$key];
|
|
}, $messages );
|
|
}
|
|
|
|
/**
|
|
* Get a regexp that matches timestamps generated using the given date format.
|
|
*
|
|
* This only supports format characters that are used by the default date format in any of
|
|
* MediaWiki's languages, namely: D, d, F, G, H, i, j, l, M, n, Y, xg, xkY (and escape characters),
|
|
* and only dates when MediaWiki existed, let's say 2000 onwards (Thai dates before 1941 are
|
|
* complicated).
|
|
*
|
|
* @param string $contLangVariant Content language variant
|
|
* @param string $format Date format
|
|
* @param string $digitsRegexp Regular expression matching a single localised digit, e.g. '[0-9]'
|
|
* @param array $tzAbbrs Associative array mapping localised timezone abbreviations to
|
|
* IANA abbreviations, for the local timezone, e.g. [ 'EDT' => 'EDT', 'EST' => 'EST' ]
|
|
* @return string Regular expression
|
|
*/
|
|
private function getTimestampRegexp(
|
|
string $contLangVariant, string $format, string $digitsRegexp, array $tzAbbrs
|
|
) : string {
|
|
$formatLength = strlen( $format );
|
|
$s = '';
|
|
// Adapted from Language::sprintfDate()
|
|
for ( $p = 0; $p < $formatLength; $p++ ) {
|
|
$num = false;
|
|
$code = $format[ $p ];
|
|
if ( $code === 'x' && $p < $formatLength - 1 ) {
|
|
$code .= $format[++$p];
|
|
}
|
|
if ( $code === 'xk' && $p < $formatLength - 1 ) {
|
|
$code .= $format[++$p];
|
|
}
|
|
|
|
switch ( $code ) {
|
|
case 'xx':
|
|
$s .= 'x';
|
|
break;
|
|
case 'xg':
|
|
$s .= self::regexpAlternateGroup(
|
|
$this->getMessages( $contLangVariant, Language::MONTH_GENITIVE_MESSAGES )
|
|
);
|
|
break;
|
|
case 'd':
|
|
$num = '2';
|
|
break;
|
|
case 'D':
|
|
$s .= self::regexpAlternateGroup(
|
|
$this->getMessages( $contLangVariant, Language::WEEKDAY_ABBREVIATED_MESSAGES )
|
|
);
|
|
break;
|
|
case 'j':
|
|
$num = '1,2';
|
|
break;
|
|
case 'l':
|
|
$s .= self::regexpAlternateGroup(
|
|
$this->getMessages( $contLangVariant, Language::WEEKDAY_MESSAGES )
|
|
);
|
|
break;
|
|
case 'F':
|
|
$s .= self::regexpAlternateGroup(
|
|
$this->getMessages( $contLangVariant, Language::MONTH_MESSAGES )
|
|
);
|
|
break;
|
|
case 'M':
|
|
$s .= self::regexpAlternateGroup(
|
|
$this->getMessages( $contLangVariant, Language::MONTH_ABBREVIATED_MESSAGES )
|
|
);
|
|
break;
|
|
case 'n':
|
|
$num = '1,2';
|
|
break;
|
|
case 'Y':
|
|
$num = '4';
|
|
break;
|
|
case 'xkY':
|
|
$num = '4';
|
|
break;
|
|
case 'G':
|
|
$num = '1,2';
|
|
break;
|
|
case 'H':
|
|
$num = '2';
|
|
break;
|
|
case 'i':
|
|
$num = '2';
|
|
break;
|
|
case '\\':
|
|
// Backslash escaping
|
|
if ( $p < $formatLength - 1 ) {
|
|
$s .= preg_quote( $format[++$p], '/' );
|
|
} else {
|
|
$s .= preg_quote( '\\', '/' );
|
|
}
|
|
break;
|
|
case '"':
|
|
// Quoted literal
|
|
if ( $p < $formatLength - 1 ) {
|
|
$endQuote = strpos( $format, '"', $p + 1 );
|
|
if ( $endQuote === false ) {
|
|
// No terminating quote, assume literal "
|
|
$s .= '"';
|
|
} else {
|
|
$s .= preg_quote( substr( $format, $p + 1, $endQuote - $p - 1 ), '/' );
|
|
$p = $endQuote;
|
|
}
|
|
} else {
|
|
// Quote at end of string, assume literal "
|
|
$s .= '"';
|
|
}
|
|
break;
|
|
default:
|
|
$s .= preg_quote( $format[$p], '/' );
|
|
}
|
|
if ( $num !== false ) {
|
|
$s .= '(' . $digitsRegexp . '{' . $num . '})';
|
|
}
|
|
}
|
|
|
|
$tzRegexp = self::regexpAlternateGroup( array_keys( $tzAbbrs ) );
|
|
|
|
// Hard-coded parentheses and space like in Parser::pstPass2
|
|
// Ignore some invisible Unicode characters that often sneak into copy-pasted timestamps (T245784)
|
|
// \uNNNN syntax can only be used from PHP 7.3
|
|
return '/' . $s . '[\\x{200E}\\x{200F}]? [\\x{200E}\\x{200F}]?\\(' . $tzRegexp . '\\)/u';
|
|
}
|
|
|
|
/**
|
|
* Get a function that parses timestamps generated using the given date format, based on the result
|
|
* of matching the regexp returned by getTimestampRegexp()
|
|
*
|
|
* @param string $contLangVariant Content language variant
|
|
* @param string $format Date format, as used by MediaWiki
|
|
* @param string[]|null $digits Localised digits from 0 to 9, e.g. `[ '0', '1', ..., '9' ]`
|
|
* @param string $localTimezone Local timezone IANA name, e.g. `America/New_York`
|
|
* @param array $tzAbbrs Map of localised timezone abbreviations to IANA abbreviations
|
|
* for the local timezone, e.g. [ 'EDT' => 'EDT', 'EST' => 'EST' ]
|
|
* @return callable Parser function
|
|
*/
|
|
private function getTimestampParser(
|
|
string $contLangVariant, string $format, ?array $digits, string $localTimezone, array $tzAbbrs
|
|
) : callable {
|
|
$untransformDigits = function ( string $text ) use ( $digits ) {
|
|
if ( !$digits ) {
|
|
return $text;
|
|
}
|
|
return preg_replace_callback(
|
|
'/[' . implode( '', $digits ) . ']/u',
|
|
function ( array $m ) use ( $digits ) {
|
|
return (string)array_search( $m[0], $digits );
|
|
},
|
|
$text
|
|
);
|
|
};
|
|
|
|
$formatLength = strlen( $format );
|
|
$matchingGroups = [];
|
|
for ( $p = 0; $p < $formatLength; $p++ ) {
|
|
$code = $format[$p];
|
|
if ( $code === 'x' && $p < $formatLength - 1 ) {
|
|
$code .= $format[++$p];
|
|
}
|
|
if ( $code === 'xk' && $p < $formatLength - 1 ) {
|
|
$code .= $format[++$p];
|
|
}
|
|
|
|
switch ( $code ) {
|
|
case 'xx':
|
|
break;
|
|
case 'xg':
|
|
case 'd':
|
|
case 'j':
|
|
case 'D':
|
|
case 'l':
|
|
case 'F':
|
|
case 'M':
|
|
case 'n':
|
|
case 'Y':
|
|
case 'xkY':
|
|
case 'G':
|
|
case 'H':
|
|
case 'i':
|
|
$matchingGroups[] = $code;
|
|
break;
|
|
case '\\':
|
|
// Backslash escaping
|
|
if ( $p < $formatLength - 1 ) {
|
|
$p++;
|
|
}
|
|
break;
|
|
case '"':
|
|
// Quoted literal
|
|
if ( $p < $formatLength - 1 ) {
|
|
$endQuote = strpos( $format, '"', $p + 1 );
|
|
if ( $endQuote !== false ) {
|
|
$p = $endQuote;
|
|
}
|
|
}
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
return function ( array $match ) use (
|
|
$matchingGroups, $untransformDigits, $localTimezone, $tzAbbrs, $contLangVariant
|
|
) {
|
|
if ( is_array( $match[0] ) ) {
|
|
// Strip PREG_OFFSET_CAPTURE data
|
|
unset( $match['offset'] );
|
|
$match = array_map( function ( array $tuple ) {
|
|
return $tuple[0];
|
|
}, $match );
|
|
}
|
|
$year = 0;
|
|
$monthIdx = 0;
|
|
$day = 0;
|
|
$hour = 0;
|
|
$minute = 0;
|
|
foreach ( $matchingGroups as $i => $code ) {
|
|
$text = $match[$i + 1];
|
|
switch ( $code ) {
|
|
case 'xg':
|
|
$monthIdx = array_search(
|
|
$text,
|
|
$this->getMessages( $contLangVariant, Language::MONTH_GENITIVE_MESSAGES )
|
|
);
|
|
break;
|
|
case 'd':
|
|
case 'j':
|
|
$day = intval( $untransformDigits( $text ) );
|
|
break;
|
|
case 'D':
|
|
case 'l':
|
|
// Day of the week - unused
|
|
break;
|
|
case 'F':
|
|
$monthIdx = array_search(
|
|
$text,
|
|
$this->getMessages( $contLangVariant, Language::MONTH_MESSAGES )
|
|
);
|
|
break;
|
|
case 'M':
|
|
$monthIdx = array_search(
|
|
$text,
|
|
$this->getMessages( $contLangVariant, Language::MONTH_ABBREVIATED_MESSAGES )
|
|
);
|
|
break;
|
|
case 'n':
|
|
$monthIdx = intval( $untransformDigits( $text ) ) - 1;
|
|
break;
|
|
case 'Y':
|
|
$year = intval( $untransformDigits( $text ) );
|
|
break;
|
|
case 'xkY':
|
|
// Thai year
|
|
$year = intval( $untransformDigits( $text ) ) - 543;
|
|
break;
|
|
case 'G':
|
|
case 'H':
|
|
$hour = intval( $untransformDigits( $text ) );
|
|
break;
|
|
case 'i':
|
|
$minute = intval( $untransformDigits( $text ) );
|
|
break;
|
|
default:
|
|
// TODO throw NotImplementedException or whatever it's called
|
|
throw new MWException( 'Not implemented' );
|
|
}
|
|
}
|
|
|
|
// The last matching group is the timezone abbreviation
|
|
$tzAbbr = $tzAbbrs[ end( $match ) ];
|
|
|
|
// Most of the time, the timezone abbreviation is not necessary to parse the date, since we
|
|
// can assume all times are in the wiki's local timezone.
|
|
$date = new DateTime();
|
|
// setTimezone must be called before setDate/setTime
|
|
$date->setTimezone( new DateTimeZone( $localTimezone ) );
|
|
$date->setDate( $year, $monthIdx + 1, $day );
|
|
$date->setTime( $hour, $minute, 0 );
|
|
|
|
// But during the "fall back" at the end of DST, some times will happen twice.
|
|
// Since the timezone abbreviation disambiguates the DST/non-DST times, we can detect
|
|
// when PHP chose the wrong one, and then try the other one. It appears that PHP always
|
|
// uses the later (non-DST) hour, but that behavior isn't documented, so we account for both.
|
|
if ( $date->format( 'T' ) !== $tzAbbr ) {
|
|
$altDate = clone $date;
|
|
if ( $date->format( 'I' ) ) {
|
|
// Parsed time is DST, try non-DST by advancing one hour
|
|
$altDate->add( new DateInterval( 'PT1H' ) );
|
|
} else {
|
|
// Parsed time is non-DST, try DST by going back one hour
|
|
$altDate->sub( new DateInterval( 'PT1H' ) );
|
|
}
|
|
if ( $altDate->format( 'T' ) === $tzAbbr ) {
|
|
$date = $altDate;
|
|
$discussionToolsWarning = 'Timestamp has timezone abbreviation for the wrong time';
|
|
} else {
|
|
$discussionToolsWarning = 'Ambiguous time at DST switchover was parsed';
|
|
}
|
|
}
|
|
|
|
// Now set the timezone back to UTC for formatting
|
|
$date->setTimezone( new DateTimeZone( 'UTC' ) );
|
|
$date = DateTimeImmutable::createFromMutable( $date );
|
|
if ( isset( $discussionToolsWarning ) ) {
|
|
// @phan-suppress-next-line PhanUndeclaredProperty
|
|
$date->discussionToolsWarning = $discussionToolsWarning;
|
|
}
|
|
|
|
return $date;
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Get a regexp that matches timestamps in the local date format, for each language variant.
|
|
*
|
|
* This calls getTimestampRegexp() with predefined data for the current wiki.
|
|
*
|
|
* @return string[] Regular expressions
|
|
*/
|
|
public function getLocalTimestampRegexps() : array {
|
|
return array_map( function ( $contLangVariant ) {
|
|
return $this->getTimestampRegexp(
|
|
$contLangVariant,
|
|
$this->dateFormat[$contLangVariant],
|
|
'[' . implode( '', $this->digits[$contLangVariant] ) . ']',
|
|
$this->timezones[$contLangVariant]
|
|
);
|
|
}, $this->language->getVariants() );
|
|
}
|
|
|
|
/**
|
|
* Get a function that parses timestamps in the local date format, for each language variant,
|
|
* based on the result of matching the regexp returned by getLocalTimestampRegexp().
|
|
*
|
|
* This calls getTimestampParser() with predefined data for the current wiki.
|
|
*
|
|
* @return callable[] Parser functions
|
|
*/
|
|
private function getLocalTimestampParsers() : array {
|
|
return array_map( function ( $contLangVariant ) {
|
|
return $this->getTimestampParser(
|
|
$contLangVariant,
|
|
$this->dateFormat[$contLangVariant],
|
|
$this->digits[$contLangVariant],
|
|
$this->localTimezone,
|
|
$this->timezones[$contLangVariant]
|
|
);
|
|
}, $this->language->getVariants() );
|
|
}
|
|
|
|
/**
|
|
* Find a user signature preceding a timestamp.
|
|
*
|
|
* The signature includes the timestamp node.
|
|
*
|
|
* A signature must contain at least one link to the user's userpage, discussion page or
|
|
* contributions (and may contain other links). The link may be nested in other elements.
|
|
*
|
|
* This function returns a two-element array. The first element is an array of sibling nodes
|
|
* comprising the signature, in reverse order (with $timestampNode or its parent node as the last
|
|
* element). The second element is the username (null for unsigned comments).
|
|
*
|
|
* @param DOMText $timestampNode
|
|
* @param DOMNode|null $until Node to stop searching at
|
|
* @return array [ nodes, username ]
|
|
*/
|
|
private function findSignature( DOMText $timestampNode, ?DOMNode $until = null ) : array {
|
|
// Support timestamps being linked to the diff introducing the comment:
|
|
// if the timestamp node is the only child of a link node, use the link node instead
|
|
if (
|
|
!$timestampNode->previousSibling && !$timestampNode->nextSibling &&
|
|
strtolower( $timestampNode->parentNode->nodeName ) === 'a'
|
|
) {
|
|
$timestampNode = $timestampNode->parentNode;
|
|
}
|
|
|
|
$node = $timestampNode;
|
|
$sigNodes = [ $node ];
|
|
$sigUsername = null;
|
|
$length = 0;
|
|
$lastLinkNode = $timestampNode;
|
|
|
|
while (
|
|
( $node = $node->previousSibling ) && $length < self::SIGNATURE_SCAN_LIMIT && $node !== $until
|
|
) {
|
|
$sigNodes[] = $node;
|
|
$length += $node->textContent ? mb_strlen( $node->textContent ) : 0;
|
|
if ( !( $node instanceof DOMElement ) ) {
|
|
continue;
|
|
}
|
|
|
|
$links = [];
|
|
if ( strtolower( $node->nodeName ) === 'a' ) {
|
|
$links = [ $node ];
|
|
} else {
|
|
// Handle links nested in formatting elements.
|
|
// Helpful accidental feature: users whose signature is not detected in full (due to
|
|
// text formatting) can just wrap it in a <span> to fix that.
|
|
// "Ten Pound Hammer • (What did I screw up now?)"
|
|
// "« Saper // dyskusja »"
|
|
$links = iterator_to_array( $node->getElementsByTagName( 'a' ) );
|
|
}
|
|
if ( !count( $links ) ) {
|
|
continue;
|
|
}
|
|
|
|
// Find the closest link before timestamp that links to the user's user page.
|
|
foreach ( array_reverse( $links ) as $link ) {
|
|
$username = null;
|
|
$title = CommentUtils::getTitleFromUrl( $link->getAttribute( 'href' ) );
|
|
if ( !$title ) {
|
|
continue;
|
|
}
|
|
if ( $title->getNamespace() === NS_USER || $title->getNamespace() === NS_USER_TALK ) {
|
|
$username = $title->getText();
|
|
if ( strpos( $username, '/' ) !== false ) {
|
|
continue;
|
|
}
|
|
} elseif ( $title->isSpecial( 'Contributions' ) ) {
|
|
$parts = explode( '/', $title->getText(), 2 );
|
|
if ( !isset( $parts[1] ) ) {
|
|
continue;
|
|
}
|
|
// Normalize the username: users may link to their contributions with an unnormalized name
|
|
$userpage = Title::makeTitleSafe( NS_USER, $parts[1] );
|
|
if ( !$userpage ) {
|
|
continue;
|
|
}
|
|
$username = $userpage->getText();
|
|
}
|
|
if ( !$username ) {
|
|
continue;
|
|
}
|
|
if ( IP::isIPv6( $username ) ) {
|
|
// Bot-generated links "Preceding unsigned comment added by" have non-standard case
|
|
$username = strtoupper( $username );
|
|
}
|
|
|
|
// Accept the first link to the user namespace, then only accept links to that user
|
|
if ( $sigUsername === null ) {
|
|
$sigUsername = $username;
|
|
}
|
|
if ( $username === $sigUsername ) {
|
|
$lastLinkNode = $node;
|
|
break;
|
|
}
|
|
}
|
|
// Keep looking if a node with links wasn't a link to a user page
|
|
// "Doc James (talk · contribs · email)"
|
|
}
|
|
// Pop excess text nodes
|
|
while ( end( $sigNodes ) !== $lastLinkNode ) {
|
|
array_pop( $sigNodes );
|
|
}
|
|
return [ $sigNodes, $sigUsername ];
|
|
}
|
|
|
|
/**
|
|
* Callback for TreeWalker that will skip over nodes where we don't want to detect
|
|
* comments (or section headings).
|
|
*
|
|
* @param DOMNode $node
|
|
* @return int Appropriate NodeFilter constant
|
|
*/
|
|
public static function acceptOnlyNodesAllowingComments( DOMNode $node ) {
|
|
// The table of contents has a heading that gets erroneously detected as a section
|
|
if ( $node instanceof DOMElement && $node->getAttribute( 'id' ) === 'toc' ) {
|
|
return NodeFilter::FILTER_REJECT;
|
|
}
|
|
// Don't detect comments within headings (but don't reject the headings themselves)
|
|
if ( ( $par = $node->parentNode ) instanceof DOMElement && preg_match( '/^h([1-6])$/i', $par->tagName ) ) {
|
|
return NodeFilter::FILTER_REJECT;
|
|
}
|
|
return NodeFilter::FILTER_ACCEPT;
|
|
}
|
|
|
|
/**
|
|
* Find a timestamps in a given text node
|
|
*
|
|
* @param DOMText $node
|
|
* @param string[] $timestampRegexps
|
|
* @return array|null Array with the following keys:
|
|
* - int 'offset' Length of extra text preceding the node that was used for matching
|
|
* - int 'parserIndex' Which of the regexps matched
|
|
* - array 'matchData' Regexp match data, which specifies the location of the match,
|
|
* and which can be parsed using getLocalTimestampParsers()
|
|
*/
|
|
public function findTimestamp( DOMText $node, array $timestampRegexps ) : ?array {
|
|
$nodeText = '';
|
|
$offset = 0;
|
|
|
|
while ( $node ) {
|
|
$nodeText = $node->nodeValue . $nodeText;
|
|
|
|
// In Parsoid HTML, entities are represented as a 'mw:Entity' node, rather than normal HTML
|
|
// entities. On Arabic Wikipedia, the "UTC" timezone name contains some non-breaking spaces,
|
|
// which apparently are often turned into entities by buggy editing tools. To handle
|
|
// this, we must piece together the text, so that our regexp can match those timestamps.
|
|
if (
|
|
( $previousSibling = $node->previousSibling ) &&
|
|
$previousSibling instanceof DOMElement &&
|
|
$previousSibling->getAttribute( 'typeof' ) === 'mw:Entity'
|
|
) {
|
|
$nodeText = $previousSibling->firstChild->nodeValue . $nodeText;
|
|
$offset += strlen( $previousSibling->firstChild->nodeValue );
|
|
|
|
// If the entity is preceded by more text, do this again
|
|
if (
|
|
$previousSibling->previousSibling &&
|
|
$previousSibling->previousSibling instanceof DOMText
|
|
) {
|
|
$offset += strlen( $previousSibling->previousSibling->nodeValue );
|
|
$node = $previousSibling->previousSibling;
|
|
} else {
|
|
$node = null;
|
|
}
|
|
} else {
|
|
$node = null;
|
|
}
|
|
}
|
|
|
|
foreach ( $timestampRegexps as $i => $timestampRegexp ) {
|
|
$matchData = null;
|
|
// Allows us to mimic match.index in #getComments
|
|
if ( preg_match( $timestampRegexp, $nodeText, $matchData, PREG_OFFSET_CAPTURE ) ) {
|
|
return [
|
|
'matchData' => $matchData,
|
|
'offset' => $offset,
|
|
'parserIndex' => $i,
|
|
];
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Get all discussion comments (and headings) within a DOM subtree.
|
|
*
|
|
* This returns a flat list, use getThreads() to get a tree structure starting at section headings.
|
|
*
|
|
* For example, for a MediaWiki discussion like this (we're dealing with HTML DOM here,
|
|
* the wikitext syntax is just for illustration):
|
|
*
|
|
* == A ==
|
|
* B. ~~~~
|
|
* : C.
|
|
* : C. ~~~~
|
|
* :: D. ~~~~
|
|
* ::: E. ~~~~
|
|
* ::: F. ~~~~
|
|
* : G. ~~~~
|
|
* H. ~~~~
|
|
* : I. ~~~~
|
|
*
|
|
* This function would return a structure like:
|
|
*
|
|
* [
|
|
* HeadingItem( { level: 0, range: (h2: A) } ),
|
|
* CommentItem( { level: 1, range: (p: B) } ),
|
|
* CommentItem( { level: 2, range: (li: C, li: C) } ),
|
|
* CommentItem( { level: 3, range: (li: D) } ),
|
|
* CommentItem( { level: 4, range: (li: E) } ),
|
|
* CommentItem( { level: 4, range: (li: F) } ),
|
|
* CommentItem( { level: 2, range: (li: G) } ),
|
|
* CommentItem( { level: 1, range: (p: H) } ),
|
|
* CommentItem( { level: 2, range: (li: I) } )
|
|
* ]
|
|
*
|
|
* @return ThreadItem[] Thread items
|
|
*/
|
|
public function getThreadItems() : array {
|
|
if ( !$this->threadItems ) {
|
|
$this->buildThreads();
|
|
}
|
|
return $this->threadItems;
|
|
}
|
|
|
|
/**
|
|
* Same as getFlatThreadItems, but only returns the CommentItems
|
|
*
|
|
* @return CommentItem[] Comment items
|
|
*/
|
|
public function getCommentItems() : array {
|
|
if ( !$this->commentItems ) {
|
|
$this->buildThreads();
|
|
}
|
|
return $this->commentItems;
|
|
}
|
|
|
|
/**
|
|
* Find a ThreadItem by its ID
|
|
*
|
|
* @param string $id ID
|
|
* @return ThreadItem|null Thread item, null if not found
|
|
*/
|
|
public function findCommentById( string $id ) : ?ThreadItem {
|
|
if ( !$this->threadItemsById ) {
|
|
$this->buildThreads();
|
|
}
|
|
return $this->threadItemsById[$id] ?? null;
|
|
}
|
|
|
|
private function buildThreadItems() : void {
|
|
$timestampRegexps = $this->getLocalTimestampRegexps();
|
|
$commentItems = [];
|
|
$threadItems = [];
|
|
$dfParsers = $this->getLocalTimestampParsers();
|
|
|
|
// Placeholder heading in case there are comments in the 0th section
|
|
$range = new ImmutableRange( $this->rootNode, 0, $this->rootNode, 0 );
|
|
$fakeHeading = new HeadingItem( $range, 99, true );
|
|
$fakeHeading->setRootNode( $this->rootNode );
|
|
|
|
$curComment = $fakeHeading;
|
|
$curCommentEnd = $range->endContainer;
|
|
|
|
$treeWalker = new TreeWalker(
|
|
$this->rootNode,
|
|
NodeFilter::SHOW_ELEMENT | NodeFilter::SHOW_TEXT,
|
|
[ self::class, 'acceptOnlyNodesAllowingComments' ]
|
|
);
|
|
$lastSigNode = null;
|
|
while ( $node = $treeWalker->nextNode() ) {
|
|
if ( $node instanceof DOMElement && preg_match( '/^h([1-6])$/i', $node->tagName, $match ) ) {
|
|
$headingNodeAndOffset = CommentUtils::getHeadlineNodeAndOffset( $node );
|
|
$headingNode = $headingNodeAndOffset['node'];
|
|
$startOffset = $headingNodeAndOffset['offset'];
|
|
$range = new ImmutableRange(
|
|
$headingNode, $startOffset, $headingNode, $headingNode->childNodes->length
|
|
);
|
|
$curComment = new HeadingItem( $range, (int)( $match[ 1 ] ) );
|
|
$curComment->setRootNode( $this->rootNode );
|
|
$threadItems[] = $curComment;
|
|
$curCommentEnd = $node;
|
|
} elseif ( $node instanceof DOMText && ( $match = $this->findTimestamp( $node, $timestampRegexps ) ) ) {
|
|
$warnings = [];
|
|
$foundSignature = $this->findSignature( $node, $lastSigNode );
|
|
$author = $foundSignature[1];
|
|
$firstSigNode = end( $foundSignature[0] );
|
|
$lastSigNode = $foundSignature[0][0];
|
|
|
|
if ( !$author ) {
|
|
// Ignore timestamps for which we couldn't find a signature. It's probably not a real
|
|
// comment, but just a false match due to a copypasted timestamp.
|
|
continue;
|
|
}
|
|
|
|
$lastSigNodeOffset = $lastSigNode === $node ?
|
|
$match['matchData'][0][1] + strlen( $match['matchData'][0][0] ) - $match['offset'] :
|
|
CommentUtils::childIndexOf( $lastSigNode ) + 1;
|
|
$sigRange = new ImmutableRange(
|
|
$firstSigNode->parentNode,
|
|
CommentUtils::childIndexOf( $firstSigNode ),
|
|
$lastSigNode === $node ? $node : $lastSigNode->parentNode,
|
|
$lastSigNodeOffset
|
|
);
|
|
|
|
// Everything from the last comment up to here is the next comment
|
|
$startNode = $this->nextInterestingLeafNode( $curCommentEnd );
|
|
$endNode = $lastSigNode;
|
|
while (
|
|
// If this isn't a last sibling:
|
|
// Skip to the end of the "paragraph". This only looks at tag names and can be fooled by CSS, but
|
|
// avoiding that would be more difficult and slower.
|
|
( $endNode->nextSibling && !CommentUtils::isBlockElement( $endNode->nextSibling ) ) ||
|
|
// When we reach a last sibling:
|
|
// Traverse up from inline nodes until the parent is a block element.
|
|
// This moves end markers from
|
|
// "timestamp|</small></span></p>"
|
|
// to
|
|
// "timestamp</small></span>|</p>"
|
|
(
|
|
!$endNode->nextSibling &&
|
|
!CommentUtils::isBlockElement( $endNode ) &&
|
|
!CommentUtils::isBlockElement( $endNode->parentNode )
|
|
)
|
|
) {
|
|
$endNode = $endNode->nextSibling ?: $endNode->parentNode;
|
|
}
|
|
|
|
$length = ( $endNode->nodeType === XML_TEXT_NODE ) ?
|
|
strlen( rtrim( $endNode->nodeValue, "\t\n\f\r " ) ) :
|
|
// PHP bug: childNodes can be null for comment nodes
|
|
// (it should always be a DOMNodeList, even if the node can't have children)
|
|
( $endNode->childNodes ? $endNode->childNodes->length : 0 );
|
|
$range = new ImmutableRange(
|
|
$startNode->parentNode,
|
|
CommentUtils::childIndexOf( $startNode ),
|
|
$endNode,
|
|
$length
|
|
);
|
|
|
|
$startLevel = CommentUtils::getIndentLevel( $startNode, $this->rootNode ) + 1;
|
|
$endLevel = CommentUtils::getIndentLevel( $node, $this->rootNode ) + 1;
|
|
if ( $startLevel !== $endLevel ) {
|
|
$warnings[] = 'Comment starts and ends with different indentation';
|
|
}
|
|
// Should this use the indent level of $startNode or $node?
|
|
$level = min( $startLevel, $endLevel );
|
|
|
|
$dateTime = $dfParsers[ $match['parserIndex'] ]( $match['matchData'] );
|
|
if ( isset( $dateTime->discussionToolsWarning ) ) {
|
|
$warnings[] = $dateTime->discussionToolsWarning;
|
|
}
|
|
// ISO 8601 date. Almost DateTimeInterface::RFC3339_EXTENDED, but ending with 'Z' instead
|
|
// of '+00:00', like Date#toISOString in JavaScript.
|
|
$dateTimeStr = $dateTime->format( 'Y-m-d\TH:i:s.v\Z' );
|
|
|
|
// Avoid generating multiple comments when there is more than one signature on a single "line".
|
|
// Often this is done when someone edits their comment later and wants to add a note about that.
|
|
// (Or when another person corrects a typo, or strikes out a comment, etc.) Multiple comments
|
|
// within one paragraph/list-item result in a confusing double "Reply" button, and we also have
|
|
// no way to indicate which one you're replying to (this might matter in the future for
|
|
// notifications or something).
|
|
if (
|
|
$curComment instanceof CommentItem &&
|
|
(
|
|
CommentUtils::closestElement(
|
|
$node, [ 'li', 'dd', 'p' ]
|
|
) ?? $node->parentNode
|
|
) ===
|
|
(
|
|
CommentUtils::closestElement(
|
|
$curComment->getRange()->endContainer, [ 'li', 'dd', 'p' ]
|
|
) ?? $curComment->getRange()->endContainer->parentNode
|
|
)
|
|
) {
|
|
// Merge this with the previous comment. Use that comment's author and timestamp.
|
|
$curComment->addSignatureRange( $sigRange );
|
|
|
|
if (
|
|
$curComment->getRange()->endContainer === $range->endContainer &&
|
|
$curComment->getRange()->endOffset <= $range->endOffset
|
|
) {
|
|
// We've already skipped over this signature, and the $range and $level are messed up,
|
|
// because that causes $startNode to be after $endNode
|
|
continue;
|
|
}
|
|
|
|
$curComment->setRange(
|
|
$curComment->getRange()->setEnd( $range->endContainer, $range->endOffset )
|
|
);
|
|
$curComment->setLevel( min( $level, $curComment->getLevel() ) );
|
|
$curCommentEnd = $range->endContainer;
|
|
|
|
continue;
|
|
}
|
|
|
|
$curComment = new CommentItem(
|
|
$level,
|
|
$range,
|
|
[ $sigRange ],
|
|
$dateTimeStr,
|
|
$author
|
|
);
|
|
$curComment->setRootNode( $this->rootNode );
|
|
if ( $warnings ) {
|
|
$curComment->addWarnings( $warnings );
|
|
}
|
|
$commentItems[] = $curComment;
|
|
$threadItems[] = $curComment;
|
|
$curCommentEnd = $curComment->getRange()->endContainer;
|
|
}
|
|
}
|
|
|
|
// Insert the fake placeholder heading if there are any comments in the 0th section
|
|
// (before the first real heading)
|
|
if ( count( $threadItems ) && !( $threadItems[ 0 ] instanceof HeadingItem ) ) {
|
|
array_unshift( $threadItems, $fakeHeading );
|
|
}
|
|
|
|
$this->commentItems = $commentItems;
|
|
$this->threadItems = $threadItems;
|
|
}
|
|
|
|
/**
|
|
* Group discussion comments into threads and associate replies to original messages.
|
|
*
|
|
* Each thread must begin with a heading. Original messages in the thread are treated as replies to
|
|
* its heading. Other replies are associated based on the order and indentation level.
|
|
*
|
|
* Note that the objects in `comments` are extended in-place with the additional data.
|
|
*
|
|
* For example, for a MediaWiki discussion like this (we're dealing with HTML DOM here,
|
|
* the wikitext syntax is just for illustration):
|
|
*
|
|
* == A ==
|
|
* B. ~~~~
|
|
* : C.
|
|
* : C. ~~~~
|
|
* :: D. ~~~~
|
|
* ::: E. ~~~~
|
|
* ::: F. ~~~~
|
|
* : G. ~~~~
|
|
* H. ~~~~
|
|
* : I. ~~~~
|
|
*
|
|
* This function would return a structure like:
|
|
*
|
|
* [
|
|
* HeadingItem( { level: 0, range: (h2: A), replies: [
|
|
* CommentItem( { level: 1, range: (p: B), replies: [
|
|
* CommentItem( { level: 2, range: (li: C, li: C), replies: [
|
|
* CommentItem( { level: 3, range: (li: D), replies: [
|
|
* CommentItem( { level: 4, range: (li: E), replies: [] },
|
|
* CommentItem( { level: 4, range: (li: F), replies: [] },
|
|
* ] },
|
|
* ] },
|
|
* CommentItem( { level: 2, range: (li: G), replies: [] },
|
|
* ] },
|
|
* CommentItem( { level: 1, range: (p: H), replies: [
|
|
* CommentItem( { level: 2, range: (li: I), replies: [] },
|
|
* ] },
|
|
* ] } )
|
|
* ]
|
|
*
|
|
* @return HeadingItem[] Tree structure of comments, top-level items are the headings.
|
|
*/
|
|
public function getThreads() : array {
|
|
if ( !$this->threads ) {
|
|
$this->buildThreads();
|
|
}
|
|
return $this->threads;
|
|
}
|
|
|
|
/**
|
|
* Given a thread item, return an identifier for it that is unique within the page.
|
|
*
|
|
* @param ThreadItem $threadItem
|
|
* @return string|null
|
|
*/
|
|
private function computeId( ThreadItem $threadItem ) : ?string {
|
|
$id = null;
|
|
|
|
if ( $threadItem instanceof HeadingItem && $threadItem->isPlaceholderHeading() ) {
|
|
// The range points to the root note, using it like below results in silly values
|
|
$id = 'h|';
|
|
} elseif ( $threadItem instanceof HeadingItem ) {
|
|
$headline = CommentUtils::getHeadlineNodeAndOffset( $threadItem->getRange()->startContainer )['node'];
|
|
$id = 'h|' . ( $headline->getAttribute( 'id' ) ?? '' );
|
|
} elseif ( $threadItem instanceof CommentItem ) {
|
|
$id = 'c|' . ( $threadItem->getAuthor() ?? '' ) . '|' . $threadItem->getTimestamp();
|
|
} else {
|
|
throw new MWException( 'Unknown ThreadItem type' );
|
|
}
|
|
|
|
// If there would be multiple comments with the same ID (i.e. the user left multiple comments
|
|
// in one edit, or within a minute), add the parent ID to disambiguate them.
|
|
$threadItemParent = $threadItem->getParent();
|
|
if ( $threadItemParent instanceof HeadingItem && !$threadItemParent->isPlaceholderHeading() ) {
|
|
$headline = CommentUtils::getHeadlineNodeAndOffset( $threadItemParent->getRange()->startContainer )['node'];
|
|
$id .= '|' . ( $headline->getAttribute( 'id' ) ?? '' );
|
|
} elseif ( $threadItemParent instanceof CommentItem ) {
|
|
$id .= '|' . ( $threadItemParent->getAuthor() ?? '' ) . '|' . $threadItemParent->getTimestamp();
|
|
}
|
|
|
|
if ( $threadItem instanceof HeadingItem ) {
|
|
// To avoid old threads re-appearing on popular pages when someone uses a vague title
|
|
// (e.g. dozens of threads titled "question" on [[Wikipedia:Help desk]]: https://w.wiki/fbN),
|
|
// include the oldest timestamp in the thread (i.e. date the thread was started) in the
|
|
// heading ID.
|
|
$timestamp = $this->getThreadStartTimestamp( $threadItem );
|
|
if ( $timestamp ) {
|
|
$id .= '|' . $timestamp;
|
|
}
|
|
}
|
|
|
|
if ( isset( $this->threadItemsById[$id] ) ) {
|
|
// Well, that's tough
|
|
$threadItem->addWarning( 'Duplicate comment ID' );
|
|
// Finally, disambiguate by adding sequential numbers, to allow replying to both comments
|
|
$number = 1;
|
|
while ( isset( $this->threadItemsById["$id|$number"] ) ) {
|
|
$number++;
|
|
}
|
|
$id = "$id|$number";
|
|
}
|
|
|
|
return $id;
|
|
}
|
|
|
|
/**
|
|
* Given a thread item, return an identifier for it like computeId(), generated according to an
|
|
* older algorithm, so that we can still match IDs from cached data.
|
|
*
|
|
* @param ThreadItem $threadItem
|
|
* @return string|null
|
|
*/
|
|
private function computeLegacyId( ThreadItem $threadItem ) : ?string {
|
|
if ( $threadItem instanceof HeadingItem ) {
|
|
// We don't need ids for section headings right now, but we might in the future
|
|
// e.g. if we allow replying directly to sections (adding top-level comments)
|
|
$id = null;
|
|
} elseif ( $threadItem instanceof CommentItem ) {
|
|
$id = ( $threadItem->getAuthor() ?? '' ) . '|' . $threadItem->getTimestamp();
|
|
|
|
// If there would be multiple comments with the same ID (i.e. the user left multiple comments
|
|
// in one edit, or within a minute), append sequential numbers
|
|
$number = 0;
|
|
while ( isset( $this->threadItemsById["$id|$number"] ) ) {
|
|
$number++;
|
|
}
|
|
$id = "$id|$number";
|
|
} else {
|
|
throw new MWException( 'Unknown ThreadItem type' );
|
|
}
|
|
|
|
return $id;
|
|
}
|
|
|
|
private function buildThreads() : void {
|
|
if ( !$this->threadItems ) {
|
|
$this->buildThreadItems();
|
|
}
|
|
|
|
$threads = [];
|
|
$replies = [];
|
|
$this->threadItemsById = [];
|
|
|
|
foreach ( $this->threadItems as $threadItem ) {
|
|
if ( count( $replies ) < $threadItem->getLevel() ) {
|
|
// Someone skipped an indentation level (or several). Pretend that the previous reply
|
|
// covers multiple indentation levels, so that following comments get connected to it.
|
|
$threadItem->addWarning( 'Comment skips indentation level' );
|
|
while ( count( $replies ) < $threadItem->getLevel() ) {
|
|
$replies[] = end( $replies );
|
|
}
|
|
}
|
|
|
|
if ( $threadItem instanceof HeadingItem ) {
|
|
// New root (thread)
|
|
$threads[] = $threadItem;
|
|
// Attach as a sub-thread to preceding higher-level heading.
|
|
// Any replies will appear in the tree twice, under the main-thread and the sub-thread.
|
|
$maybeParent = count( $threads ) > 1 ? $threads[ count( $threads ) - 2 ] : null;
|
|
while ( $maybeParent && $maybeParent->getHeadingLevel() >= $threadItem->getHeadingLevel() ) {
|
|
$maybeParent = $maybeParent->getParent();
|
|
}
|
|
if ( $maybeParent ) {
|
|
$threadItem->setParent( $maybeParent );
|
|
$maybeParent->addReply( $threadItem );
|
|
}
|
|
} elseif ( isset( $replies[ $threadItem->getLevel() - 1 ] ) ) {
|
|
// Add as a reply to the closest less-nested comment
|
|
$threadItem->setParent( $replies[ $threadItem->getLevel() - 1 ] );
|
|
$threadItem->getParent()->addReply( $threadItem );
|
|
} else {
|
|
$threadItem->addWarning( 'Comment could not be connected to a thread' );
|
|
}
|
|
|
|
$replies[ $threadItem->getLevel() ] = $threadItem;
|
|
// Cut off more deeply nested replies
|
|
array_splice( $replies, $threadItem->getLevel() + 1 );
|
|
}
|
|
|
|
$this->threads = $threads;
|
|
|
|
foreach ( $this->threadItems as $threadItem ) {
|
|
// Set the IDs used to refer to comments and headings.
|
|
// This has to be a separate pass because we don't have the list of replies before
|
|
// this point.
|
|
$id = $this->computeId( $threadItem );
|
|
if ( $id ) {
|
|
$this->threadItemsById[$id] = $threadItem;
|
|
}
|
|
$threadItem->setId( $id );
|
|
$legacyId = $this->computeLegacyId( $threadItem );
|
|
$threadItem->setLegacyId( $legacyId );
|
|
if ( $legacyId ) {
|
|
$this->threadItemsById[$legacyId] = $threadItem;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param ThreadItem $threadItem
|
|
* @return string|null
|
|
*/
|
|
private function getThreadStartTimestamp( ThreadItem $threadItem ) {
|
|
$timestamp = null;
|
|
if ( $threadItem instanceof CommentItem ) {
|
|
$timestamp = $threadItem->getTimestamp();
|
|
}
|
|
// Check all replies. This can't just use the first comment because threads are often summarized
|
|
// at the top when the discussion is closed.
|
|
foreach ( $threadItem->getReplies() as $comment ) {
|
|
// Don't include sub-threads to avoid changing the ID when threads are "merged".
|
|
if ( $comment instanceof CommentItem ) {
|
|
$timestampInReplies = $this->getThreadStartTimestamp( $comment );
|
|
if ( !$timestamp || $timestampInReplies < $timestamp ) {
|
|
$timestamp = $timestampInReplies;
|
|
}
|
|
}
|
|
}
|
|
return $timestamp;
|
|
}
|
|
}
|