2004-12-11 09:59:06 +00:00
|
|
|
|
<?php
|
|
|
|
|
|
2007-11-12 07:44:17 +00:00
|
|
|
|
if ( !defined( 'MEDIAWIKI' ) ) {
|
|
|
|
|
exit;
|
|
|
|
|
}
|
2004-12-11 09:59:06 +00:00
|
|
|
|
|
2016-06-30 21:26:36 +00:00
|
|
|
|
use \MediaWiki\MediaWikiServices;
|
|
|
|
|
|
2012-01-17 06:13:46 +00:00
|
|
|
|
class SpamBlacklist extends BaseBlacklist {
|
2016-08-30 03:36:36 +00:00
|
|
|
|
const STASH_TTL = 60;
|
|
|
|
|
const STASH_AGE_DYING = 30;
|
2007-01-06 20:56:46 +00:00
|
|
|
|
|
2016-01-08 21:58:59 +00:00
|
|
|
|
/**
|
2016-05-23 17:51:45 +00:00
|
|
|
|
* Changes to external links, for logging purposes
|
2016-01-08 21:58:59 +00:00
|
|
|
|
* @var array[]
|
|
|
|
|
*/
|
2016-05-18 21:00:00 +00:00
|
|
|
|
private $urlChangeLog = array();
|
2016-01-08 21:58:59 +00:00
|
|
|
|
|
2007-07-20 21:13:26 +00:00
|
|
|
|
/**
|
2012-01-17 06:13:46 +00:00
|
|
|
|
* Returns the code for the blacklist implementation
|
|
|
|
|
*
|
|
|
|
|
* @return string
|
2007-07-20 21:13:26 +00:00
|
|
|
|
*/
|
2012-01-17 06:13:46 +00:00
|
|
|
|
protected function getBlacklistType() {
|
|
|
|
|
return 'spam';
|
2006-06-22 19:59:43 +00:00
|
|
|
|
}
|
2007-01-06 20:56:46 +00:00
|
|
|
|
|
2007-11-12 07:44:17 +00:00
|
|
|
|
/**
|
2013-07-25 14:05:13 +00:00
|
|
|
|
* Apply some basic anti-spoofing to the links before they get filtered,
|
|
|
|
|
* see @bug 12896
|
|
|
|
|
*
|
|
|
|
|
* @param string $text
|
|
|
|
|
*
|
|
|
|
|
* @return string
|
2007-11-12 07:44:17 +00:00
|
|
|
|
*/
|
2013-07-25 14:05:13 +00:00
|
|
|
|
protected function antiSpoof( $text ) {
|
|
|
|
|
$text = str_replace( '.', '.', $text );
|
|
|
|
|
return $text;
|
|
|
|
|
}
|
2006-01-19 17:14:10 +00:00
|
|
|
|
|
2013-07-25 14:05:13 +00:00
|
|
|
|
/**
|
|
|
|
|
* @param string[] $links An array of links to check against the blacklist
|
|
|
|
|
* @param Title $title The title of the page to which the filter shall be applied.
|
|
|
|
|
* This is used to load the old links already on the page, so
|
|
|
|
|
* the filter is only applied to links that got added. If not given,
|
|
|
|
|
* the filter is applied to all $links.
|
2013-09-21 23:47:21 +00:00
|
|
|
|
* @param boolean $preventLog Whether to prevent logging of hits. Set to true when
|
|
|
|
|
* the action is testing the links rather than attempting to save them
|
|
|
|
|
* (e.g. the API spamblacklist action)
|
2016-08-30 03:36:36 +00:00
|
|
|
|
* @param string $mode Either 'check' or 'stash'
|
2013-07-25 14:05:13 +00:00
|
|
|
|
*
|
2016-06-30 21:26:36 +00:00
|
|
|
|
* @return string[]|bool Matched text(s) if the edit should not be allowed; false otherwise
|
2013-07-25 14:05:13 +00:00
|
|
|
|
*/
|
2016-08-30 03:36:36 +00:00
|
|
|
|
function filter( array $links, Title $title = null, $preventLog = false, $mode = 'check' ) {
|
2016-06-30 21:26:36 +00:00
|
|
|
|
$statsd = MediaWikiServices::getInstance()->getStatsdDataFactory();
|
|
|
|
|
$cache = ObjectCache::getLocalClusterInstance();
|
2016-09-09 00:44:26 +00:00
|
|
|
|
|
|
|
|
|
sort( $links );
|
2016-06-30 21:26:36 +00:00
|
|
|
|
$key = $cache->makeKey(
|
|
|
|
|
'blacklist',
|
|
|
|
|
$this->getBlacklistType(),
|
|
|
|
|
'pass',
|
|
|
|
|
sha1( implode( "\n", $links ) ),
|
|
|
|
|
(string)$title
|
|
|
|
|
);
|
|
|
|
|
// Skip blacklist checks if nothing matched during edit stashing...
|
2016-08-30 03:36:36 +00:00
|
|
|
|
$knownNonMatchAsOf = $cache->get( $key );
|
|
|
|
|
if ( $mode === 'check' ) {
|
|
|
|
|
if ( $knownNonMatchAsOf ) {
|
|
|
|
|
$statsd->increment( 'spamblacklist.check-stash.hit' );
|
|
|
|
|
|
|
|
|
|
return false;
|
|
|
|
|
} else {
|
|
|
|
|
$statsd->increment( 'spamblacklist.check-stash.miss' );
|
|
|
|
|
}
|
|
|
|
|
} elseif ( $mode === 'stash' ) {
|
|
|
|
|
if ( $knownNonMatchAsOf && ( time() - $knownNonMatchAsOf ) < self::STASH_AGE_DYING ) {
|
|
|
|
|
return false; // OK; not about to expire soon
|
|
|
|
|
}
|
2016-06-30 21:26:36 +00:00
|
|
|
|
}
|
|
|
|
|
|
2007-07-07 17:21:49 +00:00
|
|
|
|
$blacklists = $this->getBlacklists();
|
2006-09-18 09:56:57 +00:00
|
|
|
|
$whitelists = $this->getWhitelists();
|
2007-01-06 20:56:46 +00:00
|
|
|
|
|
2007-07-07 17:21:49 +00:00
|
|
|
|
if ( count( $blacklists ) ) {
|
2013-07-25 14:05:13 +00:00
|
|
|
|
// poor man's anti-spoof, see bug 12896
|
|
|
|
|
$newLinks = array_map( array( $this, 'antiSpoof' ), $links );
|
2008-11-02 22:40:02 +00:00
|
|
|
|
|
2013-07-25 14:05:13 +00:00
|
|
|
|
$oldLinks = array();
|
|
|
|
|
if ( $title !== null ) {
|
|
|
|
|
$oldLinks = $this->getCurrentLinks( $title );
|
|
|
|
|
$addedLinks = array_diff( $newLinks, $oldLinks );
|
|
|
|
|
} else {
|
|
|
|
|
// can't load old links, so treat all links as added.
|
|
|
|
|
$addedLinks = $newLinks;
|
2011-01-23 10:34:56 +00:00
|
|
|
|
}
|
2008-11-02 22:40:02 +00:00
|
|
|
|
|
2008-05-13 23:31:33 +00:00
|
|
|
|
wfDebugLog( 'SpamBlacklist', "Old URLs: " . implode( ', ', $oldLinks ) );
|
|
|
|
|
wfDebugLog( 'SpamBlacklist', "New URLs: " . implode( ', ', $newLinks ) );
|
|
|
|
|
wfDebugLog( 'SpamBlacklist', "Added URLs: " . implode( ', ', $addedLinks ) );
|
2008-11-02 22:40:02 +00:00
|
|
|
|
|
2016-06-30 21:55:24 +00:00
|
|
|
|
if ( !$preventLog ) {
|
|
|
|
|
$this->logUrlChanges( $oldLinks, $newLinks, $addedLinks );
|
|
|
|
|
}
|
2016-01-08 21:58:59 +00:00
|
|
|
|
|
2008-05-13 23:31:33 +00:00
|
|
|
|
$links = implode( "\n", $addedLinks );
|
2007-01-06 20:56:46 +00:00
|
|
|
|
|
2006-06-22 19:59:43 +00:00
|
|
|
|
# Strip whitelisted URLs from the match
|
2006-09-18 09:56:57 +00:00
|
|
|
|
if( is_array( $whitelists ) ) {
|
2007-10-03 00:19:36 +00:00
|
|
|
|
wfDebugLog( 'SpamBlacklist', "Excluding whitelisted URLs from " . count( $whitelists ) .
|
2006-09-18 09:56:57 +00:00
|
|
|
|
" regexes: " . implode( ', ', $whitelists ) . "\n" );
|
|
|
|
|
foreach( $whitelists as $regex ) {
|
2007-08-08 15:42:36 +00:00
|
|
|
|
wfSuppressWarnings();
|
2007-10-03 00:48:57 +00:00
|
|
|
|
$newLinks = preg_replace( $regex, '', $links );
|
2007-08-08 15:42:36 +00:00
|
|
|
|
wfRestoreWarnings();
|
2007-10-03 00:48:57 +00:00
|
|
|
|
if( is_string( $newLinks ) ) {
|
|
|
|
|
// If there wasn't a regex error, strip the matching URLs
|
|
|
|
|
$links = $newLinks;
|
|
|
|
|
}
|
2006-09-18 09:56:57 +00:00
|
|
|
|
}
|
2006-06-22 19:59:43 +00:00
|
|
|
|
}
|
2006-04-12 04:59:27 +00:00
|
|
|
|
|
2004-12-11 09:59:06 +00:00
|
|
|
|
# Do the match
|
2007-10-03 00:19:36 +00:00
|
|
|
|
wfDebugLog( 'SpamBlacklist', "Checking text against " . count( $blacklists ) .
|
2007-07-07 17:21:49 +00:00
|
|
|
|
" regexes: " . implode( ', ', $blacklists ) . "\n" );
|
2006-09-18 09:56:57 +00:00
|
|
|
|
$retVal = false;
|
2007-07-07 17:21:49 +00:00
|
|
|
|
foreach( $blacklists as $regex ) {
|
2007-07-20 21:13:26 +00:00
|
|
|
|
wfSuppressWarnings();
|
2011-01-23 10:34:56 +00:00
|
|
|
|
$matches = array();
|
2012-03-27 20:42:49 +00:00
|
|
|
|
$check = ( preg_match_all( $regex, $links, $matches ) > 0 );
|
2007-07-20 21:13:26 +00:00
|
|
|
|
wfRestoreWarnings();
|
|
|
|
|
if( $check ) {
|
2007-10-03 00:19:36 +00:00
|
|
|
|
wfDebugLog( 'SpamBlacklist', "Match!\n" );
|
2014-03-31 06:01:45 +00:00
|
|
|
|
global $wgRequest;
|
|
|
|
|
$ip = $wgRequest->getIP();
|
2014-10-28 04:05:27 +00:00
|
|
|
|
$fullUrls = array();
|
|
|
|
|
$fullLineRegex = substr( $regex, 0, strrpos( $regex, '/' ) ) . '.*/Sim';
|
|
|
|
|
preg_match_all( $fullLineRegex, $links, $fullUrls );
|
|
|
|
|
$imploded = implode( ' ', $fullUrls[0] );
|
2014-03-31 06:01:45 +00:00
|
|
|
|
wfDebugLog( 'SpamBlacklistHit', "$ip caught submitting spam: $imploded\n" );
|
2013-09-21 23:47:21 +00:00
|
|
|
|
if( !$preventLog ) {
|
|
|
|
|
$this->logFilterHit( $title, $imploded ); // Log it
|
|
|
|
|
}
|
2012-03-27 20:42:49 +00:00
|
|
|
|
if( $retVal === false ){
|
|
|
|
|
$retVal = array();
|
|
|
|
|
}
|
2014-10-28 04:05:27 +00:00
|
|
|
|
$retVal = array_merge( $retVal, $fullUrls[1] );
|
2006-09-18 09:56:57 +00:00
|
|
|
|
}
|
2004-12-11 09:59:06 +00:00
|
|
|
|
}
|
2013-01-11 20:02:17 +00:00
|
|
|
|
if ( is_array( $retVal ) ) {
|
|
|
|
|
$retVal = array_unique( $retVal );
|
|
|
|
|
}
|
2004-12-11 09:59:06 +00:00
|
|
|
|
} else {
|
|
|
|
|
$retVal = false;
|
|
|
|
|
}
|
2015-02-10 22:07:45 +00:00
|
|
|
|
|
2016-06-30 21:26:36 +00:00
|
|
|
|
if ( $retVal === false ) {
|
|
|
|
|
// Cache the typical negative results
|
2016-08-30 03:36:36 +00:00
|
|
|
|
$cache->set( $key, time(), $cache::TTL_MINUTE );
|
|
|
|
|
if ( $mode === 'stash' ) {
|
|
|
|
|
$statsd->increment( 'spamblacklist.check-stash.store' );
|
|
|
|
|
}
|
2016-06-30 21:26:36 +00:00
|
|
|
|
}
|
|
|
|
|
|
2004-12-11 09:59:06 +00:00
|
|
|
|
return $retVal;
|
|
|
|
|
}
|
2008-11-02 22:40:02 +00:00
|
|
|
|
|
2016-05-18 21:00:00 +00:00
|
|
|
|
private function isLoggingEnabled() {
|
2016-01-08 21:58:59 +00:00
|
|
|
|
global $wgSpamBlacklistEventLogging;
|
|
|
|
|
return $wgSpamBlacklistEventLogging && class_exists( 'EventLogging' );
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Diff added/removed urls and generate events for them
|
|
|
|
|
*
|
|
|
|
|
* @param string[] $oldLinks
|
|
|
|
|
* @param string[] $newLinks
|
|
|
|
|
* @param string[] $addedLinks
|
|
|
|
|
*/
|
|
|
|
|
private function logUrlChanges( $oldLinks, $newLinks, $addedLinks ) {
|
2016-05-18 21:00:00 +00:00
|
|
|
|
if ( !$this->isLoggingEnabled() ) {
|
2016-01-08 21:58:59 +00:00
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$removedLinks = array_diff( $oldLinks, $newLinks );
|
|
|
|
|
foreach ( $addedLinks as $url ) {
|
|
|
|
|
$this->logUrlChange( $url, 'insert' );
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
foreach ( $removedLinks as $url ) {
|
|
|
|
|
$this->logUrlChange( $url, 'remove' );
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Actually push the url change events post-save
|
|
|
|
|
*
|
|
|
|
|
* @param User $user
|
|
|
|
|
* @param Title $title
|
|
|
|
|
* @param Revision $rev
|
|
|
|
|
*/
|
|
|
|
|
public function doLogging( User $user, Title $title, Revision $rev ) {
|
2016-05-18 21:00:00 +00:00
|
|
|
|
if ( !$this->isLoggingEnabled() ) {
|
2016-01-08 21:58:59 +00:00
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$baseInfo = array(
|
|
|
|
|
'revId' => $rev->getId(),
|
|
|
|
|
'pageId' => $title->getArticleID(),
|
|
|
|
|
'pageNamespace' => $title->getNamespace(),
|
|
|
|
|
'userId' => $user->getId(),
|
|
|
|
|
'userText' => $user->getName(),
|
|
|
|
|
);
|
2016-05-18 21:00:00 +00:00
|
|
|
|
$changes = $this->urlChangeLog;
|
|
|
|
|
// Empty the changes queue in case this function gets called more than once
|
|
|
|
|
$this->urlChangeLog = array();
|
2016-01-08 21:58:59 +00:00
|
|
|
|
|
|
|
|
|
DeferredUpdates::addCallableUpdate( function() use ( $changes, $baseInfo ) {
|
|
|
|
|
foreach ( $changes as $change ) {
|
|
|
|
|
EventLogging::logEvent(
|
2016-05-02 15:50:59 +00:00
|
|
|
|
'ExternalLinksChange',
|
2016-06-22 10:18:11 +00:00
|
|
|
|
15716074,
|
2016-01-08 21:58:59 +00:00
|
|
|
|
$baseInfo + $change
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
} );
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
2016-05-18 21:00:00 +00:00
|
|
|
|
* Queue log data about change for a url addition or removal
|
2016-01-08 21:58:59 +00:00
|
|
|
|
*
|
|
|
|
|
* @param string $url
|
2016-05-18 21:00:00 +00:00
|
|
|
|
* @param string $action 'insert' or 'remove'
|
2016-01-08 21:58:59 +00:00
|
|
|
|
*/
|
2016-05-18 21:00:00 +00:00
|
|
|
|
private function logUrlChange( $url, $action ) {
|
2016-01-08 21:58:59 +00:00
|
|
|
|
$parsed = wfParseUrl( $url );
|
2016-06-22 10:18:11 +00:00
|
|
|
|
if ( !isset( $parsed['host'] ) ) {
|
|
|
|
|
wfDebugLog( 'SpamBlacklist', "Unable to parse $url" );
|
|
|
|
|
return;
|
|
|
|
|
}
|
2016-01-08 21:58:59 +00:00
|
|
|
|
$info = array(
|
2016-05-18 21:00:00 +00:00
|
|
|
|
'action' => $action,
|
2016-01-08 21:58:59 +00:00
|
|
|
|
'protocol' => $parsed['scheme'],
|
2016-05-18 21:00:00 +00:00
|
|
|
|
'domain' => $parsed['host'],
|
2016-06-22 10:18:11 +00:00
|
|
|
|
'path' => $parsed['path'] !== null ? $parsed['path'] : '',
|
|
|
|
|
'query' => $parsed['query'] !== null ? $parsed['query'] : '',
|
|
|
|
|
'fragment' => $parsed['fragment'] !== null ? $parsed['fragment'] : '',
|
2016-01-08 21:58:59 +00:00
|
|
|
|
);
|
|
|
|
|
|
2016-05-18 21:00:00 +00:00
|
|
|
|
$this->urlChangeLog[] = $info;
|
2016-01-08 21:58:59 +00:00
|
|
|
|
}
|
|
|
|
|
|
2008-05-13 23:31:33 +00:00
|
|
|
|
/**
|
|
|
|
|
* Look up the links currently in the article, so we can
|
|
|
|
|
* ignore them on a second run.
|
|
|
|
|
*
|
|
|
|
|
* WARNING: I can add more *of the same link* with no problem here.
|
2012-01-17 06:13:46 +00:00
|
|
|
|
* @param $title Title
|
|
|
|
|
* @return array
|
2008-05-13 23:31:33 +00:00
|
|
|
|
*/
|
2016-02-16 21:39:06 +00:00
|
|
|
|
function getCurrentLinks( Title $title ) {
|
|
|
|
|
$cache = ObjectCache::getMainWANInstance();
|
|
|
|
|
return $cache->getWithSetCallback(
|
|
|
|
|
// Key is warmed via warmCachesForFilter() from ApiStashEdit
|
|
|
|
|
$cache->makeKey( 'external-link-list', $title->getLatestRevID() ),
|
|
|
|
|
$cache::TTL_MINUTE,
|
|
|
|
|
function ( $oldValue, &$ttl, array &$setOpts ) use ( $title ) {
|
|
|
|
|
$dbr = wfGetDB( DB_SLAVE );
|
|
|
|
|
$setOpts += Database::getCacheSetOptions( $dbr );
|
|
|
|
|
|
|
|
|
|
return $dbr->selectFieldValues(
|
|
|
|
|
'externallinks',
|
|
|
|
|
'el_to',
|
|
|
|
|
array( 'el_from' => $title->getArticleID() ), // should be zero queries
|
|
|
|
|
__METHOD__
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
2016-06-30 21:26:36 +00:00
|
|
|
|
public function warmCachesForFilter( Title $title, array $entries ) {
|
2016-08-30 03:36:36 +00:00
|
|
|
|
$this->filter( $entries, $title, true /* no logging */, 'stash' );
|
2008-05-13 23:31:33 +00:00
|
|
|
|
}
|
2012-01-18 23:29:37 +00:00
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Returns the start of the regex for matches
|
|
|
|
|
*
|
|
|
|
|
* @return string
|
|
|
|
|
*/
|
|
|
|
|
public function getRegexStart() {
|
2012-07-14 17:40:05 +00:00
|
|
|
|
return '/(?:https?:)?\/\/+[a-z0-9_\-.]*(';
|
2012-01-18 23:29:37 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Returns the end of the regex for matches
|
|
|
|
|
*
|
|
|
|
|
* @param $batchSize
|
|
|
|
|
* @return string
|
|
|
|
|
*/
|
|
|
|
|
public function getRegexEnd( $batchSize ) {
|
|
|
|
|
return ')' . parent::getRegexEnd( $batchSize );
|
|
|
|
|
}
|
2013-06-18 11:20:19 +00:00
|
|
|
|
/**
|
|
|
|
|
* Logs the filter hit to Special:Log if
|
|
|
|
|
* $wgLogSpamBlacklistHits is enabled.
|
|
|
|
|
*
|
|
|
|
|
* @param Title $title
|
|
|
|
|
* @param string $url URL that the user attempted to add
|
|
|
|
|
*/
|
|
|
|
|
public function logFilterHit( $title, $url ) {
|
|
|
|
|
global $wgUser, $wgLogSpamBlacklistHits;
|
|
|
|
|
if ( $wgLogSpamBlacklistHits ) {
|
|
|
|
|
$logEntry = new ManualLogEntry( 'spamblacklist', 'hit' );
|
|
|
|
|
$logEntry->setPerformer( $wgUser );
|
|
|
|
|
$logEntry->setTarget( $title );
|
|
|
|
|
$logEntry->setParameters( array(
|
|
|
|
|
'4::url' => $url,
|
|
|
|
|
) );
|
|
|
|
|
$logid = $logEntry->insert();
|
|
|
|
|
$logEntry->publish( $logid, "rc" );
|
|
|
|
|
}
|
|
|
|
|
}
|
2012-02-03 20:15:02 +00:00
|
|
|
|
}
|