mirror of
https://gerrit.wikimedia.org/r/mediawiki/extensions/AbuseFilter.git
synced 2024-11-23 21:53:35 +00:00
Merge "Add a new FilterProfiler service"
This commit is contained in:
commit
c0defc1055
|
@ -163,6 +163,7 @@
|
|||
"AbuseFilterVariableHolder": "includes/AbuseFilterVariableHolder.php",
|
||||
"MediaWiki\\Extension\\AbuseFilter\\KeywordsManager": "includes/KeywordsManager.php",
|
||||
"MediaWiki\\Extension\\AbuseFilter\\AbuseFilterServices": "includes/AbuseFilterServices.php",
|
||||
"MediaWiki\\Extension\\AbuseFilter\\FilterProfiler": "includes/FilterProfiler.php",
|
||||
"AFComputedVariable": "includes/AFComputedVariable.php",
|
||||
"AFPData": "includes/parser/AFPData.php",
|
||||
"AFPException": "includes/parser/AFPException.php",
|
||||
|
|
|
@ -166,42 +166,6 @@ class AbuseFilter {
|
|||
return $runner->checkAllFilters();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param int|string $filter
|
||||
* @internal
|
||||
*/
|
||||
public static function resetFilterProfile( $filter ) {
|
||||
$stash = MediaWikiServices::getInstance()->getMainObjectStash();
|
||||
$profileKey = self::filterProfileKey( $filter );
|
||||
|
||||
$stash->delete( $profileKey );
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve per-filter statistics.
|
||||
*
|
||||
* @param string $filter
|
||||
* @return array
|
||||
*/
|
||||
public static function getFilterProfile( $filter ) {
|
||||
$stash = MediaWikiServices::getInstance()->getMainObjectStash();
|
||||
$profile = $stash->get( self::filterProfileKey( $filter ) );
|
||||
|
||||
if ( $profile !== false ) {
|
||||
$curCount = $profile['count'];
|
||||
$curTotalTime = $profile['total-time'];
|
||||
$curTotalConds = $profile['total-cond'];
|
||||
} else {
|
||||
return [ 0, 0, 0, 0 ];
|
||||
}
|
||||
|
||||
// Return in milliseconds, rounded to 2dp
|
||||
$avgTime = round( $curTotalTime / $curCount, 2 );
|
||||
$avgCond = round( $curTotalConds / $curCount, 1 );
|
||||
|
||||
return [ $curCount, $profile['matches'], $avgTime, $avgCond ];
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function to split "<GLOBAL_FILTER_PREFIX>$index" to an array [ $id, $global ], where
|
||||
* $id is $index casted to int, and $global is a boolean: true if the filter is global,
|
||||
|
@ -705,28 +669,6 @@ class AbuseFilter {
|
|||
return $value[$group] ?? $value['default'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the memcache access key used to store per-filter profiling data.
|
||||
*
|
||||
* @param string|int $filter
|
||||
* @return string
|
||||
*/
|
||||
public static function filterProfileKey( $filter ) {
|
||||
$cache = MediaWikiServices::getInstance()->getMainWANObjectCache();
|
||||
return $cache->makeKey( 'abusefilter-profile', 'v3', $filter );
|
||||
}
|
||||
|
||||
/**
|
||||
* Memcache access key used to store overall profiling data for rule groups
|
||||
*
|
||||
* @param string $group
|
||||
* @return string
|
||||
*/
|
||||
public static function filterProfileGroupKey( $group ) {
|
||||
$cache = MediaWikiServices::getInstance()->getMainWANObjectCache();
|
||||
return $cache->makeKey( 'abusefilter-profile', 'group', $group );
|
||||
}
|
||||
|
||||
/**
|
||||
* @return User
|
||||
*/
|
||||
|
@ -1212,7 +1154,7 @@ class AbuseFilter {
|
|||
AbuseFilterHooks::purgeTagCache();
|
||||
}
|
||||
|
||||
self::resetFilterProfile( $new_id );
|
||||
AbuseFilterServices::getFilterProfiler()->resetFilterProfile( $new_id );
|
||||
return [ $new_id, $history_id ];
|
||||
}
|
||||
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
<?php
|
||||
|
||||
use MediaWiki\Block\DatabaseBlock;
|
||||
use MediaWiki\Extension\AbuseFilter\AbuseFilterServices;
|
||||
use MediaWiki\Extension\AbuseFilter\FilterProfiler;
|
||||
use MediaWiki\Extension\AbuseFilter\Hooks\AbuseFilterHookRunner;
|
||||
use MediaWiki\Extension\AbuseFilter\VariableGenerator\VariableGenerator;
|
||||
use MediaWiki\Logger\LoggerFactory;
|
||||
|
@ -66,6 +68,9 @@ class AbuseFilterRunner {
|
|||
/** @var AbuseFilterHookRunner */
|
||||
private $hookRunner;
|
||||
|
||||
/** @var FilterProfiler */
|
||||
private $filterProfiler;
|
||||
|
||||
/**
|
||||
* @param User $user The user who performed the action being filtered
|
||||
* @param Title $title The title where the action being filtered was performed
|
||||
|
@ -89,6 +94,7 @@ class AbuseFilterRunner {
|
|||
$this->group = $group;
|
||||
$this->action = $vars->getVar( 'action' )->toString();
|
||||
$this->hookRunner = AbuseFilterHookRunner::getRunner();
|
||||
$this->filterProfiler = AbuseFilterServices::getFilterProfiler();
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -454,196 +460,21 @@ class AbuseFilterRunner {
|
|||
* @param string[] $allFilters
|
||||
*/
|
||||
protected function profileExecution( array $result, array $matchedFilters, array $allFilters ) {
|
||||
$this->checkResetProfiling( $allFilters );
|
||||
$this->recordRuntimeProfilingResult(
|
||||
$this->filterProfiler->checkResetProfiling( $this->group, $allFilters );
|
||||
$this->filterProfiler->recordRuntimeProfilingResult(
|
||||
count( $allFilters ),
|
||||
$result['condCount'],
|
||||
$result['runtime']
|
||||
);
|
||||
$this->recordPerFilterProfiling( $result['profiling'] );
|
||||
$this->recordStats( $result['condCount'], $result['runtime'], (bool)$matchedFilters );
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if profiling data for all filters is lesser than the limit. If not, delete it and
|
||||
* also delete per-filter profiling for all filters. Note that we don't need to reset it for
|
||||
* disabled filters too, as their profiling data will be reset upon re-enabling anyway.
|
||||
*
|
||||
* @param array $allFilters
|
||||
*/
|
||||
protected function checkResetProfiling( array $allFilters ) {
|
||||
global $wgAbuseFilterProfileActionsCap;
|
||||
|
||||
$profileKey = AbuseFilter::filterProfileGroupKey( $this->group );
|
||||
$stash = MediaWikiServices::getInstance()->getMainObjectStash();
|
||||
|
||||
$profile = $stash->get( $profileKey );
|
||||
$total = $profile['total'] ?? 0;
|
||||
|
||||
if ( $total > $wgAbuseFilterProfileActionsCap ) {
|
||||
$stash->delete( $profileKey );
|
||||
foreach ( $allFilters as $filter ) {
|
||||
AbuseFilter::resetFilterProfile( $filter );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record per-filter profiling, for all filters
|
||||
*
|
||||
* @param array $data Profiling data, as stored in $this->profilingData
|
||||
* @phan-param array<string,array{time:float,conds:int,result:bool}> $data
|
||||
*/
|
||||
protected function recordPerFilterProfiling( array $data ) {
|
||||
global $wgAbuseFilterSlowFilterRuntimeLimit;
|
||||
|
||||
foreach ( $data as $filterName => $params ) {
|
||||
list( $filterID, $global ) = AbuseFilter::splitGlobalName( $filterName );
|
||||
if ( !$global ) {
|
||||
// @todo Maybe add a parameter to recordProfilingResult to record global filters
|
||||
// data separately (in the foreign wiki)
|
||||
$this->recordProfilingResult( $filterID, $params['time'], $params['conds'], $params['result'] );
|
||||
}
|
||||
|
||||
if ( $params['time'] > $wgAbuseFilterSlowFilterRuntimeLimit ) {
|
||||
$this->recordSlowFilter( $filterName, $params['time'], $params['conds'], $params['result'] );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record per-filter profiling data
|
||||
*
|
||||
* @param int $filter
|
||||
* @param float $time Time taken, in milliseconds
|
||||
* @param int $conds
|
||||
* @param bool $matched
|
||||
*/
|
||||
protected function recordProfilingResult( $filter, $time, $conds, $matched ) {
|
||||
// Defer updates to avoid massive (~1 second) edit time increases
|
||||
DeferredUpdates::addCallableUpdate( function () use ( $filter, $time, $conds, $matched ) {
|
||||
$stash = MediaWikiServices::getInstance()->getMainObjectStash();
|
||||
$profileKey = AbuseFilter::filterProfileKey( $filter );
|
||||
$profile = $stash->get( $profileKey );
|
||||
|
||||
if ( $profile !== false ) {
|
||||
// Number of observed executions of this filter
|
||||
$profile['count']++;
|
||||
if ( $matched ) {
|
||||
// Number of observed matches of this filter
|
||||
$profile['matches']++;
|
||||
}
|
||||
// Total time spent on this filter from all observed executions
|
||||
$profile['total-time'] += $time;
|
||||
// Total number of conditions for this filter from all executions
|
||||
$profile['total-cond'] += $conds;
|
||||
} else {
|
||||
$profile = [
|
||||
'count' => 1,
|
||||
'matches' => (int)$matched,
|
||||
'total-time' => $time,
|
||||
'total-cond' => $conds
|
||||
];
|
||||
}
|
||||
// Note: It is important that all key information be stored together in a single
|
||||
// memcache entry to avoid race conditions where competing Apache instances
|
||||
// partially overwrite the stats.
|
||||
$stash->set( $profileKey, $profile, 3600 );
|
||||
} );
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs slow filter's runtime data for later analysis
|
||||
*
|
||||
* @param string $filterId
|
||||
* @param float $runtime
|
||||
* @param int $totalConditions
|
||||
* @param bool $matched
|
||||
*/
|
||||
protected function recordSlowFilter( $filterId, $runtime, $totalConditions, $matched ) {
|
||||
$logger = LoggerFactory::getInstance( 'AbuseFilter' );
|
||||
$logger->info(
|
||||
'Edit filter {filter_id} on {wiki} is taking longer than expected',
|
||||
[
|
||||
'wiki' => WikiMap::getCurrentWikiDbDomain()->getId(),
|
||||
'filter_id' => $filterId,
|
||||
'title' => $this->title->getPrefixedText(),
|
||||
'runtime' => $runtime,
|
||||
'matched' => $matched,
|
||||
'total_conditions' => $totalConditions
|
||||
]
|
||||
$this->filterProfiler->recordPerFilterProfiling( $this->title, $result['profiling'] );
|
||||
$this->filterProfiler->recordStats(
|
||||
$this->group,
|
||||
$result['condCount'],
|
||||
$result['runtime'],
|
||||
(bool)$matchedFilters
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update global statistics
|
||||
*
|
||||
* @param int $condsUsed The amount of used conditions
|
||||
* @param float $totalTime Time taken, in milliseconds
|
||||
* @param bool $anyMatch Whether at least one filter matched the action
|
||||
*/
|
||||
protected function recordStats( $condsUsed, $totalTime, $anyMatch ) {
|
||||
$profileKey = AbuseFilter::filterProfileGroupKey( $this->group );
|
||||
$stash = MediaWikiServices::getInstance()->getMainObjectStash();
|
||||
|
||||
// Note: All related data is stored in a single memcache entry and updated via merge()
|
||||
// to avoid race conditions where partial updates on competing instances corrupt the data.
|
||||
$stash->merge(
|
||||
$profileKey,
|
||||
function ( $cache, $key, $profile ) use ( $condsUsed, $totalTime, $anyMatch ) {
|
||||
global $wgAbuseFilterConditionLimit;
|
||||
|
||||
if ( $profile === false ) {
|
||||
$profile = [
|
||||
// Total number of actions observed
|
||||
'total' => 0,
|
||||
// Number of actions ending by exceeding condition limit
|
||||
'overflow' => 0,
|
||||
// Total time of execution of all observed actions
|
||||
'total-time' => 0,
|
||||
// Total number of conditions from all observed actions
|
||||
'total-cond' => 0,
|
||||
// Total number of filters matched
|
||||
'matches' => 0
|
||||
];
|
||||
}
|
||||
|
||||
$profile['total']++;
|
||||
$profile['total-time'] += $totalTime;
|
||||
$profile['total-cond'] += $condsUsed;
|
||||
|
||||
// Increment overflow counter, if our condition limit overflowed
|
||||
if ( $condsUsed > $wgAbuseFilterConditionLimit ) {
|
||||
$profile['overflow']++;
|
||||
}
|
||||
|
||||
// Increment counter by 1 if there was at least one match
|
||||
if ( $anyMatch ) {
|
||||
$profile['matches']++;
|
||||
}
|
||||
|
||||
return $profile;
|
||||
},
|
||||
AbuseFilter::$statsStoragePeriod
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Record runtime profiling data for all filters together
|
||||
*
|
||||
* @param int $totalFilters
|
||||
* @param int $totalConditions
|
||||
* @param float $runtime
|
||||
*/
|
||||
protected function recordRuntimeProfilingResult( $totalFilters, $totalConditions, $runtime ) {
|
||||
$keyPrefix = 'abusefilter.runtime-profile.' . WikiMap::getCurrentWikiDbDomain()->getId() . '.';
|
||||
|
||||
$statsd = MediaWikiServices::getInstance()->getStatsdDataFactory();
|
||||
$statsd->timing( $keyPrefix . 'runtime', $runtime );
|
||||
$statsd->timing( $keyPrefix . 'total_filters', $totalFilters );
|
||||
$statsd->timing( $keyPrefix . 'total_conditions', $totalConditions );
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a set of actions.
|
||||
*
|
||||
|
@ -1374,10 +1205,9 @@ class AbuseFilterRunner {
|
|||
* @param string[] $filters The filters to check
|
||||
*/
|
||||
protected function checkEmergencyDisable( array $filters ) {
|
||||
$stash = MediaWikiServices::getInstance()->getMainObjectStash();
|
||||
// @ToDo this is an amount between 1 and AbuseFilterProfileActionsCap, which means that the
|
||||
// reliability of this number may strongly vary. We should instead use a fixed one.
|
||||
$groupProfile = $stash->get( AbuseFilter::filterProfileGroupKey( $this->group ) );
|
||||
$groupProfile = $this->filterProfiler->getGroupProfile( $this->group );
|
||||
$totalActions = $groupProfile['total'];
|
||||
|
||||
foreach ( $filters as $filter ) {
|
||||
|
@ -1385,8 +1215,8 @@ class AbuseFilterRunner {
|
|||
$hitCountLimit = AbuseFilter::getEmergencyValue( 'count', $this->group );
|
||||
$maxAge = AbuseFilter::getEmergencyValue( 'age', $this->group );
|
||||
|
||||
$filterProfile = $stash->get( AbuseFilter::filterProfileKey( $filter ) );
|
||||
$matchCount = $filterProfile['matches'] ?? 1;
|
||||
$filterProfile = $this->filterProfiler->getFilterProfile( $filter );
|
||||
$matchCount = ( $filterProfile['matches'] ?? 0 ) + 1;
|
||||
|
||||
// Figure out if the filter is subject to being throttled.
|
||||
$filterAge = (int)wfTimestamp( TS_UNIX, AbuseFilter::getFilter( $filter )->af_timestamp );
|
||||
|
|
|
@ -12,4 +12,12 @@ class AbuseFilterServices {
|
|||
public static function getKeywordsManager() : KeywordsManager {
|
||||
return MediaWikiServices::getInstance()->getService( KeywordsManager::SERVICE_NAME );
|
||||
}
|
||||
|
||||
/**
|
||||
* Conveniency wrapper for strong typing
|
||||
* @return FilterProfiler
|
||||
*/
|
||||
public static function getFilterProfiler() : FilterProfiler {
|
||||
return MediaWikiServices::getInstance()->getService( FilterProfiler::SERVICE_NAME );
|
||||
}
|
||||
}
|
||||
|
|
314
includes/FilterProfiler.php
Normal file
314
includes/FilterProfiler.php
Normal file
|
@ -0,0 +1,314 @@
|
|||
<?php
|
||||
|
||||
namespace MediaWiki\Extension\AbuseFilter;
|
||||
|
||||
use AbuseFilter;
|
||||
use BagOStuff;
|
||||
use DeferredUpdates;
|
||||
use IBufferingStatsdDataFactory;
|
||||
use MediaWiki\Config\ServiceOptions;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Title;
|
||||
|
||||
/**
|
||||
* This class is used to create, store, and retrieve profiling information for single filters and
|
||||
* groups of filters.
|
||||
* @internal
|
||||
*/
|
||||
class FilterProfiler {
|
||||
public const SERVICE_NAME = 'AbuseFilterFilterProfiler';
|
||||
|
||||
public const CONSTRUCTOR_OPTIONS = [
|
||||
'AbuseFilterProfileActionsCap',
|
||||
'AbuseFilterConditionLimit',
|
||||
'AbuseFilterSlowFilterRuntimeLimit',
|
||||
];
|
||||
|
||||
/**
|
||||
* @var int How long to keep profiling data in cache (in seconds)
|
||||
*/
|
||||
private const STATS_STORAGE_PERIOD = 86400;
|
||||
|
||||
/** @var BagOStuff */
|
||||
private $objectStash;
|
||||
|
||||
/** @var ServiceOptions */
|
||||
private $options;
|
||||
|
||||
/** @var string */
|
||||
private $localWikiID;
|
||||
|
||||
/** @var IBufferingStatsdDataFactory */
|
||||
private $statsd;
|
||||
|
||||
/** @var LoggerInterface */
|
||||
private $logger;
|
||||
|
||||
/**
|
||||
* @param BagOStuff $objectStash
|
||||
* @param ServiceOptions $options
|
||||
* @param string $localWikiID
|
||||
* @param IBufferingStatsdDataFactory $statsd
|
||||
* @param LoggerInterface $logger
|
||||
*/
|
||||
public function __construct(
|
||||
BagOStuff $objectStash,
|
||||
ServiceOptions $options,
|
||||
string $localWikiID,
|
||||
IBufferingStatsdDataFactory $statsd,
|
||||
LoggerInterface $logger
|
||||
) {
|
||||
$this->objectStash = $objectStash;
|
||||
$options->assertRequiredOptions( self::CONSTRUCTOR_OPTIONS );
|
||||
$this->options = $options;
|
||||
$this->localWikiID = $localWikiID;
|
||||
$this->statsd = $statsd;
|
||||
$this->logger = $logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param int|string $filter
|
||||
*/
|
||||
public function resetFilterProfile( $filter ) : void {
|
||||
$profileKey = $this->filterProfileKey( $filter );
|
||||
$this->objectStash->delete( $profileKey );
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve per-filter statistics.
|
||||
*
|
||||
* @param string $filter
|
||||
* @return array
|
||||
*/
|
||||
public function getFilterProfile( string $filter ) : array {
|
||||
$profile = $this->objectStash->get( $this->filterProfileKey( $filter ) );
|
||||
|
||||
if ( $profile !== false ) {
|
||||
$curCount = $profile['count'];
|
||||
$curTotalTime = $profile['total-time'];
|
||||
$curTotalConds = $profile['total-cond'];
|
||||
} else {
|
||||
return [ 0, 0, 0, 0 ];
|
||||
}
|
||||
|
||||
// Return in milliseconds, rounded to 2dp
|
||||
$avgTime = round( $curTotalTime / $curCount, 2 );
|
||||
$avgCond = round( $curTotalConds / $curCount, 1 );
|
||||
|
||||
return [ $curCount, $profile['matches'], $avgTime, $avgCond ];
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve per-group statistics
|
||||
* @param string $group
|
||||
* @return array|false
|
||||
* @phan-return array{total:int,overflow:int,matches:int}|false
|
||||
*/
|
||||
public function getGroupProfile( string $group ) {
|
||||
return $this->objectStash->get( $this->filterProfileGroupKey( $group ) );
|
||||
}
|
||||
|
||||
/**
|
||||
* Record per-filter profiling data
|
||||
*
|
||||
* @param int $filter
|
||||
* @param float $time Time taken, in milliseconds
|
||||
* @param int $conds
|
||||
* @param bool $matched
|
||||
*/
|
||||
private function recordProfilingResult( int $filter, float $time, int $conds, bool $matched ) : void {
|
||||
// Defer updates to avoid massive (~1 second) edit time increases
|
||||
DeferredUpdates::addCallableUpdate( function () use ( $filter, $time, $conds, $matched ) {
|
||||
$profileKey = $this->filterProfileKey( $filter );
|
||||
$profile = $this->objectStash->get( $profileKey );
|
||||
|
||||
if ( $profile !== false ) {
|
||||
// Number of observed executions of this filter
|
||||
$profile['count']++;
|
||||
if ( $matched ) {
|
||||
// Number of observed matches of this filter
|
||||
$profile['matches']++;
|
||||
}
|
||||
// Total time spent on this filter from all observed executions
|
||||
$profile['total-time'] += $time;
|
||||
// Total number of conditions for this filter from all executions
|
||||
$profile['total-cond'] += $conds;
|
||||
} else {
|
||||
$profile = [
|
||||
'count' => 1,
|
||||
'matches' => (int)$matched,
|
||||
'total-time' => $time,
|
||||
'total-cond' => $conds
|
||||
];
|
||||
}
|
||||
// Note: It is important that all key information be stored together in a single
|
||||
// memcache entry to avoid race conditions where competing Apache instances
|
||||
// partially overwrite the stats.
|
||||
$this->objectStash->set( $profileKey, $profile, 3600 );
|
||||
} );
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if profiling data for all filters is lesser than the limit. If not, delete it and
|
||||
* also delete per-filter profiling for all filters. Note that we don't need to reset it for
|
||||
* disabled filters too, as their profiling data will be reset upon re-enabling anyway.
|
||||
*
|
||||
* @param string $group
|
||||
* @param array $allFilters
|
||||
*/
|
||||
public function checkResetProfiling( string $group, array $allFilters ) : void {
|
||||
$profileKey = $this->filterProfileGroupKey( $group );
|
||||
|
||||
$profile = $this->objectStash->get( $profileKey );
|
||||
$total = $profile['total'] ?? 0;
|
||||
|
||||
if ( $total > $this->options->get( 'AbuseFilterProfileActionsCap' ) ) {
|
||||
$this->objectStash->delete( $profileKey );
|
||||
foreach ( $allFilters as $filter ) {
|
||||
$this->resetFilterProfile( $filter );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update global statistics
|
||||
*
|
||||
* @param string $group
|
||||
* @param int $condsUsed The amount of used conditions
|
||||
* @param float $totalTime Time taken, in milliseconds
|
||||
* @param bool $anyMatch Whether at least one filter matched the action
|
||||
*/
|
||||
public function recordStats( string $group, int $condsUsed, float $totalTime, bool $anyMatch ) : void {
|
||||
$profileKey = $this->filterProfileGroupKey( $group );
|
||||
|
||||
// Note: All related data is stored in a single memcache entry and updated via merge()
|
||||
// to avoid race conditions where partial updates on competing instances corrupt the data.
|
||||
$this->objectStash->merge(
|
||||
$profileKey,
|
||||
function ( $cache, $key, $profile ) use ( $condsUsed, $totalTime, $anyMatch ) {
|
||||
if ( $profile === false ) {
|
||||
$profile = [
|
||||
// Total number of actions observed
|
||||
'total' => 0,
|
||||
// Number of actions ending by exceeding condition limit
|
||||
'overflow' => 0,
|
||||
// Total time of execution of all observed actions
|
||||
'total-time' => 0,
|
||||
// Total number of conditions from all observed actions
|
||||
'total-cond' => 0,
|
||||
// Total number of filters matched
|
||||
'matches' => 0
|
||||
];
|
||||
}
|
||||
|
||||
$profile['total']++;
|
||||
$profile['total-time'] += $totalTime;
|
||||
$profile['total-cond'] += $condsUsed;
|
||||
|
||||
// Increment overflow counter, if our condition limit overflowed
|
||||
if ( $condsUsed > $this->options->get( 'AbuseFilterConditionLimit' ) ) {
|
||||
$profile['overflow']++;
|
||||
}
|
||||
|
||||
// Increment counter by 1 if there was at least one match
|
||||
if ( $anyMatch ) {
|
||||
$profile['matches']++;
|
||||
}
|
||||
|
||||
return $profile;
|
||||
},
|
||||
self::STATS_STORAGE_PERIOD
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Record runtime profiling data for all filters together
|
||||
*
|
||||
* @param int $totalFilters
|
||||
* @param int $totalConditions
|
||||
* @param float $runtime
|
||||
*/
|
||||
public function recordRuntimeProfilingResult( int $totalFilters, int $totalConditions, float $runtime ) : void {
|
||||
$keyPrefix = 'abusefilter.runtime-profile.' . $this->localWikiID . '.';
|
||||
|
||||
$this->statsd->timing( $keyPrefix . 'runtime', $runtime );
|
||||
$this->statsd->timing( $keyPrefix . 'total_filters', $totalFilters );
|
||||
$this->statsd->timing( $keyPrefix . 'total_conditions', $totalConditions );
|
||||
}
|
||||
|
||||
/**
|
||||
* Record per-filter profiling, for all filters
|
||||
*
|
||||
* @param Title $title
|
||||
* @param array $data Profiling data, as stored in $this->profilingData
|
||||
* @phan-param array<string,array{time:float,conds:int,result:bool}> $data
|
||||
*/
|
||||
public function recordPerFilterProfiling( Title $title, array $data ) : void {
|
||||
foreach ( $data as $filterName => $params ) {
|
||||
list( $filterID, $global ) = AbuseFilter::splitGlobalName( $filterName );
|
||||
if ( !$global ) {
|
||||
// @todo Maybe add a parameter to recordProfilingResult to record global filters
|
||||
// data separately (in the foreign wiki)
|
||||
$this->recordProfilingResult(
|
||||
$filterID,
|
||||
$params['time'],
|
||||
$params['conds'],
|
||||
$params['result']
|
||||
);
|
||||
}
|
||||
|
||||
if ( $params['time'] > $this->options->get( 'AbuseFilterSlowFilterRuntimeLimit' ) ) {
|
||||
$this->recordSlowFilter( $title, $filterName, $params['time'], $params['conds'], $params['result'] );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs slow filter's runtime data for later analysis
|
||||
*
|
||||
* @param Title $title
|
||||
* @param string $filterId
|
||||
* @param float $runtime
|
||||
* @param int $totalConditions
|
||||
* @param bool $matched
|
||||
*/
|
||||
private function recordSlowFilter(
|
||||
Title $title,
|
||||
string $filterId,
|
||||
float $runtime,
|
||||
int $totalConditions,
|
||||
bool $matched
|
||||
) : void {
|
||||
$this->logger->info(
|
||||
'Edit filter {filter_id} on {wiki} is taking longer than expected',
|
||||
[
|
||||
'wiki' => $this->localWikiID,
|
||||
'filter_id' => $filterId,
|
||||
'title' => $title->getPrefixedText(),
|
||||
'runtime' => $runtime,
|
||||
'matched' => $matched,
|
||||
'total_conditions' => $totalConditions
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the memcache access key used to store per-filter profiling data.
|
||||
*
|
||||
* @param string|int $filter
|
||||
* @return string
|
||||
*/
|
||||
private function filterProfileKey( $filter ) : string {
|
||||
return $this->objectStash->makeKey( 'abusefilter-profile', 'v3', $filter );
|
||||
}
|
||||
|
||||
/**
|
||||
* Memcache access key used to store overall profiling data for rule groups
|
||||
*
|
||||
* @param string $group
|
||||
* @return string
|
||||
*/
|
||||
private function filterProfileGroupKey( string $group ) : string {
|
||||
return $this->objectStash->makeKey( 'abusefilter-profile', 'group', $group );
|
||||
}
|
||||
}
|
|
@ -1,7 +1,10 @@
|
|||
<?php
|
||||
|
||||
use MediaWiki\Config\ServiceOptions;
|
||||
use MediaWiki\Extension\AbuseFilter\FilterProfiler;
|
||||
use MediaWiki\Extension\AbuseFilter\Hooks\AbuseFilterHookRunner;
|
||||
use MediaWiki\Extension\AbuseFilter\KeywordsManager;
|
||||
use MediaWiki\Logger\LoggerFactory;
|
||||
use MediaWiki\MediaWikiServices;
|
||||
|
||||
return [
|
||||
|
@ -10,4 +13,16 @@ return [
|
|||
new AbuseFilterHookRunner( $services->getHookContainer() )
|
||||
);
|
||||
},
|
||||
FilterProfiler::SERVICE_NAME => function ( MediaWikiServices $services ): FilterProfiler {
|
||||
return new FilterProfiler(
|
||||
$services->getMainObjectStash(),
|
||||
new ServiceOptions(
|
||||
FilterProfiler::CONSTRUCTOR_OPTIONS,
|
||||
$services->getMainConfig()
|
||||
),
|
||||
WikiMap::getCurrentWikiDbDomain()->getId(),
|
||||
$services->getStatsdDataFactory(),
|
||||
LoggerFactory::getInstance( 'AbuseFilter' )
|
||||
);
|
||||
},
|
||||
];
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
<?php
|
||||
|
||||
use MediaWiki\Extension\AbuseFilter\AbuseFilterServices;
|
||||
use MediaWiki\MediaWikiServices;
|
||||
|
||||
class AbuseFilterViewEdit extends AbuseFilterView {
|
||||
|
@ -308,7 +309,7 @@ class AbuseFilterViewEdit extends AbuseFilterView {
|
|||
if ( $filter !== 'new' && $row->af_enabled ) {
|
||||
// Statistics
|
||||
list( $totalCount, $matchesCount, $avgTime, $avgCond ) =
|
||||
AbuseFilter::getFilterProfile( $filter );
|
||||
AbuseFilterServices::getFilterProfiler()->getFilterProfile( $filter );
|
||||
|
||||
if ( $totalCount > 0 ) {
|
||||
$matchesPercent = round( 100 * $matchesCount / $totalCount, 2 );
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
<?php
|
||||
|
||||
use MediaWiki\MediaWikiServices;
|
||||
use MediaWiki\Extension\AbuseFilter\AbuseFilterServices;
|
||||
|
||||
/**
|
||||
* The default view used in Special:AbuseFilter
|
||||
|
@ -263,13 +263,13 @@ class AbuseFilterViewList extends AbuseFilterView {
|
|||
* Generates a summary of filter activity using the internal statistics.
|
||||
*/
|
||||
public function showStatus() {
|
||||
$stash = MediaWikiServices::getInstance()->getMainObjectStash();
|
||||
$filterProfiler = AbuseFilterServices::getFilterProfiler();
|
||||
|
||||
$totalCount = 0;
|
||||
$matchCount = 0;
|
||||
$overflowCount = 0;
|
||||
foreach ( $this->getConfig()->get( 'AbuseFilterValidGroups' ) as $group ) {
|
||||
$profile = $stash->get( AbuseFilter::filterProfileGroupKey( $group ) );
|
||||
$profile = $filterProfiler->getGroupProfile( $group );
|
||||
if ( $profile !== false ) {
|
||||
$totalCount += $profile[ 'total' ];
|
||||
$overflowCount += $profile[ 'overflow' ];
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
<?php
|
||||
|
||||
use MediaWiki\Block\DatabaseBlock;
|
||||
use MediaWiki\Extension\AbuseFilter\AbuseFilterServices;
|
||||
use MediaWiki\MediaWikiServices;
|
||||
use MediaWiki\Storage\PageEditStash;
|
||||
use PHPUnit\Framework\MockObject\MockObject;
|
||||
|
@ -1033,7 +1034,7 @@ class AbuseFilterConsequencesTest extends MediaWikiTestCase {
|
|||
* @param int[] $createIds IDs of the filters to create
|
||||
* @param array $actionParams Details of the action we need to execute to trigger filters
|
||||
* @covers AbuseFilterRunner::checkFilter
|
||||
* @covers AbuseFilterRunner::recordSlowFilter
|
||||
* @covers \MediaWiki\Extension\AbuseFilter\FilterProfiler::recordSlowFilter
|
||||
* @dataProvider provideFiltersNoConsequences
|
||||
*/
|
||||
public function testTimeLimit( $createIds, $actionParams ) {
|
||||
|
@ -1782,11 +1783,11 @@ class AbuseFilterConsequencesTest extends MediaWikiTestCase {
|
|||
* @param array $actionParams Details of the action we need to execute to trigger filters
|
||||
* @param array $expectedGlobal Expected global stats
|
||||
* @param array $expectedPerFilter Expected stats for every created filter
|
||||
* @covers AbuseFilter::filterProfileKey
|
||||
* @covers AbuseFilter::filterProfileGroupKey
|
||||
* @covers AbuseFilter::getFilterProfile
|
||||
* @covers \MediaWiki\Extension\AbuseFilter\FilterProfiler::filterProfileKey
|
||||
* @covers \MediaWiki\Extension\AbuseFilter\FilterProfiler::filterProfileGroupKey
|
||||
* @covers \MediaWiki\Extension\AbuseFilter\FilterProfiler::getFilterProfile
|
||||
* @covers AbuseFilterRunner::checkAllFilters
|
||||
* @covers AbuseFilterRunner::recordStats
|
||||
* @covers \MediaWiki\Extension\AbuseFilter\FilterProfiler::recordStats
|
||||
* @dataProvider provideProfilingFilters
|
||||
*/
|
||||
public function testProfiling( $createIds, $actionParams, $expectedGlobal, $expectedPerFilter ) {
|
||||
|
@ -1806,9 +1807,9 @@ class AbuseFilterConsequencesTest extends MediaWikiTestCase {
|
|||
$this->doAction( $actionParams );
|
||||
MWTimestamp::setFakeTime( false );
|
||||
|
||||
$stash = MediaWikiServices::getInstance()->getMainObjectStash();
|
||||
$profiler = AbuseFilterServices::getFilterProfiler();
|
||||
// Global stats shown on the top of Special:AbuseFilter
|
||||
$globalStats = $stash->get( AbuseFilter::filterProfileGroupKey( 'default' ) );
|
||||
$globalStats = $profiler->getGroupProfile( 'default' );
|
||||
$actualGlobalStats = [
|
||||
'totalMatches' => $globalStats['matches'],
|
||||
'totalActions' => $globalStats['total'],
|
||||
|
@ -1822,7 +1823,7 @@ class AbuseFilterConsequencesTest extends MediaWikiTestCase {
|
|||
|
||||
// Per-filter stats shown on the top of Special:AbuseFilter/xxx
|
||||
foreach ( $createIds as $id ) {
|
||||
list( $totalActions, $matches, , $conds ) = AbuseFilter::getFilterProfile( $id );
|
||||
list( $totalActions, $matches, , $conds ) = $profiler->getFilterProfile( $id );
|
||||
$actualStats = [
|
||||
'matches' => $matches,
|
||||
'actions' => $totalActions,
|
||||
|
|
Loading…
Reference in a new issue