wikimedia/mediawiki-extensions-LoginNotify
1
0
Fork 0
mirror of https://gerrit.wikimedia.org/r/mediawiki/extensions/LoginNotify synced 2024-11-27 16:30:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge "LoginNotify seen subnets table"

Browse source
This commit is contained in:
jenkins-bot 2023-09-13 09:54:52 +00:00 committed by Gerrit Code Review
parent 923c328594 534e3ce4b3
commit 79f766d525
12 changed files with 828 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
extension.json
View file

@ -32,7 +32,9 @@
"BeforeCreateEchoEvent": "echo",
"EchoGetBundleRules": "echo",
"AuthManagerLoginAuthenticateAudit": "main",
"LocalUserCreated": "main"
"LocalUserCreated": "main",
"RecentChange_save": "main",
"LoadExtensionSchemaUpdates": "schema"
},
"HookHandlers": {
"main": {
@ -43,6 +45,9 @@
},
"echo": {
"class": "LoginNotify\\EchoHooks"
},
"schema": {
"class": "LoginNotify\\SchemaHooks"
}
},
"JobClasses": {
@ -51,6 +56,12 @@
"services": [
"UserFactory"
]
},
"LoginNotifyPurgeSeen": {
"class": "LoginNotify\\PurgeSeenJob",
"services": [
"LoginNotify.LoginNotify"
]
}
},
"ServiceWiringFiles": [
@ -100,6 +111,30 @@
"LoginNotifyCacheLoginIPExpiry": {
"description": "Set to false to disable caching IPs in memcache. Set to 0 to cache forever. Default 60 days.",
"value": 5184000
},
"LoginNotifySeenDatabase": {
"description": "The database to store the loginnotify_seen_net table. This can be a shared database if CentralIdLookupProvider is configured to return a unique ID for the user.",
"value": null
},
"LoginNotifySeenCluster": {
"description": "The external cluster to store the loginnotify_seen_net table in. The default is to store it in the core database.",
"value": null
},
"LoginNotifyUseCheckUser": {
"description": "Use the CheckUser cu_changes table if it is available. This is redundant with LoginNotify's own table, available with MediaWiki 1.41. Setting this to true will be deprecated in a later release. Defaults to true temporarily during WMF pilot.",
"value": true
},
"LoginNotifyUseSeenTable": {
"description": "Use the loginnotify_seen_net table. This is redundant with LoginNotifyUseCheckUser although both can be enabled during migration. Defaults to false temporarily during WMF pilot.",
"value": false
},
"LoginNotifySeenExpiry": {
"description": "The expiry time of data in the loginnotify_seen_net table, in seconds. This should be a multiple of LoginNotifyBucketSize. Default is 180 days.",
"value": 15552000
},
"LoginNotifySeenBucketSize": {
"description": "The size of a time bucket used when storing data in the loginnotify_seen_net table, in seconds. If this is changed, the data in the loginnotify_seen_net will become invalid and the table should be truncated. Setting this to a small number will require additional storage space. Setting this to a large number will cause the data expiry time to be less accurate. Default: 15 days.",
"value": 1296000
}
},
"manifest_version": 2

20
includes/Hooks.php
View file

@ -4,17 +4,22 @@
* @ingroup Extensions
*/
// phpcs:disable MediaWiki.NamingConventions.LowerCamelFunctionsName
namespace LoginNotify;
use MediaWiki\Auth\AuthenticationResponse;
use MediaWiki\Auth\Hook\AuthManagerLoginAuthenticateAuditHook;
use MediaWiki\Auth\Hook\LocalUserCreatedHook;
use MediaWiki\Hook\RecentChange_saveHook;
use MediaWiki\User\UserFactory;
use RecentChange;
use User;
class Hooks implements
AuthManagerLoginAuthenticateAuditHook,
LocalUserCreatedHook
LocalUserCreatedHook,
RecentChange_saveHook
{
/** @var UserFactory */
private $userFactory;
@ -64,7 +69,7 @@ class Hooks implements
$loginNotify = LoginNotify::getInstance();
$loginNotify->clearCounters( $user );
$loginNotify->sendSuccessNotice( $user );
$loginNotify->setCurrentAddressAsKnown( $user );
$loginNotify->recordKnownWithCookie( $user );
}
/**
@ -89,7 +94,16 @@ class Hooks implements
public function onLocalUserCreated( $user, $autocreated ) {
if ( !$autocreated ) {
$loginNotify = LoginNotify::getInstance();
$loginNotify->setCurrentAddressAsKnown( $user );
$loginNotify->recordKnownWithCookie( $user );
}
}
/**
* @param RecentChange $recentChange
*/
public function onRecentChange_save( $recentChange ) {
$loginNotify = LoginNotify::getInstance();
$user = $this->userFactory->newFromUserIdentity( $recentChange->getPerformerIdentity() );
$loginNotify->recordKnown( $user );
}
}

431
includes/LoginNotify.php
View file

@ -9,6 +9,7 @@
namespace LoginNotify;
use BagOStuff;
use CentralIdLookup;
use Exception;
use ExtensionRegistry;
use IBufferingStatsdDataFactory;
@ -27,6 +28,8 @@ use User;
use WebRequest;
use Wikimedia\Assert\Assert;
use Wikimedia\IPUtils;
use Wikimedia\Rdbms\IDatabase;
use Wikimedia\Rdbms\ILoadBalancer;
use Wikimedia\Rdbms\IMaintainableDatabase;
use Wikimedia\Rdbms\IReadableDatabase;
use Wikimedia\Rdbms\LBFactory;
@ -50,7 +53,14 @@ class LoginNotify implements LoggerAwareInterface {
'LoginNotifyExpiryNewIP',
'LoginNotifyMaxCookieRecords',
'LoginNotifySecretKey',
'LoginNotifySeenBucketSize',
'LoginNotifySeenCluster',
'LoginNotifySeenDatabase',
'LoginNotifySeenExpiry',
'LoginNotifyUseCheckUser',
'LoginNotifyUseSeenTable',
'SecretKey',
'UpdateRowsPerQuery'
];
private const COOKIE_NAME = 'loginnotify_prevlogins';
@ -79,6 +89,10 @@ class LoginNotify implements LoggerAwareInterface {
private $lbFactory;
/** @var JobQueueGroup */
private $jobQueueGroup;
/** @var CentralIdLookup */
private $centralIdLookup;
/** @var int|null */
private $fakeTime;
public static function getInstance(): self {
return MediaWikiServices::getInstance()->get( 'LoginNotify.LoginNotify' );
@ -91,6 +105,7 @@ class LoginNotify implements LoggerAwareInterface {
* @param IBufferingStatsdDataFactory $stats
* @param LBFactory $lbFactory
* @param JobQueueGroup $jobQueueGroup
* @param CentralIdLookup $centralIdLookup
*/
public function __construct(
ServiceOptions $options,
@ -98,7 +113,8 @@ class LoginNotify implements LoggerAwareInterface {
LoggerInterface $log,
IBufferingStatsdDataFactory $stats,
LBFactory $lbFactory,
JobQueueGroup $jobQueueGroup
JobQueueGroup $jobQueueGroup,
CentralIdLookup $centralIdLookup
) {
$this->config = $options;
$this->cache = $cache;
@ -113,6 +129,7 @@ class LoginNotify implements LoggerAwareInterface {
$this->stats = $stats;
$this->lbFactory = $lbFactory;
$this->jobQueueGroup = $jobQueueGroup;
$this->centralIdLookup = $centralIdLookup;
}
/**
@ -177,14 +194,35 @@ class LoginNotify implements LoggerAwareInterface {
* @return string One of USER_* constants
*/
private function isKnownSystemFast( User $user, WebRequest $request ) {
$logContext = [ 'user' => $user->getName() ];
$result = $this->userIsInCookie( $user, $request );
if ( $result !== self::USER_KNOWN ) {
$result = $this->mergeResults( $result, $this->userIsInCache( $user, $request ) );
if ( $result === self::USER_KNOWN ) {
$this->log->debug( 'Found user {user} in cookie', $logContext );
return $result;
}
$this->log->debug( 'Checking cookies and cache for {user}: {result}', [
'function' => __METHOD__,
if ( $this->config->get( 'LoginNotifyUseSeenTable' ) ) {
$id = $this->getMaybeCentralId( $user );
$hash = $this->getSeenHash( $request, $id );
$result = $this->mergeResults( $result, $this->userIsInSeenTable( $id, $hash ) );
if ( $result === self::USER_KNOWN ) {
$this->log->debug( 'Found user {user} in table', $logContext );
return $result;
}
}
// No need for caching unless CheckUser will be used
if ( $this->config->get( 'LoginNotifyUseCheckUser' ) ) {
$result = $this->mergeResults( $result, $this->userIsInCache( $user, $request ) );
if ( $result === self::USER_KNOWN ) {
$this->log->debug( 'Found user {user} in cache', $logContext );
return $result;
}
} else {
$result = self::USER_NOT_KNOWN;
}
$this->log->debug( 'Fast checks for {user}: {result}', [
'user' => $user->getName(),
'result' => $result,
] );
@ -255,6 +293,223 @@ class LoginNotify implements LoggerAwareInterface {
return self::USER_NO_INFO;
}
/**
* Check if the user is in our own table in a non-expired bucket
*
* @param int $centralUserId
* @param int|string $hash
* @return string One of USER_* constants
*/
private function userIsInSeenTable( int $centralUserId, $hash ) {
if ( !$centralUserId ) {
return self::USER_NO_INFO;
}
$dbr = $this->getSeenPrimaryDb();
$seen = $dbr->newSelectQueryBuilder()
->select( '1' )
->from( 'loginnotify_seen_net' )
->where( [
'lsn_user' => $centralUserId,
'lsn_subnet' => $hash,
'lsn_time_bucket >= ' . $dbr->addQuotes( $this->getMinBucket() )
] )
->caller( __METHOD__ )
->fetchField();
if ( $seen ) {
return self::USER_KNOWN;
} elseif ( $this->config->get( 'LoginNotifyUseCheckUser' ) ) {
// We still need to check CheckUser
return self::USER_NO_INFO;
} else {
return self::USER_NOT_KNOWN;
}
}
/**
* Check if the user is in our table in the current bucket
*
* @param int $centralUserId
* @param string $hash
* @param bool $usePrimary
* @return bool
*/
private function userIsInCurrentSeenBucket( int $centralUserId, $hash, $usePrimary = false ) {
if ( !$centralUserId ) {
return false;
}
if ( $usePrimary ) {
$dbr = $this->getSeenPrimaryDb();
} else {
$dbr = $this->getSeenReplicaDb();
}
return (bool)$dbr->newSelectQueryBuilder()
->select( '1' )
->from( 'loginnotify_seen_net' )
->where( [
'lsn_user' => $centralUserId,
'lsn_subnet' => $hash,
'lsn_time_bucket' => $this->getCurrentBucket(),
] )
->caller( __METHOD__ )
->fetchField();
}
/**
* Combine the user ID and IP prefix into a 64-bit hash. Return the hash
* as either an integer or a decimal string.
*
* @param WebRequest $request
* @param int $centralUserId
* @return int|string
* @throws Exception
*/
private function getSeenHash( WebRequest $request, int $centralUserId ) {
$ipPrefix = $this->getIPNetwork( $request->getIP() );
$hash = hash_hmac( 'sha1', "$centralUserId|$ipPrefix", $this->secret, true );
// Truncate to 64 bits
return self::packedSignedInt64ToDecimal( substr( $hash, 0, 8 ) );
}
/**
* Convert an 8-byte string to a 64-bit integer, and return it either as a
* native integer, or if PHP integers are 32 bits, as a decimal string.
*
* Signed 64-bit integers are a compact and portable way to store a 64-bit
* hash in a DBMS. On a 64-bit platform, PHP can easily generate and handle
* such integers, but on a 32-bit platform it is a bit awkward.
*
* @param string $str
* @return int|string
*/
private static function packedSignedInt64ToDecimal( $str ) {
if ( PHP_INT_SIZE >= 8 ) {
// The manual is confusing -- this does in fact return a signed number
return unpack( 'Jv', $str )['v'];
} else {
// PHP has precious few facilities for manipulating 64-bit numbers on a
// 32-bit platform. String bitwise operators are a nice hack though.
if ( ( $str[0] & "\x80" ) !== "\x00" ) {
// The number is negative. Find 2's complement and add minus sign.
$sign = '-';
$str = ~$str;
$carry = 1;
// Add with carry in big endian order
for ( $i = 7; $i >= 0 && $carry; $i-- ) {
$sum = ord( $str[$i] ) + $carry;
$carry = ( $sum & 0x100 ) >> 8;
$str[$i] = chr( $sum & 0xff );
}
} else {
$sign = '';
}
return $sign . \Wikimedia\base_convert( bin2hex( $str ), 16, 10 );
}
}
/**
* Get read a connection to the database holding the loginnotify_seen_net table.
*
* @return IReadableDatabase
*/
private function getSeenReplicaDb(): IReadableDatabase {
$dbName = $this->config->get( 'LoginNotifySeenDatabase' ) ?? false;
return $this->getSeenLoadBalancer()->getConnection( DB_REPLICA, [], $dbName );
}
/**
* Get a write connection to the database holding the loginnotify_seen_net table.
*
* @return IDatabase
*/
private function getSeenPrimaryDb(): IDatabase {
$dbName = $this->config->get( 'LoginNotifySeenDatabase' ) ?? false;
return $this->getSeenLoadBalancer()->getConnection( DB_PRIMARY, [], $dbName );
}
/**
* Is the database holding the loginnotify_seen_net table replicated to
* multiple servers?
*
* @return bool
*/
private function isSeenDbReplicated() {
return $this->getSeenLoadBalancer()->hasReplicaServers();
}
/**
* Get the LoadBalancer holding the loginnotify_seen_net table.
*
* @return ILoadBalancer
*/
private function getSeenLoadBalancer() {
$cluster = $this->config->get( 'LoginNotifySeenCluster' );
if ( $cluster ) {
return $this->lbFactory->getExternalLB( $cluster );
} else {
$dbName = $this->config->get( 'LoginNotifySeenDatabase' ) ?? false;
return $this->lbFactory->getMainLB( $dbName );
}
}
/**
* Get the lowest time bucket index which is not expired.
*
* @return int
*/
private function getMinBucket() {
$now = $this->getCurrentTime();
$expiry = $this->config->get( 'LoginNotifySeenExpiry' );
$size = $this->config->get( 'LoginNotifySeenBucketSize' );
return (int)( ( $now - $expiry ) / $size );
}
/**
* Get the current time bucket index.
*
* @return int
*/
private function getCurrentBucket() {
return (int)( $this->getCurrentTime() / $this->config->get( 'LoginNotifySeenBucketSize' ) );
}
/**
* Get the current UNIX time
*
* @return int
*/
private function getCurrentTime() {
return $this->fakeTime ?? time();
}
/**
* Set a fake time to be returned by getCurrentTime(), for testing.
*
* @param int|null $time
*/
public function setFakeTime( $time ) {
$this->fakeTime = $time;
}
/**
* If LoginNotifySeenDatabase is configured, indicating a shared table,
* get the central user ID. Otherwise, get the local user ID.
*
* If CentralAuth is not installed, $this->centralIdLookup will be a
* LocalIdLookup and the local user ID will be returned regardless. But
* using CentralIdLookup unconditionally can fail if CentralAuth is
* installed but no users are attached to it, as is the case in CI.
*
* @param User $user
* @return int
*/
private function getMaybeCentralId( User $user ) {
if ( ( $this->config->get( 'LoginNotifySeenDatabase' ) ?? false ) !== false ) {
return $this->centralIdLookup->centralIdFromLocalUser( $user );
} else {
return $user->getId();
}
}
/**
* Is the subnet of the current IP in the CheckUser data for the user.
*
@ -429,7 +684,7 @@ class LoginNotify implements LoggerAwareInterface {
private function setLoginCookie( User $user ) {
$cookie = $this->getPrevLoginCookie( $user->getRequest() );
list( , $newCookie ) = $this->checkAndGenerateCookie( $user, $cookie );
$expire = time() + $this->config->get( 'LoginNotifyCookieExpire' );
$expire = $this->getCurrentTime() + $this->config->get( 'LoginNotifyCookieExpire' );
$resp = $user->getRequest()->response();
$resp->setCookie(
self::COOKIE_NAME,
@ -444,15 +699,32 @@ class LoginNotify implements LoggerAwareInterface {
}
/**
* Give the user a cookie and cache address in memcache
* Give the user a cookie and store the address in memcached and the DB.
*
* It is expected this be called upon successful log in.
*
* @param User $user The user in question.
*/
public function setCurrentAddressAsKnown( User $user ) {
$this->cacheLoginIP( $user );
public function recordKnownWithCookie( User $user ) {
if ( !$user->isNamed() ) {
return;
}
$this->setLoginCookie( $user );
$this->recordKnown( $user );
}
/**
* Store the user's IP address in memcached and the DB
*
* @param User $user
* @return void
*/
public function recordKnown( User $user ) {
if ( !$user->isNamed() ) {
return;
}
$this->cacheLoginIP( $user );
$this->recordUserInSeenTable( $user );
$this->log->debug( 'Recording user {user} as known',
[
@ -472,13 +744,139 @@ class LoginNotify implements LoggerAwareInterface {
// It's assumed that most of the time, we'll be able to rely on
// the cookie or CheckUser data.
$expiry = $this->config->get( 'LoginNotifyCacheLoginIPExpiry' );
if ( $expiry !== false ) {
$useCU = $this->config->get( 'LoginNotifyUseCheckUser' );
if ( $useCU && $expiry !== false ) {
$ipPrefix = $this->getIPNetwork( $user->getRequest()->getIP() );
$key = $this->getKey( $user, 'prevSubnet' );
$this->cache->set( $key, $ipPrefix, $expiry );
}
}
/**
* If the user/subnet combination is not already in the database, add it.
* Also queue a job to clean up expired rows, if necessary.
*
* @param User $user
* @return void
*/
private function recordUserInSeenTable( User $user ) {
if ( !$this->config->get( 'LoginNotifyUseSeenTable' ) ) {
return;
}
$id = $this->getMaybeCentralId( $user );
if ( !$id ) {
return;
}
$request = $user->getRequest();
$hash = $this->getSeenHash( $request, $id );
// Check if the user/hash is in the replica DB
if ( $this->userIsInCurrentSeenBucket( $id, $hash ) ) {
return;
}
// Check whether purging is required
if ( !mt_rand( 0, (int)( $this->config->get( 'UpdateRowsPerQuery' ) / 4 ) ) ) {
$minId = $this->getMinExpiredId();
if ( $minId !== null ) {
$this->log->debug( 'Queueing purge job starting from lsn_id={minId}',
[ 'minId' => $minId ] );
// Deferred call to purgeSeen()
// removeDuplicates effectively limits concurrency to 1, since
// no more work will be queued until the DELETE is committed.
$job = new JobSpecification(
'LoginNotifyPurgeSeen',
[ 'minId' => $minId ],
[ 'removeDuplicates' => true ]
);
$this->jobQueueGroup->push( $job );
}
}
// Insert a row
$dbw = $this->getSeenPrimaryDb();
$isReplicated = $this->isSeenDbReplicated();
$fname = __METHOD__;
$dbw->onTransactionCommitOrIdle(
function () use ( $dbw, $id, $hash, $isReplicated, $fname ) {
// Check if the user/hash is in the primary DB, as late as
// possible before the insert. (Trying to reduce the number of
// no-op queries in the binlog)
if ( $isReplicated && $this->userIsInCurrentSeenBucket( $id, $hash, true ) ) {
return;
}
$dbw->newInsertQueryBuilder()
->insert( 'loginnotify_seen_net' )
->ignore()
->row( [
'lsn_time_bucket' => $this->getCurrentBucket(),
'lsn_user' => $id,
'lsn_subnet' => $hash
] )
->caller( $fname )
->execute();
}
);
}
/**
* Estimate the minimum lsn_id which has an expired time bucket.
*
* The primary key is approximately monotonic in time. Guess whether
* purging is required by looking at the first row ordered by
* primary key. If this check misses a row, it will be cleaned up
* when the next bucket expires.
*
* @return int|null
*/
public function getMinExpiredId() {
$minRow = $this->getSeenPrimaryDb()->newSelectQueryBuilder()
->select( [ 'lsn_id', 'lsn_time_bucket' ] )
->from( 'loginnotify_seen_net' )
->orderBy( 'lsn_id' )
->limit( 1 )
->caller( __METHOD__ )
->fetchRow();
if ( !$minRow ) {
return null;
} elseif ( $minRow->lsn_time_bucket < $this->getMinBucket() ) {
return (int)$minRow->lsn_id;
} else {
return null;
}
}
/**
* Purge rows from the loginnotify_seen_net table that are expired.
*
* @param int $minId The lsn_id to start at
* @return int|null The lsn_id to continue at, or null if no more expired
* rows are expected.
*/
public function purgeSeen( $minId ) {
$dbw = $this->getSeenPrimaryDb();
$maxId = $minId + $this->config->get( 'UpdateRowsPerQuery' );
$dbw->newDeleteQueryBuilder()
->delete( 'loginnotify_seen_net' )
->where( [
'lsn_id >= ' . $dbw->addQuotes( $minId ),
'lsn_id < ' . $dbw->addQuotes( $maxId ),
'lsn_time_bucket < ' . $dbw->addQuotes( $this->getMinBucket() )
] )
->caller( __METHOD__ )
->execute();
// If there were affected rows, tell the maintenance script to keep looking
if ( $dbw->affectedRows() ) {
return $maxId;
} else {
return null;
}
}
/**
* Merges results of various isKnownSystem*() checks
*
@ -793,10 +1191,12 @@ class LoginNotify implements LoggerAwareInterface {
$known = $this->isKnownSystemFast( $user, $user->getRequest() );
if ( $known === self::USER_KNOWN ) {
$this->recordLoginFailureFromKnownSystem( $user );
} else {
} elseif ( $this->config->get( 'LoginNotifyUseCheckUser' ) ) {
$this->createJob( DeferredChecksJob::TYPE_LOGIN_FAILED,
$user, $user->getRequest(), $known
);
} else {
$this->recordLoginFailureFromUnknownSystem( $user );
}
}
@ -827,10 +1227,15 @@ class LoginNotify implements LoggerAwareInterface {
}
$this->incrStats( 'success.total' );
$result = $this->isKnownSystemFast( $user, $user->getRequest() );
if ( $result !== self::USER_KNOWN ) {
if ( $result === self::USER_KNOWN ) {
// No need to notify
} elseif ( $this->config->get( 'LoginNotifyUseCheckUser' ) ) {
$this->createJob( DeferredChecksJob::TYPE_LOGIN_SUCCESS,
$user, $user->getRequest(), $result
);
} elseif ( $result === self::USER_NOT_KNOWN ) {
$this->incrStats( 'success.notifications' );
$this->sendNotice( $user, 'login-success' );
}
}

20
includes/PurgeSeenJob.php Normal file
View file

@ -0,0 +1,20 @@
<?php
namespace LoginNotify;
use MediaWiki\Title\Title;
class PurgeSeenJob extends \Job {
private $loginNotify;
public function __construct( Title $title, array $params, LoginNotify $loginNotify ) {
parent::__construct( 'LoginNotifyPurgeSeen', $title, $params );
$this->loginNotify = $loginNotify;
}
public function run() {
$minId = $this->getParams()['minId'];
$this->loginNotify->purgeSeen( $minId );
return true;
}
}

18
includes/SchemaHooks.php Normal file
View file

@ -0,0 +1,18 @@
<?php
namespace LoginNotify;
use DatabaseUpdater;
use MediaWiki\Installer\Hook\LoadExtensionSchemaUpdatesHook;
class SchemaHooks implements LoadExtensionSchemaUpdatesHook {
/**
* @param DatabaseUpdater $updater
*/
public function onLoadExtensionSchemaUpdates( $updater ) {
$updater->addExtensionTable(
'loginnotify_seen_net',
dirname( __DIR__ ) . "/sql/{$updater->getDB()->getType()}/tables-generated.sql"
);
}
}

3
includes/ServiceWiring.php
View file

@ -16,7 +16,8 @@ return [
LoggerFactory::getInstance( 'LoginNotify' ),
$services->getStatsdDataFactory(),
$services->getDBLoadBalancerFactory(),
$services->getJobQueueGroup()
$services->getJobQueueGroup(),
$services->getCentralIdLookup()
);
}
];

30
maintenance/purgeSeen.php Normal file
View file

@ -0,0 +1,30 @@
<?php
namespace LoginNotify\Maintenance;
use LoginNotify\LoginNotify;
$IP = getenv( 'MW_INSTALL_PATH' );
if ( $IP === false ) {
$IP = __DIR__ . '/../../..';
}
require_once "$IP/maintenance/Maintenance.php";
class PurgeSeen extends \Maintenance {
public function __construct() {
parent::__construct();
$this->addDescription( 'Purge expired user IP address information stored by LoginNotify' );
}
public function execute() {
$loginNotify = LoginNotify::getInstance();
$minId = $loginNotify->getMinExpiredId();
for ( ; $minId !== null; $this->waitForReplication() ) {
$minId = $loginNotify->purgeSeen( $minId );
}
}
}
$maintClass = PurgeSeen::class;
require_once RUN_MAINTENANCE_IF_MAIN;

14
sql/mysql/tables-generated.sql Normal file
View file

@ -0,0 +1,14 @@
-- This file is automatically generated using maintenance/generateSchemaSql.php.
-- Source: sql/tables.json
-- Do not modify this file directly.
-- See https://www.mediawiki.org/wiki/Manual:Schema_changes
CREATE TABLE /*_*/loginnotify_seen_net (
lsn_id INT UNSIGNED AUTO_INCREMENT NOT NULL,
lsn_time_bucket SMALLINT UNSIGNED NOT NULL,
lsn_user INT UNSIGNED NOT NULL,
lsn_subnet BIGINT NOT NULL,
UNIQUE INDEX loginnotify_seen_net_user (
lsn_user, lsn_subnet, lsn_time_bucket
),
PRIMARY KEY(lsn_id)
) /*$wgDBTableOptions*/;

15
sql/postgres/tables-generated.sql Normal file
View file

@ -0,0 +1,15 @@
-- This file is automatically generated using maintenance/generateSchemaSql.php.
-- Source: sql/tables.json
-- Do not modify this file directly.
-- See https://www.mediawiki.org/wiki/Manual:Schema_changes
CREATE TABLE loginnotify_seen_net (
lsn_id SERIAL NOT NULL,
lsn_time_bucket SMALLINT NOT NULL,
lsn_user INT NOT NULL,
lsn_subnet BIGINT NOT NULL,
PRIMARY KEY(lsn_id)
);
CREATE UNIQUE INDEX loginnotify_seen_net_user ON loginnotify_seen_net (
lsn_user, lsn_subnet, lsn_time_bucket
);

14
sql/sqlite/tables-generated.sql Normal file
View file

@ -0,0 +1,14 @@
-- This file is automatically generated using maintenance/generateSchemaSql.php.
-- Source: sql/tables.json
-- Do not modify this file directly.
-- See https://www.mediawiki.org/wiki/Manual:Schema_changes
CREATE TABLE /*_*/loginnotify_seen_net (
lsn_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
lsn_time_bucket SMALLINT UNSIGNED NOT NULL,
lsn_user INTEGER UNSIGNED NOT NULL,
lsn_subnet BIGINT NOT NULL
);
CREATE UNIQUE INDEX loginnotify_seen_net_user ON /*_*/loginnotify_seen_net (
lsn_user, lsn_subnet, lsn_time_bucket
);

57
sql/tables.json Normal file
View file

@ -0,0 +1,57 @@
[
{
"name": "loginnotify_seen_net",
"comment": "Summary of subnets used by local or global users",
"columns": [
{
"name": "lsn_id",
"comment": "Primary key",
"type": "integer",
"options": {
"unsigned": true,
"notnull": true,
"autoincrement": true
}
},
{
"name": "lsn_time_bucket",
"comment": "Time since epoch divided by the bucket duration, e.g. 15 days",
"type": "smallint",
"options": {
"unsigned": true,
"notnull": true
}
},
{
"name": "lsn_user",
"comment": "globaluser.gu_id or user.user_id (CentralIdLookup)",
"type": "integer",
"options": {
"unsigned": true,
"notnull": true
}
},
{
"name": "lsn_subnet",
"comment": "Truncated hash of IP address subnet",
"type": "bigint",
"options": {
"notnull": true
}
}
],
"indexes": [
{
"name": "loginnotify_seen_net_user",
"comment": "To check if the subnet is known, on login or before insertion.",
"columns": [ "lsn_user", "lsn_subnet", "lsn_time_bucket" ],
"unique": true
}
],
"pk": [ "lsn_id" ]
},
{
"name": "loginnotify_purge_claim",
"comment": ""
}
]

205
tests/phpunit/LoginNotifyTest.php
View file

@ -8,6 +8,8 @@ use MediaWiki\Request\FauxRequest;
use MediaWiki\User\UserFactory;
use Wikimedia\TestingAccessWrapper;
// phpcs:disable MediaWiki.WhiteSpace.SpaceBeforeSingleLineComment.NewLineComment
/**
* @covers \LoginNotify\LoginNotify
* @group LoginNotify
@ -22,19 +24,27 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
private $userFactory;
public function setUpLoginNotify( $configValues = [] ) {
$day = 86400;
$config = new HashConfig( $configValues + [
"LoginNotifyAttemptsKnownIP" => 15,
"LoginNotifyExpiryKnownIP" => 604800,
"LoginNotifyExpiryKnownIP" => 7 * $day,
"LoginNotifyAttemptsNewIP" => 5,
"LoginNotifyExpiryNewIP" => 1209600,
"LoginNotifyExpiryNewIP" => 14 * $day,
"LoginNotifyCheckKnownIPs" => true,
"LoginNotifyEnableOnSuccess" => true,
"LoginNotifySecretKey" => "Secret Stuff!",
"SecretKey" => "",
"LoginNotifyCookieExpire" => 15552000,
"LoginNotifyCookieExpire" => 180 * $day,
"LoginNotifyCookieDomain" => null,
"LoginNotifyMaxCookieRecords" => 6,
"LoginNotifyCacheLoginIPExpiry" => 60 * 60 * 24 * 60
"LoginNotifyCacheLoginIPExpiry" => 60 * $day,
'LoginNotifySeenCluster' => null,
"LoginNotifySeenDatabase" => null,
"LoginNotifyUseCheckUser" => false,
"LoginNotifyUseSeenTable" => true,
"LoginNotifySeenExpiry" => 180 * $day,
"LoginNotifySeenBucketSize" => 15 * $day,
"UpdateRowsPerQuery" => 100,
] );
$services = $this->getServiceContainer();
$this->inst = TestingAccessWrapper::newFromObject(
@ -44,22 +54,26 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
LoggerFactory::getInstance( 'LoginNotify' ),
$services->getStatsdDataFactory(),
$services->getDBLoadBalancerFactory(),
$services->getJobQueueGroup()
$services->getJobQueueGroup(),
new LocalIdLookup(
new HashConfig( [
'SharedDB' => false,
'SharedTables' => [],
'LocalDatabases' => []
] ),
$services->getDBLoadBalancerFactory()
)
)
);
$this->inst->setLogger( LoggerFactory::getInstance( 'LoginNotify' ) );
$this->userFactory = $this->getServiceContainer()->getUserFactory();
}
public function setUp(): void {
parent::setUp();
$this->setUpLoginNotify();
}
/**
* @dataProvider provideGetIPNetwork
*/
public function testGetIPNetwork( $ip, $expected ) {
$this->setUpLoginNotify();
$actual = $this->inst->getIPNetwork( $ip );
$this->assertSame( $expected, $actual );
}
@ -76,6 +90,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
}
public function testGetIPNetworkInvalid() {
$this->setUpLoginNotify();
$this->expectException( UnexpectedValueException::class );
$this->inst->getIPNetwork( 'localhost' );
}
@ -84,6 +99,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
* @dataProvider provideGenerateUserCookieRecord
*/
public function testGenerateUserCookieRecord( $username, $year, $salt, $expected ) {
$this->setUpLoginNotify();
$actual = $this->inst->generateUserCookieRecord( $username, $year, $salt );
$this->assertEquals( $expected, $actual );
}
@ -101,6 +117,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
* @dataProvider provideIsUserRecordGivenCookie
*/
public function testIsUserRecordGivenCookie( $cookieOptions, $expected, $desc ) {
$this->setUpLoginNotify();
$user = $this->userFactory->newFromName( 'Foo', UserFactory::RIGOR_NONE );
if ( is_string( $cookieOptions ) ) {
$cookieRecord = $cookieOptions;
@ -130,6 +147,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
}
public function testGetPrevLoginCookie() {
$this->setUpLoginNotify();
$req = new FauxRequest();
$res1 = $this->inst->getPrevLoginCookie( $req );
$this->assertSame( '', $res1, "no cookie set" );
@ -140,6 +158,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
}
public function testGetKey() {
$this->setUpLoginNotify();
$user1 = $this->userFactory->newFromName( 'Foo_bar' );
// Make sure proper normalization happens.
$user2 = $this->userFactory->newFromName( 'Foo__bar' );
@ -164,6 +183,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
}
public function testCheckAndIncKey() {
$this->setUpLoginNotify();
$key = 'global:loginnotify:new:tuwpi7e2h9pidovmaxxswk6aq327ewg';
for ( $i = 1; $i < 5; $i++ ) {
$res = $this->inst->checkAndIncKey( $key, 5, 3600 );
@ -187,6 +207,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
* @dataProvider provideClearCounters
*/
public function testClearCounters( $key ) {
$this->setUpLoginNotify();
$user = $this->userFactory->newFromName( "Fred" );
$key = $this->inst->getKey( $user, $key );
@ -216,6 +237,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
$expectedNewCookie,
$desc
) {
$this->setUpLoginNotify();
$user = $this->userFactory->newFromName( 'Foo' );
list( $actualSeenBefore, $actualNewCookie ) =
$this->inst->checkAndGenerateCookie( $user, $cookie );
@ -300,6 +322,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
* @dataProvider provideValidateCookieRecord
*/
public function testValidateCookieRecord( $cookie, $expected ) {
$this->setUpLoginNotify();
$this->assertEquals( $expected, $this->inst->validateCookieRecord( $cookie ) );
}
@ -316,6 +339,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
}
public function testUserIsInCache() {
$this->setUpLoginNotify( [ "LoginNotifyUseCheckUser" => true ] );
$u = $this->userFactory->newFromName( 'Xyzzy' );
$this->assertSame(
LoginNotify::USER_NO_INFO,
@ -337,10 +361,25 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
);
}
public function testRecordFailureKnownCache() {
$this->setupRecordFailure();
public static function provideRecordFailureKnownCacheOrTable() {
return [
[ 'cache' ],
[ 'table' ]
];
}
/**
* @dataProvider provideRecordFailureKnownCacheOrTable
* @param string $type
*/
public function testRecordFailureKnownCacheOrTable( $type ) {
$config = [
'LoginNotifyUseCheckUser' => $type === 'cache',
'LoginNotifyUseSeenTable' => $type !== 'cache'
];
$this->setupRecordFailure( $config );
$user = $this->getTestUser()->getUser();
$this->inst->setCurrentAddressAsKnown( $user );
$this->inst->recordKnown( $user );
// Record a failure, does not notify because the interval is 2
$this->inst->recordFailure( $user );
@ -400,7 +439,32 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
$this->assertNotificationCount( $user, 'login-fail-known', 0 );
}
public function testSendSuccessNotice() {
public function testRecordFailureSeenExpired() {
$this->setupRecordFailure( [
'LoginNotifyAttemptsKnownIP' => 1,
'LoginNotifyAttemptsNewIP' => 1,
] );
$user = $this->getTestUser()->getUser();
$day = 86400;
// Mark the IP as known
$this->inst->setFakeTime( 2 * $day );
$this->inst->recordKnown( $user );
// 30 days later, still known
$this->inst->setFakeTime( 32 * $day );
$this->inst->recordFailure( $user );
$this->assertNotificationCount( $user, 'login-fail-new', 0 );
$this->assertNotificationCount( $user, 'login-fail-known', 1 );
// 180 days later, rounded up to the nearest bucket, data is expired
$this->inst->setFakeTime( 210 * $day );
$this->inst->recordFailure( $user );
$this->assertNotificationCount( $user, 'login-fail-new', 1 );
$this->assertNotificationCount( $user, 'login-fail-known', 1 );
}
public function testSendSuccessNoticeCheckUser() {
$this->setupRecordFailureWithCheckUser();
$helper = new TestRecentChangesHelper;
$user = $this->getTestUser()->getUser();
@ -428,21 +492,48 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
$this->assertTrue( $emailSent );
}
private function setupRecordFailure() {
$config = [
public function testSendSuccessNoticeSeen() {
$this->setupRecordFailure();
$user = $this->getTestUser()->getUser();
$user->setEmail( 'test@test.mediawiki.org' );
$user->confirmEmail();
$user->saveSettings();
$emailSent = false;
$this->setTemporaryHook( 'EchoAbortEmailNotification',
static function () use ( &$emailSent ) {
$emailSent = true;
return false;
}
);
// Record 127.0.0.1
$this->inst->recordKnown( $user );
// Change the IP and record a success
RequestContext::getMain()->getRequest()->setIP( '127.1.0.0' );
$this->inst->sendSuccessNotice( $user );
$this->assertTrue( $emailSent );
}
private function setupRecordFailure( $config = [] ) {
$config += [
'LoginNotifyAttemptsKnownIP' => 2,
'LoginNotifyAttemptsNewIP' => 2,
];
$this->setUpLoginNotify( $config );
$this->overrideConfigValues( $config );
$this->overrideConfigValues( $config ); // for jobs
$this->tablesUsed[] = 'user';
$this->tablesUsed[] = 'echo_event';
$this->tablesUsed[] = 'echo_notification';
if ( $config['LoginNotifyUseSeenTable'] ?? true ) {
$this->tablesUsed[] = 'loginnotify_seen_net';
}
}
private function setupRecordFailureWithCheckUser() {
$this->markTestSkippedIfExtensionNotLoaded( 'CheckUser' );
$this->setupRecordFailure();
$this->setupRecordFailure( [ 'LoginNotifyUseCheckUser' => true ] );
$this->tablesUsed[] = 'comment';
$this->tablesUsed[] = 'cu_changes';
}
@ -465,4 +556,82 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase {
] )
->assertFieldValue( $expected );
}
public static function providePackedSignedInt64ToDecimal() {
return [
[ '0000000000000000', 0 ],
[ '0000000000000001', 1 ],
[ 'ffffffffffffffff', -1 ],
[ '7fffffffffffffff', '9223372036854775807' ],
[ '8000000000000000', '-9223372036854775808' ],
];
}
/**
* @dataProvider providePackedSignedInt64ToDecimal
* @param string $hexInput
* @param string|int $expected
*/
public function testPackedSignedInt64ToDecimal( $hexInput, $expected ) {
$class = TestingAccessWrapper::newFromClass( LoginNotify::class );
$input = hex2bin( $hexInput );
$result = $class->packedSignedInt64ToDecimal( $input );
$this->assertSame( (string)$expected, (string)$result );
}
public function testPurgeSeen() {
$this->setupLoginNotify( [ 'UpdateRowsPerQuery' => 1 ] );
$this->tablesUsed[] = 'user';
$this->tablesUsed[] = 'loginnotify_seen_net';
$user = $this->getTestUser()->getUser();
$day = 86400;
$this->inst->setFakeTime( 0 );
$this->inst->recordKnown( $user );
$this->inst->setFakeTime( 90 * $day );
$this->inst->recordKnown( $user );
$this->inst->setFakeTime( 210 * $day );
$this->inst->recordKnown( $user );
$this->assertSeenCount( 3 );
$this->inst->setFakeTime( 300 * $day );
$minId = $this->inst->getMinExpiredId();
$this->assertGreaterThan( 0, $minId );
$nextId = $this->inst->purgeSeen( $minId );
$this->assertSeenCount( 2 );
$this->assertGreaterThan( $minId, $nextId );
$nextId = $this->inst->purgeSeen( $nextId );
$this->assertGreaterThan( $minId, $nextId );
$this->assertSeenCount( 1 );
$nextId = $this->inst->purgeSeen( $nextId );
$this->assertNull( $nextId );
}
public function testPurgeViaJob() {
$this->setupLoginNotify( [ 'UpdateRowsPerQuery' => 1 ] );
$this->tablesUsed[] = 'user';
$this->tablesUsed[] = 'loginnotify_seen_net';
$user = $this->getTestUser()->getUser();
$this->inst->setFakeTime( 0 ); // 1970
$this->inst->recordUserInSeenTable( $user );
$this->assertSeenCount( 1 );
$this->inst->setFakeTime( null ); // real current time
$this->inst->recordUserInSeenTable( $user );
$this->assertSeenCount( 2 );
$this->runJobs();
$this->assertSeenCount( 1 );
}
private function assertSeenCount( $expected ) {
$this->newSelectQueryBuilder()
->select( 'COUNT(*)' )
->from( 'loginnotify_seen_net' )
->assertFieldValue( $expected );
}
}