diff --git a/extension.json b/extension.json index ea110d1..d76ae44 100644 --- a/extension.json +++ b/extension.json @@ -32,7 +32,9 @@ "BeforeCreateEchoEvent": "echo", "EchoGetBundleRules": "echo", "AuthManagerLoginAuthenticateAudit": "main", - "LocalUserCreated": "main" + "LocalUserCreated": "main", + "RecentChange_save": "main", + "LoadExtensionSchemaUpdates": "schema" }, "HookHandlers": { "main": { @@ -43,6 +45,9 @@ }, "echo": { "class": "LoginNotify\\EchoHooks" + }, + "schema": { + "class": "LoginNotify\\SchemaHooks" } }, "JobClasses": { @@ -51,6 +56,12 @@ "services": [ "UserFactory" ] + }, + "LoginNotifyPurgeSeen": { + "class": "LoginNotify\\PurgeSeenJob", + "services": [ + "LoginNotify.LoginNotify" + ] } }, "ServiceWiringFiles": [ @@ -100,6 +111,30 @@ "LoginNotifyCacheLoginIPExpiry": { "description": "Set to false to disable caching IPs in memcache. Set to 0 to cache forever. Default 60 days.", "value": 5184000 + }, + "LoginNotifySeenDatabase": { + "description": "The database to store the loginnotify_seen_net table. This can be a shared database if CentralIdLookupProvider is configured to return a unique ID for the user.", + "value": null + }, + "LoginNotifySeenCluster": { + "description": "The external cluster to store the loginnotify_seen_net table in. The default is to store it in the core database.", + "value": null + }, + "LoginNotifyUseCheckUser": { + "description": "Use the CheckUser cu_changes table if it is available. This is redundant with LoginNotify's own table, available with MediaWiki 1.41. Setting this to true will be deprecated in a later release. Defaults to true temporarily during WMF pilot.", + "value": true + }, + "LoginNotifyUseSeenTable": { + "description": "Use the loginnotify_seen_net table. This is redundant with LoginNotifyUseCheckUser although both can be enabled during migration. Defaults to false temporarily during WMF pilot.", + "value": false + }, + "LoginNotifySeenExpiry": { + "description": "The expiry time of data in the loginnotify_seen_net table, in seconds. This should be a multiple of LoginNotifyBucketSize. Default is 180 days.", + "value": 15552000 + }, + "LoginNotifySeenBucketSize": { + "description": "The size of a time bucket used when storing data in the loginnotify_seen_net table, in seconds. If this is changed, the data in the loginnotify_seen_net will become invalid and the table should be truncated. Setting this to a small number will require additional storage space. Setting this to a large number will cause the data expiry time to be less accurate. Default: 15 days.", + "value": 1296000 } }, "manifest_version": 2 diff --git a/includes/Hooks.php b/includes/Hooks.php index 7a4b01c..05559a0 100644 --- a/includes/Hooks.php +++ b/includes/Hooks.php @@ -4,17 +4,22 @@ * @ingroup Extensions */ +// phpcs:disable MediaWiki.NamingConventions.LowerCamelFunctionsName + namespace LoginNotify; use MediaWiki\Auth\AuthenticationResponse; use MediaWiki\Auth\Hook\AuthManagerLoginAuthenticateAuditHook; use MediaWiki\Auth\Hook\LocalUserCreatedHook; +use MediaWiki\Hook\RecentChange_saveHook; use MediaWiki\User\UserFactory; +use RecentChange; use User; class Hooks implements AuthManagerLoginAuthenticateAuditHook, - LocalUserCreatedHook + LocalUserCreatedHook, + RecentChange_saveHook { /** @var UserFactory */ private $userFactory; @@ -64,7 +69,7 @@ class Hooks implements $loginNotify = LoginNotify::getInstance(); $loginNotify->clearCounters( $user ); $loginNotify->sendSuccessNotice( $user ); - $loginNotify->setCurrentAddressAsKnown( $user ); + $loginNotify->recordKnownWithCookie( $user ); } /** @@ -89,7 +94,16 @@ class Hooks implements public function onLocalUserCreated( $user, $autocreated ) { if ( !$autocreated ) { $loginNotify = LoginNotify::getInstance(); - $loginNotify->setCurrentAddressAsKnown( $user ); + $loginNotify->recordKnownWithCookie( $user ); } } + + /** + * @param RecentChange $recentChange + */ + public function onRecentChange_save( $recentChange ) { + $loginNotify = LoginNotify::getInstance(); + $user = $this->userFactory->newFromUserIdentity( $recentChange->getPerformerIdentity() ); + $loginNotify->recordKnown( $user ); + } } diff --git a/includes/LoginNotify.php b/includes/LoginNotify.php index 9354127..c7bc6fe 100644 --- a/includes/LoginNotify.php +++ b/includes/LoginNotify.php @@ -9,6 +9,7 @@ namespace LoginNotify; use BagOStuff; +use CentralIdLookup; use Exception; use ExtensionRegistry; use IBufferingStatsdDataFactory; @@ -27,6 +28,8 @@ use User; use WebRequest; use Wikimedia\Assert\Assert; use Wikimedia\IPUtils; +use Wikimedia\Rdbms\IDatabase; +use Wikimedia\Rdbms\ILoadBalancer; use Wikimedia\Rdbms\IMaintainableDatabase; use Wikimedia\Rdbms\IReadableDatabase; use Wikimedia\Rdbms\LBFactory; @@ -50,7 +53,14 @@ class LoginNotify implements LoggerAwareInterface { 'LoginNotifyExpiryNewIP', 'LoginNotifyMaxCookieRecords', 'LoginNotifySecretKey', + 'LoginNotifySeenBucketSize', + 'LoginNotifySeenCluster', + 'LoginNotifySeenDatabase', + 'LoginNotifySeenExpiry', + 'LoginNotifyUseCheckUser', + 'LoginNotifyUseSeenTable', 'SecretKey', + 'UpdateRowsPerQuery' ]; private const COOKIE_NAME = 'loginnotify_prevlogins'; @@ -79,6 +89,10 @@ class LoginNotify implements LoggerAwareInterface { private $lbFactory; /** @var JobQueueGroup */ private $jobQueueGroup; + /** @var CentralIdLookup */ + private $centralIdLookup; + /** @var int|null */ + private $fakeTime; public static function getInstance(): self { return MediaWikiServices::getInstance()->get( 'LoginNotify.LoginNotify' ); @@ -91,6 +105,7 @@ class LoginNotify implements LoggerAwareInterface { * @param IBufferingStatsdDataFactory $stats * @param LBFactory $lbFactory * @param JobQueueGroup $jobQueueGroup + * @param CentralIdLookup $centralIdLookup */ public function __construct( ServiceOptions $options, @@ -98,7 +113,8 @@ class LoginNotify implements LoggerAwareInterface { LoggerInterface $log, IBufferingStatsdDataFactory $stats, LBFactory $lbFactory, - JobQueueGroup $jobQueueGroup + JobQueueGroup $jobQueueGroup, + CentralIdLookup $centralIdLookup ) { $this->config = $options; $this->cache = $cache; @@ -113,6 +129,7 @@ class LoginNotify implements LoggerAwareInterface { $this->stats = $stats; $this->lbFactory = $lbFactory; $this->jobQueueGroup = $jobQueueGroup; + $this->centralIdLookup = $centralIdLookup; } /** @@ -177,14 +194,35 @@ class LoginNotify implements LoggerAwareInterface { * @return string One of USER_* constants */ private function isKnownSystemFast( User $user, WebRequest $request ) { + $logContext = [ 'user' => $user->getName() ]; $result = $this->userIsInCookie( $user, $request ); - - if ( $result !== self::USER_KNOWN ) { - $result = $this->mergeResults( $result, $this->userIsInCache( $user, $request ) ); + if ( $result === self::USER_KNOWN ) { + $this->log->debug( 'Found user {user} in cookie', $logContext ); + return $result; } - $this->log->debug( 'Checking cookies and cache for {user}: {result}', [ - 'function' => __METHOD__, + if ( $this->config->get( 'LoginNotifyUseSeenTable' ) ) { + $id = $this->getMaybeCentralId( $user ); + $hash = $this->getSeenHash( $request, $id ); + $result = $this->mergeResults( $result, $this->userIsInSeenTable( $id, $hash ) ); + if ( $result === self::USER_KNOWN ) { + $this->log->debug( 'Found user {user} in table', $logContext ); + return $result; + } + } + + // No need for caching unless CheckUser will be used + if ( $this->config->get( 'LoginNotifyUseCheckUser' ) ) { + $result = $this->mergeResults( $result, $this->userIsInCache( $user, $request ) ); + if ( $result === self::USER_KNOWN ) { + $this->log->debug( 'Found user {user} in cache', $logContext ); + return $result; + } + } else { + $result = self::USER_NOT_KNOWN; + } + + $this->log->debug( 'Fast checks for {user}: {result}', [ 'user' => $user->getName(), 'result' => $result, ] ); @@ -255,6 +293,223 @@ class LoginNotify implements LoggerAwareInterface { return self::USER_NO_INFO; } + /** + * Check if the user is in our own table in a non-expired bucket + * + * @param int $centralUserId + * @param int|string $hash + * @return string One of USER_* constants + */ + private function userIsInSeenTable( int $centralUserId, $hash ) { + if ( !$centralUserId ) { + return self::USER_NO_INFO; + } + $dbr = $this->getSeenPrimaryDb(); + $seen = $dbr->newSelectQueryBuilder() + ->select( '1' ) + ->from( 'loginnotify_seen_net' ) + ->where( [ + 'lsn_user' => $centralUserId, + 'lsn_subnet' => $hash, + 'lsn_time_bucket >= ' . $dbr->addQuotes( $this->getMinBucket() ) + ] ) + ->caller( __METHOD__ ) + ->fetchField(); + if ( $seen ) { + return self::USER_KNOWN; + } elseif ( $this->config->get( 'LoginNotifyUseCheckUser' ) ) { + // We still need to check CheckUser + return self::USER_NO_INFO; + } else { + return self::USER_NOT_KNOWN; + } + } + + /** + * Check if the user is in our table in the current bucket + * + * @param int $centralUserId + * @param string $hash + * @param bool $usePrimary + * @return bool + */ + private function userIsInCurrentSeenBucket( int $centralUserId, $hash, $usePrimary = false ) { + if ( !$centralUserId ) { + return false; + } + if ( $usePrimary ) { + $dbr = $this->getSeenPrimaryDb(); + } else { + $dbr = $this->getSeenReplicaDb(); + } + return (bool)$dbr->newSelectQueryBuilder() + ->select( '1' ) + ->from( 'loginnotify_seen_net' ) + ->where( [ + 'lsn_user' => $centralUserId, + 'lsn_subnet' => $hash, + 'lsn_time_bucket' => $this->getCurrentBucket(), + ] ) + ->caller( __METHOD__ ) + ->fetchField(); + } + + /** + * Combine the user ID and IP prefix into a 64-bit hash. Return the hash + * as either an integer or a decimal string. + * + * @param WebRequest $request + * @param int $centralUserId + * @return int|string + * @throws Exception + */ + private function getSeenHash( WebRequest $request, int $centralUserId ) { + $ipPrefix = $this->getIPNetwork( $request->getIP() ); + $hash = hash_hmac( 'sha1', "$centralUserId|$ipPrefix", $this->secret, true ); + // Truncate to 64 bits + return self::packedSignedInt64ToDecimal( substr( $hash, 0, 8 ) ); + } + + /** + * Convert an 8-byte string to a 64-bit integer, and return it either as a + * native integer, or if PHP integers are 32 bits, as a decimal string. + * + * Signed 64-bit integers are a compact and portable way to store a 64-bit + * hash in a DBMS. On a 64-bit platform, PHP can easily generate and handle + * such integers, but on a 32-bit platform it is a bit awkward. + * + * @param string $str + * @return int|string + */ + private static function packedSignedInt64ToDecimal( $str ) { + if ( PHP_INT_SIZE >= 8 ) { + // The manual is confusing -- this does in fact return a signed number + return unpack( 'Jv', $str )['v']; + } else { + // PHP has precious few facilities for manipulating 64-bit numbers on a + // 32-bit platform. String bitwise operators are a nice hack though. + if ( ( $str[0] & "\x80" ) !== "\x00" ) { + // The number is negative. Find 2's complement and add minus sign. + $sign = '-'; + $str = ~$str; + $carry = 1; + // Add with carry in big endian order + for ( $i = 7; $i >= 0 && $carry; $i-- ) { + $sum = ord( $str[$i] ) + $carry; + $carry = ( $sum & 0x100 ) >> 8; + $str[$i] = chr( $sum & 0xff ); + } + } else { + $sign = ''; + } + return $sign . \Wikimedia\base_convert( bin2hex( $str ), 16, 10 ); + } + } + + /** + * Get read a connection to the database holding the loginnotify_seen_net table. + * + * @return IReadableDatabase + */ + private function getSeenReplicaDb(): IReadableDatabase { + $dbName = $this->config->get( 'LoginNotifySeenDatabase' ) ?? false; + return $this->getSeenLoadBalancer()->getConnection( DB_REPLICA, [], $dbName ); + } + + /** + * Get a write connection to the database holding the loginnotify_seen_net table. + * + * @return IDatabase + */ + private function getSeenPrimaryDb(): IDatabase { + $dbName = $this->config->get( 'LoginNotifySeenDatabase' ) ?? false; + return $this->getSeenLoadBalancer()->getConnection( DB_PRIMARY, [], $dbName ); + } + + /** + * Is the database holding the loginnotify_seen_net table replicated to + * multiple servers? + * + * @return bool + */ + private function isSeenDbReplicated() { + return $this->getSeenLoadBalancer()->hasReplicaServers(); + } + + /** + * Get the LoadBalancer holding the loginnotify_seen_net table. + * + * @return ILoadBalancer + */ + private function getSeenLoadBalancer() { + $cluster = $this->config->get( 'LoginNotifySeenCluster' ); + if ( $cluster ) { + return $this->lbFactory->getExternalLB( $cluster ); + } else { + $dbName = $this->config->get( 'LoginNotifySeenDatabase' ) ?? false; + return $this->lbFactory->getMainLB( $dbName ); + } + } + + /** + * Get the lowest time bucket index which is not expired. + * + * @return int + */ + private function getMinBucket() { + $now = $this->getCurrentTime(); + $expiry = $this->config->get( 'LoginNotifySeenExpiry' ); + $size = $this->config->get( 'LoginNotifySeenBucketSize' ); + return (int)( ( $now - $expiry ) / $size ); + } + + /** + * Get the current time bucket index. + * + * @return int + */ + private function getCurrentBucket() { + return (int)( $this->getCurrentTime() / $this->config->get( 'LoginNotifySeenBucketSize' ) ); + } + + /** + * Get the current UNIX time + * + * @return int + */ + private function getCurrentTime() { + return $this->fakeTime ?? time(); + } + + /** + * Set a fake time to be returned by getCurrentTime(), for testing. + * + * @param int|null $time + */ + public function setFakeTime( $time ) { + $this->fakeTime = $time; + } + + /** + * If LoginNotifySeenDatabase is configured, indicating a shared table, + * get the central user ID. Otherwise, get the local user ID. + * + * If CentralAuth is not installed, $this->centralIdLookup will be a + * LocalIdLookup and the local user ID will be returned regardless. But + * using CentralIdLookup unconditionally can fail if CentralAuth is + * installed but no users are attached to it, as is the case in CI. + * + * @param User $user + * @return int + */ + private function getMaybeCentralId( User $user ) { + if ( ( $this->config->get( 'LoginNotifySeenDatabase' ) ?? false ) !== false ) { + return $this->centralIdLookup->centralIdFromLocalUser( $user ); + } else { + return $user->getId(); + } + } + /** * Is the subnet of the current IP in the CheckUser data for the user. * @@ -429,7 +684,7 @@ class LoginNotify implements LoggerAwareInterface { private function setLoginCookie( User $user ) { $cookie = $this->getPrevLoginCookie( $user->getRequest() ); list( , $newCookie ) = $this->checkAndGenerateCookie( $user, $cookie ); - $expire = time() + $this->config->get( 'LoginNotifyCookieExpire' ); + $expire = $this->getCurrentTime() + $this->config->get( 'LoginNotifyCookieExpire' ); $resp = $user->getRequest()->response(); $resp->setCookie( self::COOKIE_NAME, @@ -444,15 +699,32 @@ class LoginNotify implements LoggerAwareInterface { } /** - * Give the user a cookie and cache address in memcache + * Give the user a cookie and store the address in memcached and the DB. * * It is expected this be called upon successful log in. * * @param User $user The user in question. */ - public function setCurrentAddressAsKnown( User $user ) { - $this->cacheLoginIP( $user ); + public function recordKnownWithCookie( User $user ) { + if ( !$user->isNamed() ) { + return; + } $this->setLoginCookie( $user ); + $this->recordKnown( $user ); + } + + /** + * Store the user's IP address in memcached and the DB + * + * @param User $user + * @return void + */ + public function recordKnown( User $user ) { + if ( !$user->isNamed() ) { + return; + } + $this->cacheLoginIP( $user ); + $this->recordUserInSeenTable( $user ); $this->log->debug( 'Recording user {user} as known', [ @@ -472,13 +744,139 @@ class LoginNotify implements LoggerAwareInterface { // It's assumed that most of the time, we'll be able to rely on // the cookie or CheckUser data. $expiry = $this->config->get( 'LoginNotifyCacheLoginIPExpiry' ); - if ( $expiry !== false ) { + $useCU = $this->config->get( 'LoginNotifyUseCheckUser' ); + if ( $useCU && $expiry !== false ) { $ipPrefix = $this->getIPNetwork( $user->getRequest()->getIP() ); $key = $this->getKey( $user, 'prevSubnet' ); $this->cache->set( $key, $ipPrefix, $expiry ); } } + /** + * If the user/subnet combination is not already in the database, add it. + * Also queue a job to clean up expired rows, if necessary. + * + * @param User $user + * @return void + */ + private function recordUserInSeenTable( User $user ) { + if ( !$this->config->get( 'LoginNotifyUseSeenTable' ) ) { + return; + } + $id = $this->getMaybeCentralId( $user ); + if ( !$id ) { + return; + } + + $request = $user->getRequest(); + $hash = $this->getSeenHash( $request, $id ); + + // Check if the user/hash is in the replica DB + if ( $this->userIsInCurrentSeenBucket( $id, $hash ) ) { + return; + } + + // Check whether purging is required + if ( !mt_rand( 0, (int)( $this->config->get( 'UpdateRowsPerQuery' ) / 4 ) ) ) { + $minId = $this->getMinExpiredId(); + if ( $minId !== null ) { + $this->log->debug( 'Queueing purge job starting from lsn_id={minId}', + [ 'minId' => $minId ] ); + // Deferred call to purgeSeen() + // removeDuplicates effectively limits concurrency to 1, since + // no more work will be queued until the DELETE is committed. + $job = new JobSpecification( + 'LoginNotifyPurgeSeen', + [ 'minId' => $minId ], + [ 'removeDuplicates' => true ] + ); + $this->jobQueueGroup->push( $job ); + } + } + + // Insert a row + $dbw = $this->getSeenPrimaryDb(); + $isReplicated = $this->isSeenDbReplicated(); + $fname = __METHOD__; + $dbw->onTransactionCommitOrIdle( + function () use ( $dbw, $id, $hash, $isReplicated, $fname ) { + // Check if the user/hash is in the primary DB, as late as + // possible before the insert. (Trying to reduce the number of + // no-op queries in the binlog) + if ( $isReplicated && $this->userIsInCurrentSeenBucket( $id, $hash, true ) ) { + return; + } + + $dbw->newInsertQueryBuilder() + ->insert( 'loginnotify_seen_net' ) + ->ignore() + ->row( [ + 'lsn_time_bucket' => $this->getCurrentBucket(), + 'lsn_user' => $id, + 'lsn_subnet' => $hash + ] ) + ->caller( $fname ) + ->execute(); + } + ); + } + + /** + * Estimate the minimum lsn_id which has an expired time bucket. + * + * The primary key is approximately monotonic in time. Guess whether + * purging is required by looking at the first row ordered by + * primary key. If this check misses a row, it will be cleaned up + * when the next bucket expires. + * + * @return int|null + */ + public function getMinExpiredId() { + $minRow = $this->getSeenPrimaryDb()->newSelectQueryBuilder() + ->select( [ 'lsn_id', 'lsn_time_bucket' ] ) + ->from( 'loginnotify_seen_net' ) + ->orderBy( 'lsn_id' ) + ->limit( 1 ) + ->caller( __METHOD__ ) + ->fetchRow(); + if ( !$minRow ) { + return null; + } elseif ( $minRow->lsn_time_bucket < $this->getMinBucket() ) { + return (int)$minRow->lsn_id; + } else { + return null; + } + } + + /** + * Purge rows from the loginnotify_seen_net table that are expired. + * + * @param int $minId The lsn_id to start at + * @return int|null The lsn_id to continue at, or null if no more expired + * rows are expected. + */ + public function purgeSeen( $minId ) { + $dbw = $this->getSeenPrimaryDb(); + $maxId = $minId + $this->config->get( 'UpdateRowsPerQuery' ); + + $dbw->newDeleteQueryBuilder() + ->delete( 'loginnotify_seen_net' ) + ->where( [ + 'lsn_id >= ' . $dbw->addQuotes( $minId ), + 'lsn_id < ' . $dbw->addQuotes( $maxId ), + 'lsn_time_bucket < ' . $dbw->addQuotes( $this->getMinBucket() ) + ] ) + ->caller( __METHOD__ ) + ->execute(); + + // If there were affected rows, tell the maintenance script to keep looking + if ( $dbw->affectedRows() ) { + return $maxId; + } else { + return null; + } + } + /** * Merges results of various isKnownSystem*() checks * @@ -793,10 +1191,12 @@ class LoginNotify implements LoggerAwareInterface { $known = $this->isKnownSystemFast( $user, $user->getRequest() ); if ( $known === self::USER_KNOWN ) { $this->recordLoginFailureFromKnownSystem( $user ); - } else { + } elseif ( $this->config->get( 'LoginNotifyUseCheckUser' ) ) { $this->createJob( DeferredChecksJob::TYPE_LOGIN_FAILED, $user, $user->getRequest(), $known ); + } else { + $this->recordLoginFailureFromUnknownSystem( $user ); } } @@ -827,10 +1227,15 @@ class LoginNotify implements LoggerAwareInterface { } $this->incrStats( 'success.total' ); $result = $this->isKnownSystemFast( $user, $user->getRequest() ); - if ( $result !== self::USER_KNOWN ) { + if ( $result === self::USER_KNOWN ) { + // No need to notify + } elseif ( $this->config->get( 'LoginNotifyUseCheckUser' ) ) { $this->createJob( DeferredChecksJob::TYPE_LOGIN_SUCCESS, $user, $user->getRequest(), $result ); + } elseif ( $result === self::USER_NOT_KNOWN ) { + $this->incrStats( 'success.notifications' ); + $this->sendNotice( $user, 'login-success' ); } } diff --git a/includes/PurgeSeenJob.php b/includes/PurgeSeenJob.php new file mode 100644 index 0000000..86c8085 --- /dev/null +++ b/includes/PurgeSeenJob.php @@ -0,0 +1,20 @@ +loginNotify = $loginNotify; + } + + public function run() { + $minId = $this->getParams()['minId']; + $this->loginNotify->purgeSeen( $minId ); + return true; + } +} diff --git a/includes/SchemaHooks.php b/includes/SchemaHooks.php new file mode 100644 index 0000000..d87f808 --- /dev/null +++ b/includes/SchemaHooks.php @@ -0,0 +1,18 @@ +addExtensionTable( + 'loginnotify_seen_net', + dirname( __DIR__ ) . "/sql/{$updater->getDB()->getType()}/tables-generated.sql" + ); + } +} diff --git a/includes/ServiceWiring.php b/includes/ServiceWiring.php index 9e468b8..58f17b4 100644 --- a/includes/ServiceWiring.php +++ b/includes/ServiceWiring.php @@ -16,7 +16,8 @@ return [ LoggerFactory::getInstance( 'LoginNotify' ), $services->getStatsdDataFactory(), $services->getDBLoadBalancerFactory(), - $services->getJobQueueGroup() + $services->getJobQueueGroup(), + $services->getCentralIdLookup() ); } ]; diff --git a/maintenance/purgeSeen.php b/maintenance/purgeSeen.php new file mode 100644 index 0000000..cf62d48 --- /dev/null +++ b/maintenance/purgeSeen.php @@ -0,0 +1,30 @@ +addDescription( 'Purge expired user IP address information stored by LoginNotify' ); + } + + public function execute() { + $loginNotify = LoginNotify::getInstance(); + $minId = $loginNotify->getMinExpiredId(); + for ( ; $minId !== null; $this->waitForReplication() ) { + $minId = $loginNotify->purgeSeen( $minId ); + } + } +} + +$maintClass = PurgeSeen::class; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/sql/mysql/tables-generated.sql b/sql/mysql/tables-generated.sql new file mode 100644 index 0000000..6d90d7c --- /dev/null +++ b/sql/mysql/tables-generated.sql @@ -0,0 +1,14 @@ +-- This file is automatically generated using maintenance/generateSchemaSql.php. +-- Source: sql/tables.json +-- Do not modify this file directly. +-- See https://www.mediawiki.org/wiki/Manual:Schema_changes +CREATE TABLE /*_*/loginnotify_seen_net ( + lsn_id INT UNSIGNED AUTO_INCREMENT NOT NULL, + lsn_time_bucket SMALLINT UNSIGNED NOT NULL, + lsn_user INT UNSIGNED NOT NULL, + lsn_subnet BIGINT NOT NULL, + UNIQUE INDEX loginnotify_seen_net_user ( + lsn_user, lsn_subnet, lsn_time_bucket + ), + PRIMARY KEY(lsn_id) +) /*$wgDBTableOptions*/; diff --git a/sql/postgres/tables-generated.sql b/sql/postgres/tables-generated.sql new file mode 100644 index 0000000..3b088bf --- /dev/null +++ b/sql/postgres/tables-generated.sql @@ -0,0 +1,15 @@ +-- This file is automatically generated using maintenance/generateSchemaSql.php. +-- Source: sql/tables.json +-- Do not modify this file directly. +-- See https://www.mediawiki.org/wiki/Manual:Schema_changes +CREATE TABLE loginnotify_seen_net ( + lsn_id SERIAL NOT NULL, + lsn_time_bucket SMALLINT NOT NULL, + lsn_user INT NOT NULL, + lsn_subnet BIGINT NOT NULL, + PRIMARY KEY(lsn_id) +); + +CREATE UNIQUE INDEX loginnotify_seen_net_user ON loginnotify_seen_net ( + lsn_user, lsn_subnet, lsn_time_bucket +); diff --git a/sql/sqlite/tables-generated.sql b/sql/sqlite/tables-generated.sql new file mode 100644 index 0000000..fa8426c --- /dev/null +++ b/sql/sqlite/tables-generated.sql @@ -0,0 +1,14 @@ +-- This file is automatically generated using maintenance/generateSchemaSql.php. +-- Source: sql/tables.json +-- Do not modify this file directly. +-- See https://www.mediawiki.org/wiki/Manual:Schema_changes +CREATE TABLE /*_*/loginnotify_seen_net ( + lsn_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + lsn_time_bucket SMALLINT UNSIGNED NOT NULL, + lsn_user INTEGER UNSIGNED NOT NULL, + lsn_subnet BIGINT NOT NULL +); + +CREATE UNIQUE INDEX loginnotify_seen_net_user ON /*_*/loginnotify_seen_net ( + lsn_user, lsn_subnet, lsn_time_bucket +); diff --git a/sql/tables.json b/sql/tables.json new file mode 100644 index 0000000..b5ce45d --- /dev/null +++ b/sql/tables.json @@ -0,0 +1,57 @@ +[ + { + "name": "loginnotify_seen_net", + "comment": "Summary of subnets used by local or global users", + "columns": [ + { + "name": "lsn_id", + "comment": "Primary key", + "type": "integer", + "options": { + "unsigned": true, + "notnull": true, + "autoincrement": true + } + }, + { + "name": "lsn_time_bucket", + "comment": "Time since epoch divided by the bucket duration, e.g. 15 days", + "type": "smallint", + "options": { + "unsigned": true, + "notnull": true + } + }, + { + "name": "lsn_user", + "comment": "globaluser.gu_id or user.user_id (CentralIdLookup)", + "type": "integer", + "options": { + "unsigned": true, + "notnull": true + } + }, + { + "name": "lsn_subnet", + "comment": "Truncated hash of IP address subnet", + "type": "bigint", + "options": { + "notnull": true + } + } + ], + "indexes": [ + { + "name": "loginnotify_seen_net_user", + "comment": "To check if the subnet is known, on login or before insertion.", + "columns": [ "lsn_user", "lsn_subnet", "lsn_time_bucket" ], + "unique": true + } + ], + "pk": [ "lsn_id" ] + }, + { + "name": "loginnotify_purge_claim", + "comment": "" + } +] diff --git a/tests/phpunit/LoginNotifyTest.php b/tests/phpunit/LoginNotifyTest.php index 566316b..239fe0d 100644 --- a/tests/phpunit/LoginNotifyTest.php +++ b/tests/phpunit/LoginNotifyTest.php @@ -8,6 +8,8 @@ use MediaWiki\Request\FauxRequest; use MediaWiki\User\UserFactory; use Wikimedia\TestingAccessWrapper; +// phpcs:disable MediaWiki.WhiteSpace.SpaceBeforeSingleLineComment.NewLineComment + /** * @covers \LoginNotify\LoginNotify * @group LoginNotify @@ -22,19 +24,27 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { private $userFactory; public function setUpLoginNotify( $configValues = [] ) { + $day = 86400; $config = new HashConfig( $configValues + [ "LoginNotifyAttemptsKnownIP" => 15, - "LoginNotifyExpiryKnownIP" => 604800, + "LoginNotifyExpiryKnownIP" => 7 * $day, "LoginNotifyAttemptsNewIP" => 5, - "LoginNotifyExpiryNewIP" => 1209600, + "LoginNotifyExpiryNewIP" => 14 * $day, "LoginNotifyCheckKnownIPs" => true, "LoginNotifyEnableOnSuccess" => true, "LoginNotifySecretKey" => "Secret Stuff!", "SecretKey" => "", - "LoginNotifyCookieExpire" => 15552000, + "LoginNotifyCookieExpire" => 180 * $day, "LoginNotifyCookieDomain" => null, "LoginNotifyMaxCookieRecords" => 6, - "LoginNotifyCacheLoginIPExpiry" => 60 * 60 * 24 * 60 + "LoginNotifyCacheLoginIPExpiry" => 60 * $day, + 'LoginNotifySeenCluster' => null, + "LoginNotifySeenDatabase" => null, + "LoginNotifyUseCheckUser" => false, + "LoginNotifyUseSeenTable" => true, + "LoginNotifySeenExpiry" => 180 * $day, + "LoginNotifySeenBucketSize" => 15 * $day, + "UpdateRowsPerQuery" => 100, ] ); $services = $this->getServiceContainer(); $this->inst = TestingAccessWrapper::newFromObject( @@ -44,22 +54,26 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { LoggerFactory::getInstance( 'LoginNotify' ), $services->getStatsdDataFactory(), $services->getDBLoadBalancerFactory(), - $services->getJobQueueGroup() + $services->getJobQueueGroup(), + new LocalIdLookup( + new HashConfig( [ + 'SharedDB' => false, + 'SharedTables' => [], + 'LocalDatabases' => [] + ] ), + $services->getDBLoadBalancerFactory() + ) ) ); $this->inst->setLogger( LoggerFactory::getInstance( 'LoginNotify' ) ); $this->userFactory = $this->getServiceContainer()->getUserFactory(); } - public function setUp(): void { - parent::setUp(); - $this->setUpLoginNotify(); - } - /** * @dataProvider provideGetIPNetwork */ public function testGetIPNetwork( $ip, $expected ) { + $this->setUpLoginNotify(); $actual = $this->inst->getIPNetwork( $ip ); $this->assertSame( $expected, $actual ); } @@ -76,6 +90,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { } public function testGetIPNetworkInvalid() { + $this->setUpLoginNotify(); $this->expectException( UnexpectedValueException::class ); $this->inst->getIPNetwork( 'localhost' ); } @@ -84,6 +99,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { * @dataProvider provideGenerateUserCookieRecord */ public function testGenerateUserCookieRecord( $username, $year, $salt, $expected ) { + $this->setUpLoginNotify(); $actual = $this->inst->generateUserCookieRecord( $username, $year, $salt ); $this->assertEquals( $expected, $actual ); } @@ -101,6 +117,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { * @dataProvider provideIsUserRecordGivenCookie */ public function testIsUserRecordGivenCookie( $cookieOptions, $expected, $desc ) { + $this->setUpLoginNotify(); $user = $this->userFactory->newFromName( 'Foo', UserFactory::RIGOR_NONE ); if ( is_string( $cookieOptions ) ) { $cookieRecord = $cookieOptions; @@ -130,6 +147,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { } public function testGetPrevLoginCookie() { + $this->setUpLoginNotify(); $req = new FauxRequest(); $res1 = $this->inst->getPrevLoginCookie( $req ); $this->assertSame( '', $res1, "no cookie set" ); @@ -140,6 +158,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { } public function testGetKey() { + $this->setUpLoginNotify(); $user1 = $this->userFactory->newFromName( 'Foo_bar' ); // Make sure proper normalization happens. $user2 = $this->userFactory->newFromName( 'Foo__bar' ); @@ -164,6 +183,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { } public function testCheckAndIncKey() { + $this->setUpLoginNotify(); $key = 'global:loginnotify:new:tuwpi7e2h9pidovmaxxswk6aq327ewg'; for ( $i = 1; $i < 5; $i++ ) { $res = $this->inst->checkAndIncKey( $key, 5, 3600 ); @@ -187,6 +207,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { * @dataProvider provideClearCounters */ public function testClearCounters( $key ) { + $this->setUpLoginNotify(); $user = $this->userFactory->newFromName( "Fred" ); $key = $this->inst->getKey( $user, $key ); @@ -216,6 +237,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { $expectedNewCookie, $desc ) { + $this->setUpLoginNotify(); $user = $this->userFactory->newFromName( 'Foo' ); list( $actualSeenBefore, $actualNewCookie ) = $this->inst->checkAndGenerateCookie( $user, $cookie ); @@ -300,6 +322,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { * @dataProvider provideValidateCookieRecord */ public function testValidateCookieRecord( $cookie, $expected ) { + $this->setUpLoginNotify(); $this->assertEquals( $expected, $this->inst->validateCookieRecord( $cookie ) ); } @@ -316,6 +339,7 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { } public function testUserIsInCache() { + $this->setUpLoginNotify( [ "LoginNotifyUseCheckUser" => true ] ); $u = $this->userFactory->newFromName( 'Xyzzy' ); $this->assertSame( LoginNotify::USER_NO_INFO, @@ -337,10 +361,25 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { ); } - public function testRecordFailureKnownCache() { - $this->setupRecordFailure(); + public static function provideRecordFailureKnownCacheOrTable() { + return [ + [ 'cache' ], + [ 'table' ] + ]; + } + + /** + * @dataProvider provideRecordFailureKnownCacheOrTable + * @param string $type + */ + public function testRecordFailureKnownCacheOrTable( $type ) { + $config = [ + 'LoginNotifyUseCheckUser' => $type === 'cache', + 'LoginNotifyUseSeenTable' => $type !== 'cache' + ]; + $this->setupRecordFailure( $config ); $user = $this->getTestUser()->getUser(); - $this->inst->setCurrentAddressAsKnown( $user ); + $this->inst->recordKnown( $user ); // Record a failure, does not notify because the interval is 2 $this->inst->recordFailure( $user ); @@ -400,7 +439,32 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { $this->assertNotificationCount( $user, 'login-fail-known', 0 ); } - public function testSendSuccessNotice() { + public function testRecordFailureSeenExpired() { + $this->setupRecordFailure( [ + 'LoginNotifyAttemptsKnownIP' => 1, + 'LoginNotifyAttemptsNewIP' => 1, + ] ); + $user = $this->getTestUser()->getUser(); + $day = 86400; + + // Mark the IP as known + $this->inst->setFakeTime( 2 * $day ); + $this->inst->recordKnown( $user ); + + // 30 days later, still known + $this->inst->setFakeTime( 32 * $day ); + $this->inst->recordFailure( $user ); + $this->assertNotificationCount( $user, 'login-fail-new', 0 ); + $this->assertNotificationCount( $user, 'login-fail-known', 1 ); + + // 180 days later, rounded up to the nearest bucket, data is expired + $this->inst->setFakeTime( 210 * $day ); + $this->inst->recordFailure( $user ); + $this->assertNotificationCount( $user, 'login-fail-new', 1 ); + $this->assertNotificationCount( $user, 'login-fail-known', 1 ); + } + + public function testSendSuccessNoticeCheckUser() { $this->setupRecordFailureWithCheckUser(); $helper = new TestRecentChangesHelper; $user = $this->getTestUser()->getUser(); @@ -428,21 +492,48 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { $this->assertTrue( $emailSent ); } - private function setupRecordFailure() { - $config = [ + public function testSendSuccessNoticeSeen() { + $this->setupRecordFailure(); + $user = $this->getTestUser()->getUser(); + $user->setEmail( 'test@test.mediawiki.org' ); + $user->confirmEmail(); + $user->saveSettings(); + + $emailSent = false; + $this->setTemporaryHook( 'EchoAbortEmailNotification', + static function () use ( &$emailSent ) { + $emailSent = true; + return false; + } + ); + + // Record 127.0.0.1 + $this->inst->recordKnown( $user ); + + // Change the IP and record a success + RequestContext::getMain()->getRequest()->setIP( '127.1.0.0' ); + $this->inst->sendSuccessNotice( $user ); + $this->assertTrue( $emailSent ); + } + + private function setupRecordFailure( $config = [] ) { + $config += [ 'LoginNotifyAttemptsKnownIP' => 2, 'LoginNotifyAttemptsNewIP' => 2, ]; $this->setUpLoginNotify( $config ); - $this->overrideConfigValues( $config ); + $this->overrideConfigValues( $config ); // for jobs $this->tablesUsed[] = 'user'; $this->tablesUsed[] = 'echo_event'; $this->tablesUsed[] = 'echo_notification'; + if ( $config['LoginNotifyUseSeenTable'] ?? true ) { + $this->tablesUsed[] = 'loginnotify_seen_net'; + } } private function setupRecordFailureWithCheckUser() { $this->markTestSkippedIfExtensionNotLoaded( 'CheckUser' ); - $this->setupRecordFailure(); + $this->setupRecordFailure( [ 'LoginNotifyUseCheckUser' => true ] ); $this->tablesUsed[] = 'comment'; $this->tablesUsed[] = 'cu_changes'; } @@ -465,4 +556,82 @@ class LoginNotifyTest extends MediaWikiIntegrationTestCase { ] ) ->assertFieldValue( $expected ); } + + public static function providePackedSignedInt64ToDecimal() { + return [ + [ '0000000000000000', 0 ], + [ '0000000000000001', 1 ], + [ 'ffffffffffffffff', -1 ], + [ '7fffffffffffffff', '9223372036854775807' ], + [ '8000000000000000', '-9223372036854775808' ], + ]; + } + + /** + * @dataProvider providePackedSignedInt64ToDecimal + * @param string $hexInput + * @param string|int $expected + */ + public function testPackedSignedInt64ToDecimal( $hexInput, $expected ) { + $class = TestingAccessWrapper::newFromClass( LoginNotify::class ); + $input = hex2bin( $hexInput ); + $result = $class->packedSignedInt64ToDecimal( $input ); + $this->assertSame( (string)$expected, (string)$result ); + } + + public function testPurgeSeen() { + $this->setupLoginNotify( [ 'UpdateRowsPerQuery' => 1 ] ); + $this->tablesUsed[] = 'user'; + $this->tablesUsed[] = 'loginnotify_seen_net'; + $user = $this->getTestUser()->getUser(); + $day = 86400; + $this->inst->setFakeTime( 0 ); + $this->inst->recordKnown( $user ); + $this->inst->setFakeTime( 90 * $day ); + $this->inst->recordKnown( $user ); + $this->inst->setFakeTime( 210 * $day ); + $this->inst->recordKnown( $user ); + + $this->assertSeenCount( 3 ); + + $this->inst->setFakeTime( 300 * $day ); + $minId = $this->inst->getMinExpiredId(); + $this->assertGreaterThan( 0, $minId ); + + $nextId = $this->inst->purgeSeen( $minId ); + $this->assertSeenCount( 2 ); + $this->assertGreaterThan( $minId, $nextId ); + + $nextId = $this->inst->purgeSeen( $nextId ); + $this->assertGreaterThan( $minId, $nextId ); + $this->assertSeenCount( 1 ); + + $nextId = $this->inst->purgeSeen( $nextId ); + $this->assertNull( $nextId ); + } + + public function testPurgeViaJob() { + $this->setupLoginNotify( [ 'UpdateRowsPerQuery' => 1 ] ); + $this->tablesUsed[] = 'user'; + $this->tablesUsed[] = 'loginnotify_seen_net'; + $user = $this->getTestUser()->getUser(); + + $this->inst->setFakeTime( 0 ); // 1970 + $this->inst->recordUserInSeenTable( $user ); + $this->assertSeenCount( 1 ); + + $this->inst->setFakeTime( null ); // real current time + $this->inst->recordUserInSeenTable( $user ); + $this->assertSeenCount( 2 ); + + $this->runJobs(); + $this->assertSeenCount( 1 ); + } + + private function assertSeenCount( $expected ) { + $this->newSelectQueryBuilder() + ->select( 'COUNT(*)' ) + ->from( 'loginnotify_seen_net' ) + ->assertFieldValue( $expected ); + } }