wikimedia/mediawiki-extensions-Echo
1
0
Fork 0
mirror of https://gerrit.wikimedia.org/r/mediawiki/extensions/Echo synced 2024-11-24 07:54:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Database updates for respecting oversight within Echo

Browse source 
Provides the first step of adding and populating a new database field
for Echo oversight deployment.  The new field is populated via a
maintenance script and Event::loadFromRow will accept both new and old
results.  Everything will still run when the 2 now unused fields are
later dropped from the db.

Bug: 48059
Change-Id: I24d4b61a061f94ed9aaaa6087f33b2ab37f773cd
This commit is contained in:
Erik Bernhardson 2013-05-09 11:50:05 -07:00
parent 61ab47b76c
commit f38ce97efd
12 changed files with 1059 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
Echo.php Normal file → Executable file
View file

@ -73,7 +73,7 @@ $wgAutoloadClasses['MWEchoNotificationEmailBundleJob'] = $dir . 'jobs/Notificati
$wgJobClasses['MWEchoNotificationEmailBundleJob'] = 'MWEchoNotificationEmailBundleJob';
// API
$wgAutoloadClasses['ApiEchoNotifications'] = $dir . 'api/ApiEchoNotifications.php';
$wgAutoloadClasses['ApiEchoNotifications'] = $dir . 'api/ApiEchoNotifications.php';
$wgAPIMetaModules['notifications'] = 'ApiEchoNotifications';
// Special page
@ -93,6 +93,13 @@ $wgAutoloadClasses['EchoArrayList'] = $dir . 'includes/ContainmentSet.php';
$wgAutoloadClasses['EchoOnWikiList'] = $dir . 'includes/ContainmentSet.php';
$wgAutoloadClasses['EchoCachedList'] = $dir . 'includes/ContainmentSet.php';
// Maintenance testing
$wgAutoloadClasses['EchoBatchRowUpdate'] = $dir . 'includes/BatchRowUpdate.php';
$wgAutoloadClasses['EchoBatchRowWriter'] = $dir . 'includes/BatchRowUpdate.php';
$wgAutoloadClasses['EchoBatchRowIterator'] = $dir . 'includes/BatchRowUpdate.php';
$wgAutoloadClasses['EchoRowUpdateGenerator'] = $dir . 'includes/BatchRowUpdate.php';
$wgAutoloadClasses['EchoSuppressionRowUpdateGenerator'] = $dir . 'includes/schemaUpdate.php';
// Housekeeping hooks
$wgHooks['LoadExtensionSchemaUpdates'][] = 'EchoHooks::getSchemaUpdates';
$wgHooks['GetPreferences'][] = 'EchoHooks::getPreferences';

4
Hooks.php
View file

@ -71,6 +71,7 @@ class EchoHooks {
$updater->dropExtensionField( 'echo_event', 'event_timestamp', "$dir/db_patches/patch-drop-echo_event-event_timestamp.sql" );
$updater->addExtensionField( 'echo_email_batch', 'eeb_event_hash',
"$dir/db_patches/patch-email_batch-new-field.sql" );
$updater->addExtensionField( 'echo_event', 'event_page_id', "$dir/db_patches/patch-add-echo_event-event_page_id.sql" );
return true;
}
@ -571,8 +572,7 @@ class EchoHooks {
'title' => $title,
'agent' => $wgUser,
'extra' => array(
'link-from-namespace' => $linksUpdate->mTitle->getNamespace(),
'link-from-title' => $linksUpdate->mTitle->getDBkey(),
'link-from-page-id' => $linksUpdate->mTitle->getArticleId(),
)
) );
$max--;

1
db_patches/patch-add-echo_event-event_page_id.sql Normal file
View file

@ -0,0 +1 @@
ALTER TABLE /*_*/echo_event ADD event_page_id int unsigned;

3
echo.sql Normal file → Executable file
View file

@ -8,7 +8,8 @@ CREATE TABLE /*_*/echo_event (
event_agent_ip varchar(39) binary null, -- IP address who triggered it, if any
event_page_namespace int unsigned null,
event_page_title varchar(255) binary null,
event_extra BLOB NULL
event_extra BLOB NULL,
event_page_id int unsigned null
) /*$wgDBTableOptions*/;
CREATE INDEX /*i*/event_type ON /*_*/echo_event (event_type);

74
formatters/PageLinkFormatter.php
View file

@ -5,6 +5,32 @@
*/
class EchoPageLinkFormatter extends EchoBasicFormatter {
/**
* This is a workaround for backwards compatibility.
* In https://gerrit.wikimedia.org/r/#/c/63076 we changed
* the schema to save link-from-page-id instead of
* link-from-namespace & link-from-title
*/
protected function extractExtra( $extra ) {
if ( isset( $extra['link-from-namespace'], $extra['link-from-title'] )
&& !isset( $extra['link-from-page-id'] )
) {
$title = Title::makeTitleSafe(
$extra['link-from-namespace'],
$extra['link-from-title']
);
if ( $title ) {
$extra['link-from-page-id'] = $title->getArticleId();
unset(
$extra['link-from-namespace'],
$extra['link-from-title']
);
}
}
return $extra;
}
/**
* This method overwrite parent method and construct the bundle iterator
* based on link from, it will be used in a message like this: Page A was
@ -21,15 +47,20 @@ class EchoPageLinkFormatter extends EchoBasicFormatter {
if ( !$data ) {
return;
}
$extra = $event->getExtra();
$extra = self::extractExtra( $event->getExtra() );
$linkFrom = array();
if ( $this->isTitleSet( $extra ) ) {
$linkFrom[$this->getTitleHash( $extra )] = true;
} else {
throw new MWException( "Link from title is required for bundling notification!" );
if ( !$this->isTitleSet( $extra ) ) {
// Link from title is required for bundling notification
return;
}
$key = $this->getTitleHash( $extra );
if ( !$key ) {
// Page no longer exists
return;
}
$linkFrom[$key] = true;
$count = 1;
foreach ( $data as $row ) {
@ -41,7 +72,7 @@ class EchoPageLinkFormatter extends EchoBasicFormatter {
if ( $this->isTitleSet( $extra ) ) {
$key = $this->getTitleHash( $extra );
if ( !isset( $linkFrom[$key] ) ) {
if ( $key && !isset( $linkFrom[$key] ) ) {
$linkFrom[$key] = true;
$count++;
}
@ -63,20 +94,16 @@ class EchoPageLinkFormatter extends EchoBasicFormatter {
* @return bool
*/
private function isTitleSet( $extra ) {
if ( isset( $extra['link-from-namespace'], $extra['link-from-title'] ) ) {
return true;
} else {
return false;
}
return isset( $extra['link-from-page-id'] );
}
/**
* Internal function to generate a unique md5 of namespace and title
* Internal function to return a unique identifier representing the page.
* @param $extra array
* @return string
* @return integer Unique identifier for the linked page
*/
private function getTitleHash( $extra ) {
return md5( $extra['link-from-namespace'] . '-' . $extra['link-from-title'] );
return $extra['link-from-page-id'];
}
/**
@ -86,20 +113,21 @@ class EchoPageLinkFormatter extends EchoBasicFormatter {
* @param $user User
*/
protected function processParam( $event, $param, $message, $user ) {
$extra = $event->getExtra();
$extra = self::extractExtra( $event->getExtra() );
switch ( $param ) {
// 'A' part in this message: link from page A and X others
case 'link-from-page':
$content = null;
if ( $this->isTitleSet( $extra ) ) {
$message->params(
Title::makeTitle(
$extra['link-from-namespace'],
$extra['link-from-title']
)
);
} else {
$message->params( '' );
$title = Title::newFromId( $extra['link-from-page-id'] );
if ( $title !== null ) {
$content = $this->formatTitle( $title );
}
}
if ( $content === null ) {
$content = wfMessage( 'echo-no-title' );
}
$message->params( $content );
break;
// example: {7} other page, {99+} other pages

425
includes/BatchRowUpdate.php Normal file
View file

@ -0,0 +1,425 @@
<?php
/**
* Provides components to update a tables rows via a batching process
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @ingroup Maintenance
*/
/**
* Ties together the batch update components to provide a composable method
* of batch updating rows in a database. To use create a class implementing
* the EchoRowUpdateGenerator interface and configure the EchoBatchRowIterator and
* EchoBatchRowWriter for access to the correct table. The components will
* handle reading, writing, and waiting for slaves while the generator implementation
* handles generating update arrays for singular rows.
*
* Instantiate:
* $updater = new EchoBatchRowUpdate(
* new EchoBatchRowIterator( $dbr, 'some_table', 'primary_key_column', 500 ),
* new EchoBatchRowWriter( $dbw, 'some_table', 'clusterName' ),
* new MyImplementationOfEchoRowUpdateGenerator
* );
*
* Run:
* $updater->execute();
*
* An example maintenance script utilizing the EchoBatchRowUpdate can be located in the Echo
* extension file maintenance/updateSchema.php
*
* @ingroup Maintenance
*/
class EchoBatchRowUpdate {
/**
* @var EchoBatchRowIterator $reader Iterator that returns an array of database rows
*/
protected $reader;
/**
* @var EchoBatchRowWriter $writer Writer capable of pushing row updates to the database
*/
protected $writer;
/**
* @var EchoRowUpdateGenerator $generator Generates single row updates based on the rows content
*/
protected $generator;
/**
* @var callable $output Output callback
*/
protected $output;
/**
* @param EchoBatchRowIterator $reader Iterator that returns an array of database rows
* @param EchoBatchRowWriter $writer Writer capable of pushing row updates to the database
* @param EchoRowUpdateGenerator $generator Generates single row updates based on the rows content
*/
public function __construct( EchoBatchRowIterator $reader, EchoBatchRowWriter $writer, EchoRowUpdateGenerator $generator ) {
$this->reader = $reader;
$this->writer = $writer;
$this->generator = $generator;
$this->output = function() {
}; // nop
}
/**
* Runs the batch update process
*/
public function execute() {
foreach ( $this->reader as $rows ) {
$updates = array();
foreach ( $rows as $row ) {
$update = $this->generator->update( $row );
if ( $update ) {
$updates[] = array(
'primaryKey' => $this->reader->extractPrimaryKeys( $row ),
'changes' => $update,
);
}
}
if ( $updates ) {
$this->output( "Processing " . count( $updates ) . " rows\n" );
$this->writer->write( $updates );
}
}
$this->output( "Completed\n" );
}
/**
* Accepts a callable which will receive a single parameter containing
* string status updates
*
* @param callable $output A callback taking a single string parameter to output
*/
public function setOutput( $output ) {
if ( !is_callable( $output ) ) {
throw new MWException( 'Provided $output param is required to be callable.' );
}
$this->output = $output;
}
/**
* Write out a status update
*
* @param string $text The value to print
*/
protected function output( $text ) {
call_user_func( $this->output, $text );
}
}
/**
* Interface for generating updates to single rows in the database.
*
* @ingroup Maintenance
*/
interface EchoRowUpdateGenerator {
/**
* Given a database row, generates an array mapping column names to updated value within the database row
*
* Sample Response:
* return array(
* 'some_col' => 'new value',
* 'other_col' => 99,
* );
*
* @param stdClass $row A row from the database
* @return array Map of column names to updated value within the database row. When no update is required
* returns an empty array.
*/
public function update( $row );
}
/**
* Updates database rows by primary key in batches. There are two options for writing to tables
* with a composite primary key.
*
* @ingroup Maintenance
*/
class EchoBatchRowWriter {
/**
* @var DatabaseBase $db The database to write to
*/
protected $db;
/**
* @var string $table The name of the table to update
*/
protected $table;
/**
* @var string $clusterName A cluster name valid for use with LBFactory
*/
protected $clusterName;
/**
* @param DatabaseBase $db The database to write to
* @param string $table The name of the table to update
* @param string $clusterName A cluster name valid for use with LBFactory
*/
public function __construct( DatabaseBase $db, $table, $clusterName = false ) {
$this->db = $db;
$this->table = $table;
$this->clusterName = $clusterName;
}
/**
* @param array $updates Array of arrays each containing two keys, 'primaryKey' and 'changes'.
* primaryKey must contain a map of column names to values sufficient to uniquely identify the row
* changes must contain a map of column names to update values to apply to the row
*/
public function write( array $updates ) {
$this->db->begin();
foreach ( $updates as $id => $update ) {
//echo "Updating: ";var_dump( $update['primaryKey'] );
//echo "With values: ";var_dump( $update['changes'] );
$this->db->update(
$this->table,
$update['changes'],
$update['primaryKey'],
__METHOD__
);
}
$this->db->commit();
wfWaitForSlaves( false, false, $this->clusterName );
}
}
/**
* Fetches rows batched into groups from the database in ascending order of the primary key(s).
*
* @ingroup Maintenance
*/
class EchoBatchRowIterator implements Iterator {
/**
* @var DatabaseBase $db The database to read from
*/
protected $db;
/**
* @var string $table The name of the table to read from
*/
protected $table;
/**
* @var array $primaryKey The name of the primary key(s)
*/
protected $primaryKey;
/**
* @var integer $batchSize The number of rows to fetch per iteration
*/
protected $batchSize;
/**
* @var array $conditions Array of strings containing SQL conditions to add to the query
*/
protected $conditions = array();
/**
* @var array $fetchColumns List of column names to select from the table suitable for use with DatabaseBase::select()
*/
protected $fetchColumns = array( '*' );
/**
* @var string $orderBy SQL Order by condition generated from $this->primaryKey
*/
protected $orderBy;
/**
* @var array $current The current iterator value
*/
private $current = array();
/**
* @var integer key 0-indexed number of pages fetched since self::reset()
*/
private $key;
/**
* @param DatabaseBase $db The database to read from
* @param string $table The name of the table to read from
* @param string|array $primaryKey The name or names of the primary key columns
* @param integer $batchSize The number of rows to fetch per iteration
*/
public function __construct( DatabaseBase $db, $table, $primaryKey, $batchSize ) {
if ( $batchSize < 1 ) {
throw new MWException( 'Batch size must be at least 1 row.' );
}
$this->db = $db;
$this->table = $table;
$this->primaryKey = (array) $primaryKey;
$this->orderBy = implode( ' ASC,', $this->primaryKey ) . ' ASC';
$this->batchSize = $batchSize;
}
/**
* @param string $condition Query conditions suitable for use with DatabaseBase::select
*/
public function addConditions( array $conditions ) {
$this->conditions = array_merge( $this->conditions, $conditions );
}
/**
* @param array $columns List of column names to select from the table suitable for use with DatabaseBase::select()
*/
public function setFetchColumns( array $columns ) {
// If it's not the all column selector merge in the primary keys we need
if ( count( $columns ) === 1 && reset( $columns ) === '*' ) {
$this->fetchColumns = $columns;
} else {
$this->fetchColumns = array_unique( array_merge( $this->primaryKey, $columns ) );
}
}
/**
* Extracts the primary key(s) from a database row.
*
* @param stdClass $row An individual database row from this iterator
* @return array Map of primary key column to value within the row
*/
public function extractPrimaryKeys( $row ) {
$pk = array();
foreach ( $this->primaryKey as $column ) {
$pk[$column] = $row->$column;
}
return $pk;
}
/**
* @return array The most recently fetched set of rows from the database
*/
public function current() {
return $this->current;
}
/**
* @return integer 0-indexed count of the page number fetched
*/
public function key() {
return $this->key;
}
/**
* Reset the iterator to the begining of the table.
*/
public function rewind() {
$this->key = -1; // self::next() will turn this into 0
$this->current = array();
$this->next();
}
/**
* @return boolean True when the iterator is in a valid state
*/
public function valid() {
return (bool) $this->current;
}
/**
* Fetch the next set of rows from the database.
*/
public function next() {
$res = $this->db->select(
$this->table,
$this->fetchColumns,
$this->buildConditions(),
__METHOD__,
array(
'LIMIT' => $this->batchSize,
'ORDER BY' => $this->orderBy,
)
);
// The iterator is converted to an array because in addition to returning it
// in self::current() we need to use the end value in self::buildConditions()
$this->current = iterator_to_array( $res );
$this->key++;
}
/**
* Uses the primary key list and the maximal result row from the previous iteration to build
* an SQL condition sufficient for selecting the next page of results. All except the final
* key use `=` conditions while the final key uses a `>` condition
*
* Example output:
* array( '( foo = 42 AND bar > 7 ) OR ( foo > 42 )' )
*
* @return array The SQL conditions necessary to select the next set of rows in the batched query
*/
protected function buildConditions() {
if ( !$this->current ) {
return $this->conditions;
}
$maxRow = end( $this->current );
$maximumValues = array();
foreach ( $this->primaryKey as $column ) {
$maximumValues[$column] = $this->db->addQuotes( $maxRow->$column );
}
$pkConditions = array();
// For example: If we have 3 primary keys
// first run through will generate
// col1 = 4 AND col2 = 7 AND col3 > 1
// second run through will generate
// col1 = 4 AND col2 > 7
// and the final run through will generate
// col1 > 4
while ( $maximumValues ) {
$pkConditions[] = $this->buildGreaterThanCondition( $maximumValues );
array_pop( $maximumValues );
}
$conditions = $this->conditions;
$conditions[] = sprintf( '( %s )', implode( ' ) OR ( ', $pkConditions ) );
return $conditions;
}
/**
* Given an array of column names and their maximum value generate an SQL
* condition where all keys except the last match $quotedMaximumValues
* exactly and the last column is greater than the matching value in $quotedMaximumValues
*
* @param array $quotedMaximumValues The maximum values quoted with $this->db->addQuotes()
* @return string An SQL condition that will select rows where all columns match the
* maximum value exactly except the last column which must be greater than the provided
* maximum value
*/
protected function buildGreaterThanCondition( array $quotedMaximumValues ) {
$keys = array_keys( $quotedMaximumValues );
$lastColumn = end( $keys );
$lastValue = array_pop( $quotedMaximumValues );
$conditions = array();
foreach ( $quotedMaximumValues as $column => $value ) {
$conditions[] = "$column = $value";
}
$conditions[] = "$lastColumn > $lastValue";
return implode( ' AND ', $conditions );
}
}

6
includes/DbEchoBackend.php
View file

@ -99,8 +99,7 @@ class MWDbEchoBackend extends MWEchoBackend {
'event_agent_id',
'event_agent_ip',
'event_extra',
'event_page_namespace',
'event_page_title'
'event_page_id'
),
array(
'notification_event=event_id',
@ -119,8 +118,7 @@ class MWDbEchoBackend extends MWEchoBackend {
'event_agent_id',
'event_agent_ip',
'event_extra',
'event_page_namespace',
'event_page_title'
'event_page_id'
),
array(
'eeb_event_id=event_id',

120
includes/schemaUpdate.php Normal file
View file

@ -0,0 +1,120 @@
<?php
/**
* Performs updates required for respecting suppression within echo:
* Updates event_page_id based on event_page_title and event_page_namespace
* Updates extra data for page-linked events to contain page id's
*/
class EchoSuppressionRowUpdateGenerator implements EchoRowUpdateGenerator
{
/**
* @param callable Hack to allow replacing Title::newFromText in tests
*/
protected $newTitleFromText = array( 'Title', 'newFromText' );
/**
* {@inheritDoc}
*/
public function update( $row ) {
$update = $this->updatePageIdFromTitle( $row );
if ( $row->event_extra !== null && $row->event_type === 'page-linked' ) {
$update = $this->updatePageLinkedExtraData( $row, $update );
}
return $update;
}
/**
* Hackish method of mocking Title::newFromText for tests
*
* @param $callable callable
*/
public function setNewTitleFromText( $callable ) {
$this->newTitleFromText = $callable;
}
/**
* Hackish method of mocking Title::newFromText for tests
*
* @param $text string The page name to look up
* @param $defaultNamespace integer The default namespace of the page to look up
* @return Title|null The title located for the text + namespace, or null if invalid
*/
protected function newTitleFromText( $text, $defaultNamespace = NS_MAIN ) {
return call_user_func( $this->newTitleFromText, $text, $defaultNamespace );
}
/**
* Migrates all echo events from having page title and namespace as rows in the table
* to having only a page id in the table. Any event from a page that doesn't have an
* article id gets the title+namespace moved to the event extra data
*
* @param $row stdClass A row from the database
* @return array All updates required for this row
*/
protected function updatePageIdFromTitle( $row ) {
$update = array();
$title = $this->newTitleFromText( $row->event_page_title, $row->event_page_namespace );
if ( $title !== null ) {
$pageId = $title->getArticleId();
if ( $pageId ) {
// If the title has a proper id from the database, store it
$update['event_page_id'] = $pageId;
} else {
// For titles that do not refer to a WikiPage stored in the database
// move the title/namespace into event_extra
$extra = $this->extra( $row );
$extra['page_title'] = $row->event_page_title;
$extra['page_namespace'] = $row->event_page_namespace;
$update['event_extra'] = serialize( $extra );
}
}
return $update;
}
/**
* Updates the extra data for page-linked events to point to the id of the article
* rather than the namespace+title combo.
*
* @param $row stdClass A row from the database
* @return array All updates required for this row
*/
protected function updatePageLinkedExtraData( $row, array $update ) {
$extra = $this->extra( $row, $update );
if ( isset( $extra['link-from-title'], $extra['link-from-namespace'] ) ) {
$title = $this->newTitleFromText( $extra['link-from-title'], $extra['link-from-namespace'] );
unset( $extra['link-from-title'], $extra['link-from-namespace'] );
// Link from page is always from a content page, if null or no article id it was
// somehow invalid
if ( $title !== null && $title->getArticleId() ) {
$extra['link-from-page-id'] = $title->getArticleId();
}
$update['event_extra'] = serialize( $extra );
}
return $update;
}
/**
* Return the extra data for a row, if an update wants to change the
* extra data returns that updated data rather than the origional. If
* no extra data exists returns array()
*
* @param $row stdClass The database row being updated
* @param $update array Updates that need to be applied to the database row
* @return array The event extra data
*/
protected function extra( $row, array $update = array() ) {
if ( isset( $update['event_extra'] ) ) {
return unserialize( $update['event_extra'] );
} elseif ( $row->event_extra ) {
return unserialize( $row->event_extra );
}
return array();
}
}

62
maintenance/updateEchoSchemaForSuppression.php Normal file
View file

@ -0,0 +1,62 @@
<?php
/**
* Update event_page_id in echo_event based on event_page_title and
* event_page_namespace
*
* @ingroup Maintenance
*/
require_once ( getenv( 'MW_INSTALL_PATH' ) !== false
? getenv( 'MW_INSTALL_PATH' ) . '/maintenance/Maintenance.php'
: __DIR__ . '/../../../maintenance/Maintenance.php' );
/**
* Maintenance script that populates the event_page_id column of echo_event
*
* @ingroup Maintenance
*/
class UpdateEchoSchemaForSuppression extends Maintenance {
/**
* @var $table string The table to update
*/
protected $table = 'echo_event';
/**
* @var $idField string The primary key column of the table to update
*/
protected $idField = 'event_id';
public function __construct() {
parent::__construct();
$this->setBatchSize( 500 );
}
public function execute() {
global $wgEchoCluster;
$reader = new EchoBatchRowIterator( MWEchoDbFactory::getDB( DB_SLAVE ), $this->table, $this->idField, $this->mBatchSize );
$reader->addConditions( array(
"event_page_title IS NOT NULL",
"event_page_id" => null,
) );
$updater = new EchoBatchRowUpdate(
$reader,
new EchoBatchRowWriter( MWEchoDbFactory::getDB( DB_MASTER ), $this->table, $wgEchoCluster ),
new EchoSuppressionRowUpdateGenerator
);
$updater->setOutput( array( $this, '__internalOutput' ) );
$updater->execute();
}
/**
* Internal use only. parent::output() is a protected method, only way to access it from
* a callback in php5.3 is to make a public function. In 5.4 can replace with a Closure.
*/
public function __internalOutput( $text ) {
$this->output( $text );
}
}
$maintClass = 'UpdateEchoSchemaForSuppression'; // Tells it to run the class
require_once ( RUN_MAINTENANCE_IF_MAIN );

26
model/Event.php
View file

@ -145,8 +145,6 @@ class EchoEvent {
'event_variant' => $this->variant,
);
$row['event_extra'] = $this->serializeExtra();
if ( $this->agent ) {
if ( $this->agent->isAnon() ) {
$row['event_agent_ip'] = $this->agent->getName();
@ -156,10 +154,20 @@ class EchoEvent {
}
if ( $this->title ) {
$row['event_page_namespace'] = $this->title->getNamespace();
$row['event_page_title'] = $this->title->getDBkey();
$pageId = $this->title->getArticleId();
if ( $pageId ) {
$row['event_page_id'] = $pageId;
} else {
if ( $this->extra === null ) {
$this->extra = array();
}
$this->extra['page_namespace'] = $this->title->getNamespace();
$this->extra['page_title'] = $this->title->getDBkey();
}
}
$row['event_extra'] = $this->serializeExtra();
$this->id = $wgEchoBackend->createEvent( $row );
}
@ -190,11 +198,19 @@ class EchoEvent {
$this->agent = User::newFromName( $row->event_agent_ip, false );
}
if ( $row->event_page_title !== null ) {
if ( $row->event_page_id ) {
$this->title = Title::newFromId( $row->event_page_id );
} elseif ( isset( $row->event_page_title ) ) {
// BC compat with orig Echo deployment
$this->title = Title::makeTitleSafe(
$row->event_page_namespace,
$row->event_page_title
);
} elseif ( isset( $this->extra['page_title'] ) ) {
$this->title = Title::makeTitleSafe(
$this->extra['page_namespace'],
$this->extra['page_title']
);
}
}

238
tests/BatchRowUpdateTest.php Normal file
View file

@ -0,0 +1,238 @@
<?php
require_once __DIR__ . "/../includes/BatchRowUpdate.php";
/**
* Tests for BatchRowUpdate and its components
*/
class BatchRowUpdateTest extends MediaWikiTestCase {
public function testWriterBasicFunctionality() {
$db = $this->mockDb();
$writer = new EchoBatchRowWriter( $db, 'echo_event' );
$updates = array(
self::mockUpdate( array( 'something' => 'changed' ) ),
self::mockUpdate( array( 'otherthing' => 'changed' ) ),
self::mockUpdate( array( 'and' => 'something', 'else' => 'changed' ) ),
);
$db->expects( $this->exactly( count( $updates ) ) )
->method( 'update' );
$writer->write( $updates );
}
static protected function mockUpdate( array $changes ) {
static $i = 0;
return array(
'primaryKey' => array( 'event_id' => $i++ ),
'changes' => $changes,
);
}
public function testReaderBasicIterate() {
$db = $this->mockDb();
$batchSize = 2;
$reader = new EchoBatchRowIterator( $db, 'some_table', 'id_field', $batchSize );
$response = $this->genSelectResult( $batchSize, /*numRows*/ 5, function() {
static $i = 0;
return array( 'id_field' => ++$i );
} );
$db->expects( $this->exactly( count( $response ) ) )
->method( 'select' )
->will( $this->consecutivelyReturnFromSelect( $response ) );
$pos = 0;
foreach ( $reader as $rows ) {
$this->assertEquals( $response[$pos], $rows, "Testing row in position $pos" );
$pos++;
}
// -1 is because the final array() marks the end and isnt included
$this->assertEquals( count( $response ) - 1, $pos );
}
static public function provider_readerGetPrimaryKey() {
$row = array(
'id_field' => 42,
'some_col' => 'dvorak',
'other_col' => 'samurai',
);
return array(
array(
'Must return single column pk when requested',
array( 'id_field' => 42 ),
$row
),
array(
'Must return multiple column pks when requested',
array( 'id_field' => 42, 'other_col' => 'samurai' ),
$row
),
);
}
/**
* @dataProvider provider_readerGetPrimaryKey
*/
public function testReaderGetPrimaryKey( $message, array $expected, array $row ) {
$reader = new EchoBatchRowIterator( $this->mockDb(), 'some_table', array_keys( $expected ), 8675309 );
$this->assertEquals( $expected, $reader->extractPrimaryKeys( (object) $row ), $message );
}
static public function provider_readerSetFetchColumns() {
return array(
array(
'Must merge primary keys into select conditions',
// Expected column select
array( 'foo', 'bar' ),
// primary keys
array( 'foo' ),
// setFetchColumn
array( 'bar' )
),
array(
'Must not merge primary keys into the all columns selector',
// Expected column select
array( '*' ),
// primary keys
array( 'foo' ),
// setFetchColumn
array( '*' ),
),
array(
'Must not duplicate primary keys into column selector',
// Expected column select.
// TODO: figure out how to only assert the array_values portion and not the keys
array( 0 => 'foo', 1 => 'bar', 3 => 'baz' ),
// primary keys
array( 'foo', 'bar', ),
// setFetchColumn
array( 'bar', 'baz' ),
),
);
}
/**
* @dataProvider provider_readerSetFetchColumns
*/
public function testReaderSetFetchColumns( $message, array $columns, array $primaryKeys, array $fetchColumns ) {
$db = $this->mockDb();
$db->expects( $this->once() )
->method( 'select' )
->with( 'some_table', $columns ) // only testing second parameter of DatabaseBase::select
->will( $this->returnValue( new ArrayIterator( array() ) ) );
$reader = new EchoBatchRowIterator( $db, 'some_table', $primaryKeys, 22 );
$reader->setFetchColumns( $fetchColumns );
// triggers first database select
$reader->rewind();
}
static public function provider_readerSelectConditions() {
return array(
array(
"With single primary key must generate id > 'value'",
// Expected second iteration
array( "( id_field > '3' )" ),
// Primary key(s)
'id_field',
),
array(
'With multiple primary keys the first conditions must use >= and the final condition must use >',
// Expected second iteration
array( "( id_field = '3' AND foo > '103' ) OR ( id_field > '3' )" ),
// Primary key(s)
array( 'id_field', 'foo' ),
),
);
}
/**
* Slightly hackish to use reflection, but asserting different parameters
* to consecutive calls of DatabaseBase::select in phpunit is error prone
*
* @dataProvider provider_readerSelectConditions
*/
public function testReaderSelectConditionsMultiplePrimaryKeys( $message, $expectedSecondIteration, $primaryKeys, $batchSize = 3 ) {
$results = $this->genSelectResult( $batchSize, $batchSize * 3, function() {
static $i = 0, $j = 100, $k = 1000;
return array( 'id_field' => ++$i, 'foo' => ++$j, 'bar' => ++$k );
} );
$db = $this->mockDbConsecutiveSelect( $results );
$conditions = array( 'bar' => 42, 'baz' => 'hai' );
$reader = new EchoBatchRowIterator( $db, 'some_table', $primaryKeys, $batchSize );
$reader->addConditions( $conditions );
$buildConditions = new ReflectionMethod( $reader, 'buildConditions' );
$buildConditions->setAccessible( true );
// On first iteration only the passed conditions must be used
$this->assertEquals( $conditions, $buildConditions->invoke( $reader ),
'First iteration must return only the conditions passed in addConditions' );
$reader->rewind();
// Second iteration must use the maximum primary key of last set
$this->assertEquals(
$conditions + $expectedSecondIteration,
$buildConditions->invoke( $reader ),
$message
);
}
protected function mockDbConsecutiveSelect( array $retvals ) {
$db = $this->mockDb();
$db->expects( $this->any() )
->method( 'select' )
->will( $this->consecutivelyReturnFromSelect( $retvals ) );
$db->expects( $this->any() )
->method( 'addQuotes' )
->will( $this->returnCallback( function( $value ) {
return "'$value'"; // not real quoting: doesn't matter in test
} ) );
return $db;
}
protected function consecutivelyReturnFromSelect( array $results ) {
$retvals = array();
foreach ( $results as $rows ) {
// The DatabaseBase::select method returns iterators, so we do too.
$retvals[] = $this->returnValue( new ArrayIterator( $rows ) );
}
return call_user_func_array( array( $this, 'onConsecutiveCalls' ), $retvals );
}
protected function genSelectResult( $batchSize, $numRows, $rowGenerator ) {
$res = array();
for ( $i = 0; $i < $numRows; $i += $batchSize ) {
$rows = array();
for ( $j = 0; $j < $batchSize && $i + $j < $numRows; $j++ ) {
$rows [] = (object) call_user_func( $rowGenerator );
}
$res[] = $rows;
}
$res[] = array(); // termination condition requires empty result for last row
return $res;
}
protected function mockDb() {
// Cant mock from DatabaseType or DatabaseBase, they dont
// have the full gamut of methods
return $this->getMock( 'DatabaseMysql' );
}
}

127
tests/SupressionMaintenanceTest.php Normal file
View file

@ -0,0 +1,127 @@
<?php
class SuppressionMaintenanceTest extends MediaWikiTestCase {
public static function provider_updateRow() {
$input = array(
'event_id' => 2,
'event_type' => 'mention',
'event_variant' => null,
'event_agent_id' => 3,
'event_agent_ip' => null,
'event_page_title' => null,
'event_page_namespace' => null,
'event_page_extra' => null,
'event_extra' => null,
'event_page_id' => null,
);
return array(
array( 'Unrelated row must result in no update', array(), $input ),
array(
'Page title and namespace for non-existant page must move into event_extra',
array( // expected update
'event_extra' => serialize( array(
'page_title' => 'Yabba Dabba Do',
'page_namespace' => NS_MAIN
) ),
),
array( // input row
'event_page_title' => 'Yabba Dabba Do',
'event_page_namespace' => NS_MAIN,
) + $input,
),
array(
'Page title and namespace for existing page must be result in update to event_page_id',
array( // expected update
'event_page_id' => 42,
),
array( // input row
'event_page_title' => 'Mount Rushmore',
'event_page_namespace' => NS_MAIN,
) + $input,
self::attachTitleFor( 42, 'Mount Rushmore', NS_MAIN )
),
array(
'When updating non-existant page must keep old extra data',
array( // expected update
'event_extra' => serialize( array(
'foo' => 'bar',
'page_title' => 'Yabba Dabba Do',
'page_namespace' => NS_MAIN
) ),
),
array( // input row
'event_page_title' => 'Yabba Dabba Do',
'event_page_namespace' => NS_MAIN,
'event_extra' => serialize( array( 'foo' => 'bar' ) ),
) + $input,
),
array(
'Must update link-from-title/namespace to link-from-page-id for page-linked events',
array( // expected update
'event_extra' => serialize( array( 'link-from-page-id' => 99 ) ),
),
array( //input row
'event_type' => 'page-linked',
'event_extra' => serialize( array(
'link-from-title' => 'Horse',
'link-from-namespace' => NS_USER_TALK
) ),
) + $input,
self::attachTitleFor( 99, 'Horse', NS_USER_TALK )
),
array(
'Must perform both generic update and page-linked update at same time',
array( // expected update
'event_extra' => serialize( array( 'link-from-page-id' => 8675309 ) ),
'event_page_id' => 8675309,
),
array( //input row
'event_type' => 'page-linked',
'event_extra' => serialize( array(
'link-from-title' => 'Jenny',
'link-from-namespace' => NS_MAIN,
) ),
'event_page_title' => 'Jenny',
'event_page_namespace' => NS_MAIN,
) + $input,
self::attachTitleFor( 8675309, 'Jenny', NS_MAIN ),
),
);
}
protected static function attachTitleFor( $id, $providedText, $providedNamespace ) {
return function( $test, $gen ) use ( $id, $providedText, $providedNamespace ) {
$title = $test->getMock( 'Title' );
$title->expects( $test->any() )
->method( 'getArticleId' )
->will( $test->returnValue( $id ) );
$titles = array( $providedNamespace => array( $providedText => $title ) );
$gen->setNewTitleFromText( function( $text, $defaultNamespace ) use( $titles ) {
if ( isset( $titles[$defaultNamespace][$text] ) ) {
return $titles[$defaultNamespace][$text];
}
return Title::newFromText( $text, $defaultNamespace );
} );
};
}
/**
* @dataProvider provider_updateRow
*/
public function testUpdateRow( $message, $expected, $input, $callable = null ) {
$gen = new EchoSuppressionRowUpdateGenerator;
if ( $callable ) {
call_user_func( $callable, $this, $gen );
}
$update = $gen->update( (object) $input );
$this->assertEquals( $expected, $update, $message );
}
}