From 27a995d5a239f0181ee57233f44b353b82cb9c01 Mon Sep 17 00:00:00 2001 From: David Lynch Date: Wed, 13 Jan 2021 00:38:11 -0600 Subject: [PATCH] A/B test bucketing for beta enrollment If DiscussionToolsABTest is enabled (set to `all` or a feature), logged in users who have never used the tool before will be assigned to an a/b test bucket. If they're in the test bucket, they get the feature enabled. If they manually set their beta feature preference, we don't override that but do maintain their bucket for logging purposes. Bug: T268191 Change-Id: I9c4d60e9f9aaef11afa7f8661b9c49130dde3ffa --- extension.json | 7 ++++- includes/Hooks.php | 68 ++++++++++++++++++++++++++++++++++++++++++++-- modules/logger.js | 8 ++++++ 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/extension.json b/extension.json index a8905674e..9b4432db6 100644 --- a/extension.json +++ b/extension.json @@ -342,7 +342,8 @@ "DefaultUserOptions": { "discussiontools-editmode": "", "discussiontools-newtopictool": 1, - "discussiontools-replytool": 1 + "discussiontools-replytool": 1, + "discussiontools-abtest": "" }, "config": { "DiscussionToolsEnable": { @@ -353,6 +354,10 @@ "value": false, "description": "Make DiscussionTools a BetaFeature." }, + "DiscussionToolsABTest": { + "value": false, + "description": "A/B test DiscussionTools features for logged in users. false, 'replytool', 'newtopictool', or 'all'" + }, "DiscussionTools_replytool": { "value": "default", "description": "Override availability of DiscussionTools reply tool. 'default', 'available', or 'unavailable'." diff --git a/includes/Hooks.php b/includes/Hooks.php index c18a46dab..2ddfadacc 100644 --- a/includes/Hooks.php +++ b/includes/Hooks.php @@ -86,11 +86,23 @@ class Hooks { // No feature-specific override found. + if ( $dtConfig->get( 'DiscussionToolsBeta' ) ) { + $betaenabled = $optionsLookup->getOption( $user, 'discussiontools-betaenable', -1 ); + if ( $betaenabled !== -1 ) { + // betaenable doesn't have a default value, so we can check + // for it being unset like this. If the user has explicitly + // enabled or disabled it, we should immediatly return that. + return $betaenabled; + } + // Otherwise, being in the "test" group for this feature means + // it's effectively beta-enabled. + return self::determineUserABTestBucket( $user, $feature ) === 'test'; + } + // Assume that if BetaFeature is turned off, or user has it enabled, that // some features are available. // If this isn't the case, then DiscussionToolsEnable should have been set to false. - return !$dtConfig->get( 'DiscussionToolsBeta' ) || - $optionsLookup->getOption( $user, 'discussiontools-betaenable' ); + return true; } /** @@ -115,6 +127,45 @@ class Hooks { ); } + /** + * Work out the A/B test bucket for the current user + * + * Checks whether the A/B test is enabled and whether the user is enrolled + * in it; if they're eligible and not enrolled, it will enroll them. + * + * @param User $user + * @param string|null $feature Feature to check for: 'replytool' or 'newtopictool'. + * Null will check for any DT feature. + * @return string 'test' if in the test group, 'control' if in the control group, or '' if they've + * never been in the test + */ + private static function determineUserABTestBucket( $user, $feature = null ) : string { + $services = MediaWikiServices::getInstance(); + $optionsManager = $services->getUserOptionsManager(); + $dtConfig = $services->getConfigFactory()->makeConfig( 'discussiontools' ); + + $abtest = $dtConfig->get( 'DiscussionToolsABTest' ); + if ( + !$user->isAnon() && + ( $abtest == 'all' || ( $feature && $abtest == $feature ) ) + ) { + // The A/B test is enabled, and the user is qualified to be in the + // test by being logged in. + $abstate = $optionsManager->getOption( $user, 'discussiontools-abtest' ); + if ( !$abstate && $optionsManager->getOption( $user, 'discussiontools-editmode' ) === '' ) { + // Assign the user to a group. This is only being done to + // users who have never used the tool before, for which we're + // using the presence of discussiontools-editmode as a proxy, + // as it should be set as soon as the user interacts with the tool. + $abstate = $user->getId() % 2 == 0 ? 'test' : 'control'; + $optionsManager->setOption( $user, 'discussiontools-abtest', $abstate ); + $optionsManager->saveOptions( $user ); + } + return $abstate; + } + return ''; + } + /** * Check if the tools are available for a given title * @@ -223,6 +274,16 @@ class Hooks { $editor ); } + $dtConfig = $services->getConfigFactory()->makeConfig( 'discussiontools' ); + $abstate = $dtConfig->get( 'DiscussionToolsABTest' ) ? + $optionsLookup->getOption( $user, 'discussiontools-abtest' ) : + false; + if ( $abstate ) { + $output->addJsConfigVars( + 'wgDiscussionToolsABTestBucket', + $abstate + ); + } } } @@ -310,6 +371,9 @@ class Hooks { $preferences['discussiontools-showadvanced'] = [ 'type' => 'api', ]; + $preferences['discussiontools-abtest'] = [ + 'type' => 'api', + ]; $preferences['discussiontools-editmode'] = [ 'type' => 'api', diff --git a/modules/logger.js b/modules/logger.js index 1c16a6ac4..031ee37e6 100644 --- a/modules/logger.js +++ b/modules/logger.js @@ -186,6 +186,10 @@ mw.loader.using( 'ext.eventLogging' ).done( function () { } } + if ( mw.config.get( 'wgDiscussionToolsABTestBucket' ) ) { + data.bucket = mw.config.get( 'wgDiscussionToolsABTestBucket' ); + } + $.extend( data, session ); if ( trackdebug ) { @@ -215,6 +219,10 @@ mw.loader.using( 'ext.eventLogging' ).done( function () { editor_interface: session.editor_interface }; + if ( mw.config.get( 'wgDiscussionToolsABTestBucket' ) ) { + event.bucket = mw.config.get( 'wgDiscussionToolsABTestBucket' ); + } + if ( trackdebug ) { log( topic, event, schemaVisualEditorFeatureUse.defaults ); } else {