mirror of
https://gerrit.wikimedia.org/r/mediawiki/skins/MinervaNeue
synced 2024-11-16 18:58:45 +00:00
303a5019fc
This test started failing on us for no apparent reason. Example: Ic95f7b0 https://integration.wikimedia.org/ci/job/quibble-vendor-mysql-php72-docker/11469/consoleFull Output: "test control group is about 25% (30.8%)" It appears like the bucketing is not really done based on an actual random number generator, but based on a hash that contains the session ID. If this session ID is not really a random number, the hash might not be random enough as well, but be skewed towards one or the other direction. We propose to take the normal distribution into account and change the narrow +/- 10% margin to +/- 20%. Change-Id: Ib163f1de4f9cff27aaf8dbc81189315142ff0d8a
57 lines
1.5 KiB
JavaScript
57 lines
1.5 KiB
JavaScript
( function ( M ) {
|
|
|
|
var AB = M.require( 'skins.minerva.scripts/AB' ),
|
|
util = M.require( 'mobile.startup' ).util,
|
|
defaultConfig = {
|
|
testName: 'WME.MinervaABTest',
|
|
samplingRate: 0.5,
|
|
sessionId: mw.user.generateRandomSessionId()
|
|
};
|
|
|
|
QUnit.module( 'Minerva AB-test' );
|
|
|
|
QUnit.test( 'Bucketing test', function ( assert ) {
|
|
var userBuckets = {
|
|
unsampled: 0,
|
|
control: 0,
|
|
treatment: 0
|
|
},
|
|
maxUsers = 1000,
|
|
bucketingTest,
|
|
config,
|
|
i;
|
|
|
|
for ( i = 0; i < maxUsers; i++ ) {
|
|
config = util.extend( {}, defaultConfig, {
|
|
sessionId: mw.user.generateRandomSessionId()
|
|
} );
|
|
bucketingTest = new AB( config );
|
|
if ( bucketingTest.isControl() ) {
|
|
++userBuckets.control;
|
|
} else if ( bucketingTest.isTreatment() ) {
|
|
++userBuckets.treatment;
|
|
} else if ( !bucketingTest.isSampled() ) {
|
|
++userBuckets.unsampled;
|
|
} else {
|
|
throw new Error( 'Unknown bucket!' );
|
|
}
|
|
}
|
|
|
|
assert.strictEqual(
|
|
( userBuckets.unsampled / maxUsers > 0.3 ) &&
|
|
( userBuckets.unsampled / maxUsers < 0.7 ),
|
|
true, 'test unsampled group is about 50% (' + userBuckets.unsampled / 10 + '%)' );
|
|
|
|
assert.strictEqual(
|
|
( userBuckets.control / maxUsers > 0.1 ) &&
|
|
( userBuckets.control / maxUsers < 0.4 ),
|
|
true, 'test control group is about 25% (' + userBuckets.control / 10 + '%)' );
|
|
|
|
assert.strictEqual(
|
|
( userBuckets.treatment / maxUsers > 0.1 ) &&
|
|
( userBuckets.treatment / maxUsers < 0.4 ),
|
|
true, 'test new treatment group is about 25% (' + userBuckets.treatment / 10 + '%)' );
|
|
} );
|
|
|
|
}( mw.mobileFrontend ) );
|