mediawiki-skins-MinervaNeue/tests/qunit/skins.minerva.scripts/AB.test.js
Thiemo Kreuz 303a5019fc Make failing Bucketing test more robust
This test started failing on us for no apparent reason.

Example: Ic95f7b0
https://integration.wikimedia.org/ci/job/quibble-vendor-mysql-php72-docker/11469/consoleFull

Output: "test control group is about 25% (30.8%)"

It appears like the bucketing is not really done based on an actual
random number generator, but based on a hash that contains the session
ID. If this session ID is not really a random number, the hash might
not be random enough as well, but be skewed towards one or the other
direction.

We propose to take the normal distribution into account and change the
narrow +/- 10% margin to +/- 20%.

Change-Id: Ib163f1de4f9cff27aaf8dbc81189315142ff0d8a
2019-07-05 21:38:58 +00:00

57 lines
1.5 KiB
JavaScript

( function ( M ) {
var AB = M.require( 'skins.minerva.scripts/AB' ),
util = M.require( 'mobile.startup' ).util,
defaultConfig = {
testName: 'WME.MinervaABTest',
samplingRate: 0.5,
sessionId: mw.user.generateRandomSessionId()
};
QUnit.module( 'Minerva AB-test' );
QUnit.test( 'Bucketing test', function ( assert ) {
var userBuckets = {
unsampled: 0,
control: 0,
treatment: 0
},
maxUsers = 1000,
bucketingTest,
config,
i;
for ( i = 0; i < maxUsers; i++ ) {
config = util.extend( {}, defaultConfig, {
sessionId: mw.user.generateRandomSessionId()
} );
bucketingTest = new AB( config );
if ( bucketingTest.isControl() ) {
++userBuckets.control;
} else if ( bucketingTest.isTreatment() ) {
++userBuckets.treatment;
} else if ( !bucketingTest.isSampled() ) {
++userBuckets.unsampled;
} else {
throw new Error( 'Unknown bucket!' );
}
}
assert.strictEqual(
( userBuckets.unsampled / maxUsers > 0.3 ) &&
( userBuckets.unsampled / maxUsers < 0.7 ),
true, 'test unsampled group is about 50% (' + userBuckets.unsampled / 10 + '%)' );
assert.strictEqual(
( userBuckets.control / maxUsers > 0.1 ) &&
( userBuckets.control / maxUsers < 0.4 ),
true, 'test control group is about 25% (' + userBuckets.control / 10 + '%)' );
assert.strictEqual(
( userBuckets.treatment / maxUsers > 0.1 ) &&
( userBuckets.treatment / maxUsers < 0.4 ),
true, 'test new treatment group is about 25% (' + userBuckets.treatment / 10 + '%)' );
} );
}( mw.mobileFrontend ) );