From 303a5019fc8e0e098cec152402071a151efdea16 Mon Sep 17 00:00:00 2001 From: Thiemo Kreuz Date: Wed, 5 Jun 2019 16:36:27 +0200 Subject: [PATCH] Make failing Bucketing test more robust This test started failing on us for no apparent reason. Example: Ic95f7b0 https://integration.wikimedia.org/ci/job/quibble-vendor-mysql-php72-docker/11469/consoleFull Output: "test control group is about 25% (30.8%)" It appears like the bucketing is not really done based on an actual random number generator, but based on a hash that contains the session ID. If this session ID is not really a random number, the hash might not be random enough as well, but be skewed towards one or the other direction. We propose to take the normal distribution into account and change the narrow +/- 10% margin to +/- 20%. Change-Id: Ib163f1de4f9cff27aaf8dbc81189315142ff0d8a --- tests/qunit/skins.minerva.scripts/AB.test.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/qunit/skins.minerva.scripts/AB.test.js b/tests/qunit/skins.minerva.scripts/AB.test.js index d450d0a74..a57e1beee 100644 --- a/tests/qunit/skins.minerva.scripts/AB.test.js +++ b/tests/qunit/skins.minerva.scripts/AB.test.js @@ -38,18 +38,18 @@ } assert.strictEqual( - ( userBuckets.unsampled / maxUsers > 0.4 ) && - ( userBuckets.unsampled / maxUsers < 0.6 ), + ( userBuckets.unsampled / maxUsers > 0.3 ) && + ( userBuckets.unsampled / maxUsers < 0.7 ), true, 'test unsampled group is about 50% (' + userBuckets.unsampled / 10 + '%)' ); assert.strictEqual( - ( userBuckets.control / maxUsers > 0.2 ) && - ( userBuckets.control / maxUsers < 0.3 ), + ( userBuckets.control / maxUsers > 0.1 ) && + ( userBuckets.control / maxUsers < 0.4 ), true, 'test control group is about 25% (' + userBuckets.control / 10 + '%)' ); assert.strictEqual( - ( userBuckets.treatment / maxUsers > 0.2 ) && - ( userBuckets.treatment / maxUsers < 0.3 ), + ( userBuckets.treatment / maxUsers > 0.1 ) && + ( userBuckets.treatment / maxUsers < 0.4 ), true, 'test new treatment group is about 25% (' + userBuckets.treatment / 10 + '%)' ); } );