Skip to content

Commit

Permalink
ID5 User Id Module: update a/b testing to be user based not request b…
Browse files Browse the repository at this point in the history 
…ased (#6281)

* convert A/B testing to be user-based, rather than request-based

* update docs to say a/b testing is user based, not request based
  • Loading branch information
@smenzer
smenzer authored Feb 8, 2021
1 parent b7dcdf9 commit 2640d08
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 41 deletions.
51 changes: 35 additions & 16 deletions modules/id5IdSystem.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ const NB_EXP_DAYS = 30;
export const ID5_STORAGE_NAME = 'id5id';
export const ID5_PRIVACY_STORAGE_NAME = `${ID5_STORAGE_NAME}_privacy`;
const LOCAL_STORAGE = 'html5';
const ABTEST_RESOLUTION = 10000;

// order the legacy cookie names in reverse priority order so the last
// cookie in the array is the most preferred to use
Expand Down Expand Up @@ -58,27 +59,18 @@ export const id5IdSubmodule = {
}

// check for A/B testing configuration and hide ID if in Control Group
let abConfig = getAbTestingConfig(config);
let controlGroup = false;
if (
abConfig.enabled === true &&
(!utils.isNumber(abConfig.controlGroupPct) ||
abConfig.controlGroupPct < 0 ||
abConfig.controlGroupPct > 1)
) {
const abConfig = getAbTestingConfig(config);
const controlGroup = isInControlGroup(universalUid, abConfig.controlGroupPct);
if (abConfig.enabled === true && typeof controlGroup === 'undefined') {
// A/B Testing is enabled, but configured improperly, so skip A/B testing
utils.logError('User ID - ID5 submodule: A/B Testing controlGroupPct must be a number >= 0 and <= 1! Skipping A/B Testing');
} else if (
abConfig.enabled === true &&
Math.random() < abConfig.controlGroupPct
) {
} else if (abConfig.enabled === true && controlGroup === true) {
// A/B Testing is enabled and user is in the Control Group, so do not share the ID5 ID
utils.logInfo('User ID - ID5 submodule: A/B Testing Enabled - request is in the Control Group, so the ID5 ID is NOT exposed');
utils.logInfo('User ID - ID5 submodule: A/B Testing Enabled - user is in the Control Group, so the ID5 ID is NOT exposed');
universalUid = linkType = 0;
controlGroup = true;
} else if (abConfig.enabled === true) {
// A/B Testing is enabled but user is not in the Control Group, so ID5 ID is shared
utils.logInfo('User ID - ID5 submodule: A/B Testing Enabled - request is NOT in the Control Group, so the ID5 ID is exposed');
utils.logInfo('User ID - ID5 submodule: A/B Testing Enabled - user is NOT in the Control Group, so the ID5 ID is exposed');
}

let responseObj = {
Expand All @@ -91,7 +83,7 @@ export const id5IdSubmodule = {
};

if (abConfig.enabled === true) {
utils.deepSetValue(responseObj, 'id5id.ext.abTestingControlGroup', controlGroup);
utils.deepSetValue(responseObj, 'id5id.ext.abTestingControlGroup', (typeof controlGroup === 'undefined' ? false : controlGroup));
}

return responseObj;
Expand Down Expand Up @@ -296,4 +288,31 @@ function getAbTestingConfig(config) {
return (config && config.params && config.params.abTesting) || { enabled: false };
}

/**
* Return a consistant random number between 0 and ABTEST_RESOLUTION-1 for this user
* Falls back to plain random if no user provided
* @param {string} userId
* @returns {number}
*/
function abTestBucket(userId) {
if (userId) {
return ((utils.cyrb53Hash(userId) % ABTEST_RESOLUTION) + ABTEST_RESOLUTION) % ABTEST_RESOLUTION;
} else {
return Math.floor(Math.random() * ABTEST_RESOLUTION);
}
}

/**
* Return a consistant boolean if this user is within the control group ratio provided
* @param {string} userId
* @param {number} controlGroupPct - Ratio [0,1] of users expected to be in the control group
* @returns {boolean}
*/
export function isInControlGroup(userId, controlGroupPct) {
if (!utils.isNumber(controlGroupPct) || controlGroupPct < 0 || controlGroupPct > 1) {
return undefined;
}
return abTestBucket(userId) < controlGroupPct * ABTEST_RESOLUTION;
}

submodule('userId', id5IdSubmodule);
4 changes: 2 additions & 2 deletions modules/id5IdSystem.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# ID5 Universal ID

The ID5 Universal ID is a shared, neutral identifier that publishers and ad tech platforms can use to recognise users even in environments where 3rd party cookies are not available. The ID5 Universal ID is designed to respect users' privacy choices and publishers’ preferences throughout the advertising value chain. For more information about the ID5 Universal ID and detailed integration docs, please visit [our documentation](https://console.id5.io/docs/public/prebid). We also recommend that you sign up for our [release notes](https://id5.io/universal-id/release-notes) to stay up-to-date with any changes to the implementation of the ID5 Universal ID in Prebid.
The ID5 Universal ID is a shared, neutral identifier that publishers and ad tech platforms can use to recognise users even in environments where 3rd party cookies are not available. The ID5 Universal ID is designed to respect users' privacy choices and publishers’ preferences throughout the advertising value chain. For more information about the ID5 Universal ID and detailed integration docs, please visit [our documentation](https://wiki.id5.io/x/BIAZ). We also recommend that you sign up for our [release notes](https://id5.io/universal-id/release-notes) to stay up-to-date with any changes to the implementation of the ID5 Universal ID in Prebid.

## ID5 Universal ID Registration

Expand Down Expand Up @@ -65,4 +65,4 @@ pbjs.setConfig({

Publishers may want to test the value of the ID5 ID with their downstream partners. While there are various ways to do this, A/B testing is a standard approach. Instead of publishers manually enabling or disabling the ID5 User ID Module based on their control group settings (which leads to fewer calls to ID5, reducing our ability to recognize the user), we have baked this in to our module directly.

To turn on A/B Testing, simply edit the configuration (see above table) to enable it and set what percentage of requests you would like to set for the control group. The control group is the set of requests where an ID5 ID will not be exposed in to bid adapters or in the various user id functions available on the `pbjs` global. An additional value of `ext.abTestingControlGroup` will be set to `true` or `false` that can be used to inform reporting systems that the request was in the control group or not. It's important to note that the control group is request based, and not user based. In other words, from one page view to another, a user may be in or out of the control group.
To turn on A/B Testing, simply edit the configuration (see above table) to enable it and set what percentage of users you would like to set for the control group. The control group is the set of user where an ID5 ID will not be exposed in to bid adapters or in the various user id functions available on the `pbjs` global. An additional value of `ext.abTestingControlGroup` will be set to `true` or `false` that can be used to inform reporting systems that the user was in the control group or not. It's important to note that the control group is user based, and not request based. In other words, from one page view to another, a user will always be in or out of the control group.
91 changes: 68 additions & 23 deletions test/spec/modules/id5IdSystem_spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ import {
expDaysStr,
nbCacheName,
getNbFromCache,
storeNbInCache
storeNbInCache,
isInControlGroup
} from 'modules/id5IdSystem.js';
import { init, requestBidsHook, setSubmoduleRegistry, coreStorage } from 'modules/userId/index.js';
import { config } from 'src/config.js';
Expand Down Expand Up @@ -477,6 +478,9 @@ describe('ID5 ID System', function() {
{ enabled: true, controlGroupPct: true }
];
testInvalidAbTestingConfigsWithError.forEach((testAbTestingConfig) => {
it('should be undefined if ratio is invalid', () => {
expect(isInControlGroup('userId', testAbTestingConfig.controlGroupPct)).to.be.undefined;
});
it('should error if config is invalid, and always return an ID', function () {
testConfig.params.abTesting = testAbTestingConfig;
let decoded = id5IdSubmodule.decode(ID5_STORED_OBJ, testConfig);
Expand All @@ -493,6 +497,9 @@ describe('ID5 ID System', function() {
{ enabled: false, controlGroupPct: true }
];
testInvalidAbTestingConfigsWithoutError.forEach((testAbTestingConfig) => {
it('should be undefined if ratio is invalid', () => {
expect(isInControlGroup('userId', testAbTestingConfig.controlGroupPct)).to.be.undefined;
});
it('should not error if config is invalid but A/B testing is off, and always return an ID', function () {
testConfig.params.abTesting = testAbTestingConfig;
let decoded = id5IdSubmodule.decode(ID5_STORED_OBJ, testConfig);
Expand All @@ -509,6 +516,9 @@ describe('ID5 ID System', function() {
{ enabled: true, controlGroupPct: 1 }
];
testValidConfigs.forEach((testAbTestingConfig) => {
it('should not be undefined if ratio is valid', () => {
expect(isInControlGroup('userId', testAbTestingConfig.controlGroupPct)).to.not.be.undefined;
});
it('should not error if config is valid', function () {
testConfig.params.abTesting = testAbTestingConfig;
id5IdSubmodule.decode(ID5_STORED_OBJ, testConfig);
Expand Down Expand Up @@ -555,34 +565,69 @@ describe('ID5 ID System', function() {
randStub.restore();
});

it('should expose ID when A/B testing is off', function () {
testConfig.params.abTesting = {
enabled: false,
controlGroupPct: 0.5
};
describe('IsInControlGroup', function () {
it('Nobody is in a 0% control group', function () {
expect(isInControlGroup('dsdndskhsdks', 0)).to.be.false;
expect(isInControlGroup('3erfghyuijkm', 0)).to.be.false;
expect(isInControlGroup('', 0)).to.be.false;
expect(isInControlGroup(undefined, 0)).to.be.false;
});

let decoded = id5IdSubmodule.decode(ID5_STORED_OBJ, testConfig);
expect(decoded).to.deep.equal(expectedDecodedObjectWithIdAbOff);
});
it('Everybody is in a 100% control group', function () {
expect(isInControlGroup('dsdndskhsdks', 1)).to.be.true;
expect(isInControlGroup('3erfghyuijkm', 1)).to.be.true;
expect(isInControlGroup('', 1)).to.be.true;
expect(isInControlGroup(undefined, 1)).to.be.true;
});

it('should expose ID when not in control group', function () {
testConfig.params.abTesting = {
enabled: true,
controlGroupPct: 0.1
};
it('Being in the control group must be consistant', function () {
const inControlGroup = isInControlGroup('dsdndskhsdks', 0.5);
expect(inControlGroup === isInControlGroup('dsdndskhsdks', 0.5)).to.be.true;
expect(inControlGroup === isInControlGroup('dsdndskhsdks', 0.5)).to.be.true;
expect(inControlGroup === isInControlGroup('dsdndskhsdks', 0.5)).to.be.true;
});

let decoded = id5IdSubmodule.decode(ID5_STORED_OBJ, testConfig);
expect(decoded).to.deep.equal(expectedDecodedObjectWithIdAbOn);
it('Control group ratio must be within a 10% error on a large sample', function () {
let nbInControlGroup = 0;
const sampleSize = 100;
for (let i = 0; i < sampleSize; i++) {
nbInControlGroup = nbInControlGroup + (isInControlGroup('R$*df' + i, 0.5) ? 1 : 0);
}
expect(nbInControlGroup).to.be.greaterThan(sampleSize / 2 - sampleSize / 10);
expect(nbInControlGroup).to.be.lessThan(sampleSize / 2 + sampleSize / 10);
});
});

it('should not expose ID when in control group', function () {
testConfig.params.abTesting = {
enabled: true,
controlGroupPct: 0.5
};
describe('Decode', function() {
it('should expose ID when A/B testing is off', function () {
testConfig.params.abTesting = {
enabled: false,
controlGroupPct: 0.5
};

let decoded = id5IdSubmodule.decode(ID5_STORED_OBJ, testConfig);
expect(decoded).to.deep.equal(expectedDecodedObjectWithoutIdAbOn);
let decoded = id5IdSubmodule.decode(ID5_STORED_OBJ, testConfig);
expect(decoded).to.deep.equal(expectedDecodedObjectWithIdAbOff);
});

it('should expose ID when no one is in control group', function () {
testConfig.params.abTesting = {
enabled: true,
controlGroupPct: 0
};

let decoded = id5IdSubmodule.decode(ID5_STORED_OBJ, testConfig);
expect(decoded).to.deep.equal(expectedDecodedObjectWithIdAbOn);
});

it('should not expose ID when everyone is in control group', function () {
testConfig.params.abTesting = {
enabled: true,
controlGroupPct: 1
};

let decoded = id5IdSubmodule.decode(ID5_STORED_OBJ, testConfig);
expect(decoded).to.deep.equal(expectedDecodedObjectWithoutIdAbOn);
});
});
});
});
Expand Down

0 comments on commit 2640d08

Please sign in to comment.