Merge branch 'master' of github.com:mozilla-services/pageshot

server/src/ab-tests.js

@@ -0,0 +1,163 @@
+// Note: these get turned into Test objects:
+let allTests = {
+  autoOpenSharePanel: {
+    description: "Open the share panel immediately after shot creation",
+    gaField: "cd3",
+    shotField: "cd4",
+    version: 2,
+    exclude: ["highlightButtonOnInstall", "myShotsDisplay"],
+    options: [
+      {name: "autoopen", probability: 0.9}
+    ]
+  },
+  highlightButtonOnInstall: {
+    description: "Highlight the Page Shot button when Page Shot is installed",
+    gaField: "cd5",
+    version: 2,
+    exclude: ["autoOpenSharePanel", "myShotsDisplay"],
+    options: [
+      {name: "uitour", probability: 0.9}
+    ]
+  },
+  myShotsDisplay: {
+    description: "Show My Shots button/CTA differently",
+    gaField: "cd6",
+    version: 2,
+    exclude: ["highlightButtonOnInstall", "autoOpenSharePanel"],
+    options: [
+      {name: "intropopup", probability: 0.9},
+      {name: "blink", probability: 0.1}
+    ]
+  }
+};
+
+let deprecatedTests = ["exampleTest"];
+
+class Test {
+  constructor(options) {
+    let requiredFields = ['name', 'gaField', 'description', 'version', 'options'];
+    let allowedFields = requiredFields.concat(['shotField', 'exclude']);
+    for (let required of requiredFields) {
+      if (! (required in options)) {
+        throw new Error(`Missing constructor field: ${required}`);
+      }
+    }
+    for (let found in options) {
+      if (! allowedFields.includes(found)) {
+        throw new Error(`Unexpected constructor field: ${found}`);
+      }
+    }
+    Object.assign(this, options);
+  }
+
+  updateTest(tests) {
+    if (tests[this.name] && tests[this.name].version >= this.version) {
+      return;
+    }
+    if (this.shouldExclude(tests)) {
+      tests[this.name] = this.testWithValue("control");
+    } else {
+      let prob = getRandom();
+      let setAny = false;
+      for (let option of this.options) {
+        if (prob < option.probability) {
+          tests[this.name] = this.testWithValue(option.name)
+          setAny = true;
+          break;
+        }
+        prob -= option.probability;
+      }
+      if (! setAny) {
+        tests[this.name] = this.testWithValue("control");
+      }
+    }
+  }
+
+  testWithValue(value) {
+    let result = {value, gaField: this.gaField, version: this.version};
+    if (this.shotField) {
+      result.shotField = this.shotField;
+    }
+    return result;
+  }
+
+  shouldExclude(tests) {
+    for (let testName of this.exclude) {
+      if (tests[testName] && tests[testName].value !== "control") {
+        return true;
+      }
+    }
+    return false;
+  }
+
+}
+
+exports.updateAbTests = function (tests) {
+  for (let testName in allTests) {
+    allTests[testName].updateTest(tests);
+  }
+  for (let testName of deprecatedTests) {
+    if (testName in tests) {
+      delete tests[testName];
+    }
+  }
+  return tests;
+};
+
+let randomSeq;
+
+exports.setRandomSequenceForTesting = function (seq) {
+  seq = seq || undefined;
+  if (seq) {
+    if (! Array.isArray(seq)) {
+      throw new Error("setRandomSequenceForTesting([]) can only take an Array");
+    }
+    for (let i of seq) {
+      if (typeof i !== "number" || i < 0 || i >= 1) {
+        throw new Error(`Bad item in array: ${JSON.stringify(i)}`);
+      }
+    }
+  }
+  randomSeq = seq;
+};
+
+exports.setAllTestsForTesting = function (x) {
+  if (x === undefined) {
+    allTests = origAllTests;
+    return;
+  }
+  setTests(x);
+};
+
+function setTests(tests) {
+  let seenFields = {};
+  allTests = {};
+  for (let testName in tests) {
+    let test = new Test(Object.assign({name: testName}, tests[testName]));
+    allTests[testName] = test;
+    if (seenFields[test.gaField]) {
+      throw new Error(`Two tests with field ${test.gaField}`);
+    }
+    seenFields[test.gaField] = true;
+    if (test.shotField) {
+      if (seenFields[test.shotField]) {
+        throw new Error(`Two tests with field ${test.shotField}`);
+      }
+      seenFields[test.shotField] = true;
+    }
+  }
+}
+
+setTests(allTests);
+let origAllTests = allTests;
+
+function getRandom() {
+  if (randomSeq) {
+    if (! randomSeq.length) {
+      throw new Error("Ran out of testing random numbers");
+    }
+    let next = randomSeq.shift();
+    return next;
+  }
+  return Math.random();
+}

server/src/b64.js

@@ -0,0 +1,10 @@
+exports.b64EncodeJson = function (obj) {
+  obj = JSON.stringify(obj);
+  let buffer = new Buffer(obj, "binary");
+  return buffer.toString("base64");
+};
+
+exports.b64DecodeJson = function (b64) {
+  let string = new Buffer(b64, "base64").toString("binary");
+  return JSON.parse(string);
+};

test/test-ab-test.js

@@ -0,0 +1,100 @@
+/* globals describe, it, before, after */
+
+const assert = require("assert");
+const abTests = require("../server/src/ab-tests.js");
+
+describe("Test Page Shot", function () {
+
+  before(() => {
+    abTests.setAllTestsForTesting({
+      simpleTest: {
+        gaField: "cd3",
+        description: "Test without conflicts",
+        version: 1,
+        exclude: [],
+        options: [
+          {
+            name: "bright",
+            probability: 0.3
+          },
+          {
+            name: "dark",
+            probability: 0.3
+          }
+        ]
+      },
+      breakEverythingTest: {
+        gaField: "cd4",
+        shotField: "cd5",
+        description: "Test with conflicts",
+        version: 2,
+        exclude: ["simpleTest"],
+        options: [
+          {
+            name: "fireworks",
+            probability: 0.1
+          },
+          {
+            name: "strobe",
+            probability: 0.1
+          }
+        ]
+      }
+    });
+    abTests.setRandomSequenceForTesting([]);
+  });
+
+  after(() => {
+    abTests.setAllTestsForTesting(undefined);
+    abTests.setRandomSequenceForTesting(undefined);
+  });
+
+  it("should set all to control when low probability", () => {
+    abTests.setRandomSequenceForTesting([0.9, 0.9]);
+    let tests = abTests.updateAbTests({});
+    assert.deepEqual(tests, {
+      simpleTest: {value: "control", gaField: "cd3", version: 1},
+      breakEverythingTest: {value: "control", gaField: "cd4", shotField: "cd5", version: 2}
+    });
+  });
+
+  it("should not overwrite existing values", () => {
+    let tests = abTests.updateAbTests({
+      simpleTest: {value: "control", gaField: "cd3", version: 1},
+      breakEverythingTest: {value: "control", gaField: "cd4", shotField: "cd5", version: 2}
+    });
+    assert.deepEqual(tests, {
+      simpleTest: {value: "control", gaField: "cd3", version: 1},
+      breakEverythingTest: {value: "control", gaField: "cd4", shotField: "cd5", version: 2}
+    });
+  });
+
+  it("should exclude the second test when the first is selected", () => {
+    let tests = abTests.updateAbTests({
+      simpleTest: {value: "bright", gaField: "cd3", version: 1}
+    });
+    assert.deepEqual(tests, {
+      simpleTest: {value: "bright", gaField: "cd3", version: 1},
+      breakEverythingTest: {value: "control", gaField: "cd4", shotField: "cd5", version: 2}
+    });
+    abTests.setRandomSequenceForTesting([0.35]);
+    tests = abTests.updateAbTests({});
+    assert.deepEqual(tests, {
+      simpleTest: {value: "dark", gaField: "cd3", version: 1},
+      breakEverythingTest: {value: "control", gaField: "cd4", shotField: "cd5", version: 2}
+    });
+  });
+
+  it("should try to take someone out of control when version is bumped", () => {
+    abTests.setRandomSequenceForTesting([0.15]);
+    let tests = abTests.updateAbTests({
+      simpleTest: {value: "control", gaField: "cd3", version: 1},
+      breakEverythingTest: {value: "control", gaField: "cd4", shotField: "cd5", version: 1}
+    });
+    assert.deepEqual(tests, {
+      simpleTest: {value: "control", gaField: "cd3", version: 1},
+      breakEverythingTest: {value: "strobe", gaField: "cd4", shotField: "cd5", version: 2}
+    });
+  });
+
+});