From a8b12fbd3cf3c78770559142fac01b9aecf42f89 Mon Sep 17 00:00:00 2001
From: "Dustin J. Mitchell" <dustin@mozilla.com>
Date: Tue, 2 Apr 2019 12:58:40 -0400
Subject: [PATCH 01/12] Bug 1517865 - factor runWithFakeTime into
 tc-lib-testing

---
 libraries/testing/README.md              | 28 +++++++++++-
 libraries/testing/src/poll.js            |  2 +-
 libraries/testing/src/sleep.js           |  8 ----
 libraries/testing/src/testing.js         | 20 ++++-----
 libraries/testing/src/time.js            | 55 ++++++++++++++++++++++++
 services/queue/test/bucket_test.js       |  2 +-
 services/queue/test/claimtask_test.js    |  4 +-
 services/queue/test/claimwork_test.js    |  4 +-
 services/queue/test/deadline_test.js     | 16 +++----
 services/queue/test/dependency_test.js   | 37 ++++++++--------
 services/queue/test/helper.js            | 54 -----------------------
 services/queue/test/queueservice_test.js | 16 +++----
 services/queue/test/retry_test.js        |  4 +-
 13 files changed, 132 insertions(+), 118 deletions(-)
 delete mode 100644 libraries/testing/src/sleep.js
 create mode 100644 libraries/testing/src/time.js

diff --git a/libraries/testing/README.md b/libraries/testing/README.md
index 1c09286e194..1a5c4e15957 100644
--- a/libraries/testing/README.md
+++ b/libraries/testing/README.md
@@ -312,8 +312,8 @@ This function assumes the following config values:
 
 And assumes that the `exports` argument has a `load` function corresponding to a sticky loader.
 
-Utilities
----------
+Time
+----
 
 ### Sleep
 
@@ -323,6 +323,30 @@ The `sleep` function returns a promise that resolves after a delay.
 poorly-isolated tests. Consider writing the tests to use a "fake" clock or to
 poll for the expected state.
 
+### Fake Time
+
+When testing functionality that involves timers, it is helpful to be able to simulate the rapid passage of time.
+The `testing.runWithFakeTime(<fn>, {mock, maxTime, ...})` uses [zurvan](https://github.com/tlewowski/zurvan) to do just that.
+It is used to wrap an argument to a Mocha `test` function, avoiding interfering with Mocha's timers:
+```js
+test('noun should verb', runWithFakeTime(async function() {
+  ...
+}, {
+  mock,
+  maxTime: 60000,
+}));
+```
+
+The `maxTime` option is the total amount of simulated time to spend running the test; it defaults to 30 seconds.
+
+The `mock` option is for use with `mockSuite` and can be omitted otherwise.
+Fake time is only used when mocking; in a real situation, we are interacting with real services and must use the same clock they do.
+
+Any other options are passed directly to zurvan.
+
+Utilities
+---------
+
 ### Poll
 
 The `poll` function will repeatedly call a function that returns a promise
diff --git a/libraries/testing/src/poll.js b/libraries/testing/src/poll.js
index 94d2002dc2f..eb48913ac06 100644
--- a/libraries/testing/src/poll.js
+++ b/libraries/testing/src/poll.js
@@ -1,6 +1,6 @@
 const Debug = require('debug');
 const debug = Debug('taskcluster-lib-testing:poll');
-const sleep = require('./sleep');
+const {sleep} = require('./time');
 
 /**
  * Poll a function that returns a promise until the promise is resolved without
diff --git a/libraries/testing/src/sleep.js b/libraries/testing/src/sleep.js
deleted file mode 100644
index cf612ab3f90..00000000000
--- a/libraries/testing/src/sleep.js
+++ /dev/null
@@ -1,8 +0,0 @@
-/** Return promise that is resolved in `delay` ms */
-const sleep = function(delay) {
-  return new Promise(function(accept) {
-    setTimeout(accept, delay);
-  });
-};
-
-module.exports = sleep;
diff --git a/libraries/testing/src/testing.js b/libraries/testing/src/testing.js
index b80f42cf23d..df03b8fcbd8 100644
--- a/libraries/testing/src/testing.js
+++ b/libraries/testing/src/testing.js
@@ -1,12 +1,10 @@
-exports.schemas = require('./schemas');
-exports.fakeauth = require('./fakeauth');
-exports.stickyLoader = require('./stickyloader');
-exports.Secrets = require('./secrets');
-exports.poll = require('./poll');
-exports.sleep = require('./sleep');
-exports.withEntity = require('./with-entity');
-exports.suiteName = require('./suite-name');
-
-exports.createMockAuthServer = () => {
-  throw new Error('No longer available; use fakeauth instead');
+module.exports = {
+  schemas: require('./schemas'),
+  fakeauth: require('./fakeauth'),
+  stickyLoader: require('./stickyloader'),
+  Secrets: require('./secrets'),
+  poll: require('./poll'),
+  ...require('./time'),
+  withEntity: require('./with-entity'),
+  suiteName: require('./suite-name'),
 };
diff --git a/libraries/testing/src/time.js b/libraries/testing/src/time.js
new file mode 100644
index 00000000000..4d6c29e6891
--- /dev/null
+++ b/libraries/testing/src/time.js
@@ -0,0 +1,55 @@
+const zurvan = require('zurvan');
+const timers = require('timers');
+
+/** Return promise that is resolved in `delay` ms */
+exports.sleep = function(delay) {
+  return new Promise(function(accept) {
+    setTimeout(accept, delay);
+  });
+};
+
+exports.runWithFakeTime = (fn, {mock=true, maxTime=30000, ...zurvanOptions}={}) => {
+  if (!mock) {
+    // if not mocking, we can't use fake time as it will cause all sorts
+    // of timeouts to occur immediately
+    return fn;
+  }
+  return async function wrap() {
+    await zurvan.interceptTimers({
+      systemTime: new Date(),
+      denyImplicitTimer: true,
+      throwOnInvalidClearTimer: false, // superagent does this..
+      rejectOnCallbackFailure: true,
+      fakeNodeDedicatedTimers: false, // so we can call a real timers.setImmediate
+      ...zurvanOptions,
+    });
+
+    let finished, err;
+    this.slow(maxTime);
+    fn.apply(this, []).then(
+      () => {
+        finished = true;
+      },
+      e => {
+        finished = true;
+        err = e;
+      });
+
+    // intermingle setImmediate calls with advanceTime calls, so that things zurvan cannot
+    // successfully fake (like JS files internal to Node) get a chance to run.
+    let time = maxTime;
+    while (time > 0 && !finished) {
+      await zurvan.advanceTime(100);
+      time -= 100;
+      await new Promise(resolve => timers.setImmediate(resolve));
+    }
+
+    await zurvan.releaseTimers();
+    if (err) {
+      throw err;
+    }
+    if (!finished) {
+      throw new Error(`test case not finished after faked passage of ${maxTime}ms`);
+    }
+  };
+};
diff --git a/services/queue/test/bucket_test.js b/services/queue/test/bucket_test.js
index fea0a3bf181..ebd7e547235 100644
--- a/services/queue/test/bucket_test.js
+++ b/services/queue/test/bucket_test.js
@@ -17,7 +17,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['aws'], function(mock, skipping)
   }
 
   let bucket;
-  setup(async function() {
+  setup('load bucket', async function() {
     if (!skipping()) {
       bucket = await helper.load('publicArtifactBucket');
     }
diff --git a/services/queue/test/claimtask_test.js b/services/queue/test/claimtask_test.js
index 704696fde66..343def703fc 100644
--- a/services/queue/test/claimtask_test.js
+++ b/services/queue/test/claimtask_test.js
@@ -37,7 +37,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     },
   });
 
-  test('can claimTask', helper.runWithFakeTime(async function() {
+  test('can claimTask', testing.runWithFakeTime(async function() {
     const taskId = slugid.v4();
 
     debug('### Creating task');
@@ -98,7 +98,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     assume(takenUntil4.getTime()).is.greaterThan(takenUntil.getTime() - 1);
     assume(takenUntil4.getTime()).is.greaterThan(takenUntil2.getTime() - 1);
     assume(takenUntil4.getTime()).is.greaterThan(takenUntil3.getTime() - 1);
-  }, mock));
+  }, {mock}));
 
   test('claimTask is idempotent', async () => {
     const taskId = slugid.v4();
diff --git a/services/queue/test/claimwork_test.js b/services/queue/test/claimwork_test.js
index fe3d41f86e6..20efef39b62 100644
--- a/services/queue/test/claimwork_test.js
+++ b/services/queue/test/claimwork_test.js
@@ -35,7 +35,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     };
   };
 
-  test('claimWork from empty queue', helper.runWithFakeTime(async function() {
+  test('claimWork from empty queue', testing.runWithFakeTime(async function() {
     helper.scopes(
       'queue:claim-work:no-provisioner-extended-extended/' + workerType,
       'queue:worker-id:my-worker-group-extended-extended/my-worker-extended-extended',
@@ -49,7 +49,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     });
     assert(result.tasks.length === 0, 'Did not expect any claims');
     assert(new Date() - started >= 20 * 1000, 'Expected 20s sleep');
-  }, mock, 25000));
+  }, {mock, maxTime: 25000}));
 
   test('claimWork requires scopes', async () => {
     // wrong provisionerId scope
diff --git a/services/queue/test/deadline_test.js b/services/queue/test/deadline_test.js
index 96ea49fca36..107cf1d1a79 100644
--- a/services/queue/test/deadline_test.js
+++ b/services/queue/test/deadline_test.js
@@ -36,7 +36,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     return {taskId: slugid.v4(), task};
   };
 
-  test('Resolve unscheduled task deadline', helper.runWithFakeTime(async () => {
+  test('Resolve unscheduled task deadline', testing.runWithFakeTime(async () => {
     const {taskId, task} = makeTask();
 
     debug('### Define task');
@@ -74,9 +74,9 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     debug('### Validate task status');
     const r2 = await helper.queue.status(taskId);
     assume(r2.status.state).equals('exception');
-  }, mock));
+  }, {mock}));
 
-  test('Resolve pending task deadline', helper.runWithFakeTime(async () => {
+  test('Resolve pending task deadline', testing.runWithFakeTime(async () => {
     const {taskId, task} = makeTask();
 
     debug('### Creating task');
@@ -105,9 +105,9 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     debug('### Validate task status');
     const r2 = await helper.queue.status(taskId);
     assume(r2.status.state).deep.equals('exception');
-  }, mock));
+  }, {mock}));
 
-  test('Resolve running task deadline', helper.runWithFakeTime(async () => {
+  test('Resolve running task deadline', testing.runWithFakeTime(async () => {
     const {taskId, task} = makeTask();
 
     debug('### Creating task');
@@ -143,9 +143,9 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     debug('### Validate task status');
     const r3 = await helper.queue.status(taskId);
     assume(r3.status.state).deep.equals('exception');
-  }, mock));
+  }, {mock}));
 
-  test('Resolve completed task by deadline (no change)', helper.runWithFakeTime(async () => {
+  test('Resolve completed task by deadline (no change)', testing.runWithFakeTime(async () => {
     const {taskId, task} = makeTask();
 
     debug('### Creating task');
@@ -179,5 +179,5 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     debug('### Validate task status');
     const r4 = await helper.queue.status(taskId);
     assume(r4.status).deep.equals(r3.status);
-  }, mock));
+  }, {mock}));
 });
diff --git a/services/queue/test/dependency_test.js b/services/queue/test/dependency_test.js
index b3c9ede7759..513766d2482 100644
--- a/services/queue/test/dependency_test.js
+++ b/services/queue/test/dependency_test.js
@@ -32,7 +32,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     },
   });
 
-  test('taskA <- taskB', helper.runWithFakeTime(async () => {
+  test('taskA <- taskB', testing.runWithFakeTime(async () => {
     let taskIdA = slugid.v4();
     let taskIdB = slugid.v4();
 
@@ -136,9 +136,9 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     }
 
     await helper.stopPollingService();
-  }, mock));
+  }, {mock}));
 
-  test('taskA <- taskB, taskC, taskD, taskE', helper.runWithFakeTime(async () => {
+  test('taskA <- taskB, taskC, taskD, taskE', testing.runWithFakeTime(async () => {
     let taskIdA = slugid.v4();
     let taskIdB = slugid.v4();
     let taskIdC = slugid.v4();
@@ -222,9 +222,9 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     assume(tids).contains(taskIdE);
 
     await helper.stopPollingService();
-  }, mock));
+  }, {mock}));
 
-  test('taskA, taskB <- taskC && taskA <- taskD', helper.runWithFakeTime(async () => {
+  test('taskA, taskB <- taskC && taskA <- taskD', testing.runWithFakeTime(async () => {
     let taskIdA = slugid.v4();
     let taskIdB = slugid.v4();
     let taskIdC = slugid.v4();
@@ -291,9 +291,9 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
       Infinity);
 
     await helper.stopPollingService();
-  }, mock));
+  }, {mock}));
 
-  test('taskA <- taskA (self-dependency)', helper.runWithFakeTime(async () => {
+  test('taskA <- taskA (self-dependency)', testing.runWithFakeTime(async () => {
     let taskIdA = slugid.v4();
     let taskA = _.defaults({
       dependencies: [taskIdA],
@@ -314,9 +314,9 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
       workerGroup: 'my-worker-group-extended-extended',
       workerId: 'my-worker-extended-extended',
     });
-  }, mock));
+  }, {mock}));
 
-  test('taskA, taskB <- taskB (self-dependency)', helper.runWithFakeTime(async () => {
+  test('taskA, taskB <- taskB (self-dependency)', testing.runWithFakeTime(async () => {
     let taskIdA = slugid.v4();
     let taskIdB = slugid.v4();
 
@@ -348,7 +348,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     let r3 = await helper.queue.status(taskIdB);
     assume(r3.status.state).equals('unscheduled');
     helper.checkNoNextMessage('task-pending'); // because of the self-dep
-  }, mock));
+  }, {mock}));
 
   test('taskX <- taskA (missing dependency)', async () => {
     let taskIdA = slugid.v4();
@@ -377,7 +377,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     );
   });
 
-  test('taskA <- taskB (reportFailed)', helper.runWithFakeTime(async () => {
+  test('taskA <- taskB (reportFailed)', testing.runWithFakeTime(async () => {
     let taskIdA = slugid.v4();
     let taskIdB = slugid.v4();
 
@@ -409,9 +409,9 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     assume(r3.status.state).equals('unscheduled');
 
     await helper.stopPollingService();
-  }, mock));
+  }, {mock}));
 
-  test('taskA <- taskB (cancelTask)', helper.runWithFakeTime(async () => {
+  test('taskA <- taskB (cancelTask)', testing.runWithFakeTime(async () => {
     let taskIdA = slugid.v4();
     let taskIdB = slugid.v4();
 
@@ -439,9 +439,9 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     assume(r3.status.state).equals('unscheduled');
 
     await helper.stopPollingService();
-  }, mock));
+  }, {mock}));
 
-  test('taskA <- taskB (reportFailed w. all-resolved)', helper.runWithFakeTime(async () => {
+  test('taskA <- taskB (reportFailed w. all-resolved)', testing.runWithFakeTime(async () => {
     let taskIdA = slugid.v4();
     let taskIdB = slugid.v4();
 
@@ -479,9 +479,9 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
       Infinity);
 
     await helper.stopPollingService();
-  }, mock));
+  }, {mock}));
 
-  test('expiration of relationships', helper.runWithFakeTime(async () => {
+  test('expiration of relationships', testing.runWithFakeTime(async () => {
     const taskIdA = slugid.v4();
     const taskA = _.defaults({
       dependencies: [taskIdA],
@@ -533,6 +533,5 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     const r10 = await TaskRequirement.load({taskId: taskIdB, requiredTaskId: taskIdB}, true);
     assert(r9, 'Expected TaskDependency');
     assert(r10, 'Expected TaskRequirement');
-  }, mock));
-
+  }, {mock}));
 });
diff --git a/services/queue/test/helper.js b/services/queue/test/helper.js
index a90cff7ca7c..2e3bd6250ce 100644
--- a/services/queue/test/helper.js
+++ b/services/queue/test/helper.js
@@ -10,8 +10,6 @@ const mockAwsS3 = require('mock-aws-s3');
 const nock = require('nock');
 const FakeBlobStore = require('./fake_blob_store');
 const {fakeauth, stickyLoader, Secrets, withEntity} = require('taskcluster-lib-testing');
-const zurvan = require('zurvan');
-const timers = require('timers');
 const {FakeClient} = require('taskcluster-lib-pulse');
 
 const helper = module.exports;
@@ -65,58 +63,6 @@ setup(async function() {
   helper.monitor.reset();
 });
 
-/**
- * helper.runWithFakeTime(<fn>, <time>), will run async function <fn> for <time>
- * fake milliseconds.  It is intended to wrap the function argument to Mocha's `test`.
- *
- * Fake time is only used when mock is true; in a real situation, we are interacting with
- * real services and must use the same clock they do.
- */
-helper.runWithFakeTime = (fn, mock, maxTime=30000) => {
-  if (!mock) {
-    // if not mocking, we can't use fake time as it will cause all sorts
-    // of timeouts to occur immediately
-    return fn;
-  }
-  return async function wrap() {
-    await zurvan.interceptTimers({
-      systemTime: new Date(),
-      denyImplicitTimer: true,
-      throwOnInvalidClearTimer: false, // superagent does this..
-      rejectOnCallbackFailure: true,
-      fakeNodeDedicatedTimers: false, // so we can call a real timers.setImmediate
-    });
-
-    let finished, err;
-    this.slow(maxTime);
-    fn.apply(this, []).then(
-      () => {
-        finished = true;
-      },
-      e => {
-        finished = true;
-        err = e;
-      });
-
-    // intermingle setImmediate calls with advanceTime calls, so that things zurvan cannot
-    // successfully fake (like JS files internal to Node) get a chance to run.
-    let time = maxTime;
-    while (time > 0 && !finished) {
-      await zurvan.advanceTime(100);
-      time -= 100;
-      await new Promise(resolve => timers.setImmediate(resolve));
-    }
-
-    await zurvan.releaseTimers();
-    if (err) {
-      throw err;
-    }
-    if (!finished) {
-      throw new Error(`test case not finished after faked passage of ${maxTime}ms`);
-    }
-  };
-};
-
 /**
  * Set up to use mock-aws-s3 for S3 operations when mocking.
  */
diff --git a/services/queue/test/queueservice_test.js b/services/queue/test/queueservice_test.js
index 0e16788df5e..a48cbe1ccae 100644
--- a/services/queue/test/queueservice_test.js
+++ b/services/queue/test/queueservice_test.js
@@ -61,7 +61,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['azure'], function(mock, skipping
   const workerType = 'no-worker';
   const provisionerId = slugid.v4(); // make a unique provisionerId
 
-  test('putDeadlineMessage, pollDeadlineQueue', helper.runWithFakeTime(async () => {
+  test('putDeadlineMessage, pollDeadlineQueue', testing.runWithFakeTime(async () => {
     const taskId = slugid.v4();
     const taskGroupId = slugid.v4();
     const schedulerId = slugid.v4();
@@ -87,9 +87,9 @@ helper.secrets.mockSuite(testing.suiteName(), ['azure'], function(mock, skipping
       });
       assert(foundTaskId, 'Expected to see taskId at some point');
     }, Infinity);
-  }, mock));
+  }, {mock}));
 
-  test('putClaimMessage, pollClaimQueue', helper.runWithFakeTime(async () => {
+  test('putClaimMessage, pollClaimQueue', testing.runWithFakeTime(async () => {
     const taskId = slugid.v4();
     const takenUntil = new Date(new Date().getTime() + 2 * 1000);
     debug('Putting message with taskId: %s', taskId);
@@ -113,9 +113,9 @@ helper.secrets.mockSuite(testing.suiteName(), ['azure'], function(mock, skipping
       });
       assert(foundTaskId, 'Expected to see taskId at some point');
     }, Infinity);
-  }, mock));
+  }, {mock}));
 
-  test('putResolvedMessage, pollResolvedQueue', helper.runWithFakeTime(async () => {
+  test('putResolvedMessage, pollResolvedQueue', testing.runWithFakeTime(async () => {
     const taskId = slugid.v4();
     const taskGroupId = slugid.v4();
     const schedulerId = slugid.v4();
@@ -140,10 +140,10 @@ helper.secrets.mockSuite(testing.suiteName(), ['azure'], function(mock, skipping
       });
       assert(foundTaskId, 'Expected to see taskId at some point');
     }, Infinity);
-  }, mock));
+  }, {mock}));
 
   // not supported for mock QueueService
-  test('put, poll, release, poll, delete (priority: lowest)', helper.runWithFakeTime(async () => {
+  test('put, poll, release, poll, delete (priority: lowest)', testing.runWithFakeTime(async () => {
     const taskId = slugid.v4();
     const runId = 0;
     const task = {
@@ -197,7 +197,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['azure'], function(mock, skipping
 
     // Remove message
     await message.remove();
-  }, mock));
+  }, {mock}));
 
   test('countPendingMessages', async () => {
     const count = await queueService.countPendingMessages(
diff --git a/services/queue/test/retry_test.js b/services/queue/test/retry_test.js
index 797458745f9..58e6ac2d086 100644
--- a/services/queue/test/retry_test.js
+++ b/services/queue/test/retry_test.js
@@ -31,7 +31,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     },
   };
 
-  test('createTask, claimTask, claim-expired, retry, ...', helper.runWithFakeTime(async () => {
+  test('createTask, claimTask, claim-expired, retry, ...', testing.runWithFakeTime(async () => {
     const taskId = slugid.v4();
 
     debug('### Creating task');
@@ -98,5 +98,5 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'aws', 'azure'], f
     const r5 = await helper.queue.status(taskId);
     // this time it's exception, since it's out of retries
     assume(r5.status.state).equals('exception');
-  }, mock));
+  }, {mock}));
 });

From d4355a22fe63530ee67d5e301bc5228f593d6521 Mon Sep 17 00:00:00 2001
From: "Dustin J. Mitchell" <dustin@mozilla.com>
Date: Tue, 2 Apr 2019 13:00:08 -0400
Subject: [PATCH 02/12] rename tc-lib-testing entry point to index.js

---
 libraries/testing/package.json                 | 2 +-
 libraries/testing/src/{testing.js => index.js} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename libraries/testing/src/{testing.js => index.js} (100%)

diff --git a/libraries/testing/package.json b/libraries/testing/package.json
index 87ce73444ca..71cd88594bc 100644
--- a/libraries/testing/package.json
+++ b/libraries/testing/package.json
@@ -24,5 +24,5 @@
   "engines": {
     "node": ">=8"
   },
-  "main": "./src/testing.js"
+  "main": "./src/index.js"
 }
diff --git a/libraries/testing/src/testing.js b/libraries/testing/src/index.js
similarity index 100%
rename from libraries/testing/src/testing.js
rename to libraries/testing/src/index.js

From 63a51c35cdb2cb4ea2b72115cf7e9b3bc30ef4d7 Mon Sep 17 00:00:00 2001
From: "Dustin J. Mitchell" <dustin@mozilla.com>
Date: Tue, 2 Apr 2019 14:46:21 -0400
Subject: [PATCH 03/12] refactor tests to be async, instead of calling done

---
 libraries/iterate/README.md            |   7 +-
 libraries/iterate/test/iterate_test.js | 365 +++++++++++--------------
 2 files changed, 161 insertions(+), 211 deletions(-)

diff --git a/libraries/iterate/README.md b/libraries/iterate/README.md
index f2f14248321..3d5f4800f25 100644
--- a/libraries/iterate/README.md
+++ b/libraries/iterate/README.md
@@ -82,10 +82,9 @@ exit with a non-zero exit code when it would otherwise be emitted.
 * `completed`: only when we have a max number of iterations, when we
   finish the last iteration
 * `iteration-start`: when an individual iteration starts
-* `iteration-success`: when an individual iteration completes with
-  success.  provides the value that handler resolves with
-* `iteration-failure`: provides iteration error
-* `iteration-complete`: when an iteration is complete regardless of outcome
+* `iteration-success`: when an individual iteration completes successfully
+* `iteration-failure`: when an individual iteration fails
+* `iteration-complete`: when an iteration completes, regardless of outcome
 * `error`: when the iteration is considered to be concluded and provides
   list of iteration errors.  If there are no handlers and this event is
   emitted, an exception will be thrown in a process.nextTick callback.
diff --git a/libraries/iterate/test/iterate_test.js b/libraries/iterate/test/iterate_test.js
index 08bfbc7c313..bd0aa478160 100644
--- a/libraries/iterate/test/iterate_test.js
+++ b/libraries/iterate/test/iterate_test.js
@@ -3,7 +3,7 @@ let sandbox = require('sinon').createSandbox();
 let assume = require('assume');
 let debug = require('debug')('iterate-test');
 let MonitorManager = require('taskcluster-lib-monitor');
-const testing = require('taskcluster-lib-testing');
+let testing = require('taskcluster-lib-testing');
 
 let possibleEvents = [
   'started',
@@ -28,57 +28,52 @@ class IterateEvents {
     }
   }
 
-  assert(f) {
+  assert() {
     let dl = () => { // dl === dump list
       return `\nExpected: ${JSON.stringify(this.expectedOrder, null, 2)}` +
         `\nActual: ${JSON.stringify(this.orderOfEmission, null, 2)}`;
     };
 
     if (this.orderOfEmission.length !== this.expectedOrder.length) {
-      return f(new Error(`order emitted differs in length from expectation ${dl()}`));
+      throw(new Error(`order emitted differs in length from expectation ${dl()}`));
     }
 
     for (let i = 0 ; i < this.orderOfEmission.length ; i++) {
       if (this.orderOfEmission[i] !== this.expectedOrder[i]) {
-        return f(new Error(`order emitted differs in content ${dl()}`));
+        throw(new Error(`order emitted differs in content ${dl()}`));
       }
     }
 
     debug(`Events Emitted: ${JSON.stringify(this.orderOfEmission)}`);
-    return f();
   }
-
 }
 
 suite(testing.suiteName(), () => {
   let manager;
   let monitor;
 
-  setup(async () => {
+  suiteSetup(async () => {
     manager = new MonitorManager({serviceName: 'iterate'});
     manager.setup({mock: true});
     monitor = manager.monitor();
   });
 
+  suiteTeardown(() => {
+    manager.terminate();
+  });
+
   teardown(() => {
     sandbox.restore();
-    manager.terminate();
   });
 
-  test('should be able to start and stop', done => {
+  test('should be able to start and stop', async function() {
     let iterations = 0;
 
     let i = new subject({
       maxIterationTime: 3000,
       waitTime: 1000,
+      watchDog: 0,
       handler: async (watchdog, state) => {
-        // In order to get the looping stuff to work, I had to stop the
-        // watchdog timer.  This will be tested in the tests for the
-        // Iterate.iterate method
-        watchdog.on('expired', () => {
-          done(new Error('incremental watch dog expiration'));
-        });
-
         debug('iterate!');
         iterations++;
         return 1;
@@ -86,88 +81,74 @@ suite(testing.suiteName(), () => {
       monitor,
     });
 
-    i.on('error', err => {
-      done(err);
-    });
-
-    i.on('stopped', () => {
-      done();
-    });
+    let err = null;
+    i.on('error', e => { err = e; });
+    i.on('stopped', () => { err = new Error('unexpected stopped event'); });
 
     i.start();
 
-    setTimeout(() => {
-      assume(iterations).equals(5);
-      assume(i.keepGoing).is.ok();
-      i.stop();
-      assume(i.keepGoing).is.not.ok();
-      assume(manager.messages.length).equals(5);
-      manager.messages.forEach(message => {
-        assume(message.Fields.status).equals('success');
-      });
-    }, 5000);
+    await testing.sleep(5000);
 
+    assume(err).to.equal(null);
+    assume(iterations).equals(5);
+    assume(i.keepGoing).is.ok();
+    i.stop();
+    assume(i.keepGoing).is.not.ok();
+    assume(manager.messages.length).equals(5);
+    manager.messages.forEach(message => {
+      assume(message.Fields.status).equals('success');
+    });
   });
 
-  test('should stop after correct number of iterations', done => {
+  test('should stop after correct number of iterations', async function() {
     let iterations = 0;
 
     let i = new subject({
       maxIterationTime: 3000,
       waitTime: 10,
       maxIterations: 5,
+      watchDog: 0,
       handler: async (watchdog, state) => {
-        watchdog.on('expired', () => {
-          done(new Error('incremental watch dog expiration'));
-        });
-
         debug('iterate!');
         iterations++;
         return 1;
       },
     });
 
-    i.on('error', err => {
-      done(err);
-    });
-
     i.start();
 
-    i.on('completed', () => {
-      assume(iterations).equals(5);
-      assume(i.keepGoing).is.not.ok();
-      i.stop();
-      done();
+    await new Promise(resolve => {
+      i.on('completed', resolve);
     });
+
+    assume(iterations).equals(5);
+    assume(i.keepGoing).is.not.ok();
+    i.stop();
   });
 
-  test('should emit error when iteration watchdog expires', done => {
+  test('should emit error when iteration watchdog expires', async function() {
     let i = new subject({
       maxIterationTime: 5000,
       watchdogTime: 1000,
       waitTime: 1000,
       maxFailures: 1,
       handler: async (watchdog, state) => {
-        // In order to get the looping stuff to work, I had to stop the
-        // watchdog timer.  This will be tested in the tests for the
-        // Iterate.iterate method
-        i.on('error', err => {
-          debug('correctly getting expired watchdog timer');
-          i.stop();
-          assume(i.keepGoing).is.not.ok();
-          done();
-        });
-        return new Promise((res, rej) => {
-          setTimeout(res, 2000);
-        });
-
+        return testing.sleep(2000);
       },
     });
 
+    let err = null;
+    i.on('error', e => { err = e; });
+
     i.start();
+    await testing.sleep(1500);
+    i.stop();
+
+    assume(i.keepGoing).is.not.ok();
+    assume(err).matches(/watchdog exceeded/);
   });
 
-  test('should emit error when overall iteration limit is hit', done => {
+  test('should emit error when overall iteration limit is hit', async function() {
     let i = new subject({
       maxIterationTime: 1000,
       waitTime: 1000,
@@ -182,16 +163,18 @@ suite(testing.suiteName(), () => {
       },
     });
 
-    i.on('error', err => {
-      i.stop();
-      assume(i.keepGoing).is.not.ok();
-      done();
-    });
+    let err = null;
+    i.on('error', e => { err = e; });
 
     i.start();
+    await testing.sleep(1500);
+    i.stop();
+
+    assume(i.keepGoing).is.not.ok();
+    assume(err).matches(/Iteration exceeded maximum time allowed/);
   });
 
-  test('should emit iteration-failure when async handler fails', done => {
+  test('should emit iteration-failure when async handler fails', async function() {
     let i = new subject({
       maxIterationTime: 1000,
       waitTime: 1000,
@@ -201,16 +184,18 @@ suite(testing.suiteName(), () => {
       },
     });
 
-    i.on('iteration-failure', err => {
-      i.stop();
-      assume(i.keepGoing).is.not.ok();
-      done();
-    });
+    let sawEvent = false;
+    i.on('iteration-failure', () => { sawEvent = true; });
 
     i.start();
+    await testing.sleep(500);
+    i.stop();
+
+    assume(i.keepGoing).is.not.ok();
+    assume(sawEvent).is.ok();
   });
 
-  test('should emit iteration-failure when sync handler fails', done => {
+  test('should emit iteration-failure when sync handler fails', async function() {
     let i = new subject({
       maxIterationTime: 1000,
       waitTime: 1000,
@@ -220,56 +205,59 @@ suite(testing.suiteName(), () => {
       },
     });
 
-    i.on('iteration-failure', err => {
-      i.stop();
-      assume(i.keepGoing).is.not.ok();
-      done();
-    });
+    let sawEvent = false;
+    i.on('iteration-failure', () => { sawEvent = true; });
 
     i.start();
+    await testing.sleep(500);
+    i.stop();
+
+    assume(i.keepGoing).is.not.ok();
+    assume(sawEvent).is.ok();
   });
 
-  test('should emit error when iteration is too quick', done => {
+  test('should emit iteration-failure when iteration is too quick', async function() {
     let i = new subject({
       maxIterationTime: 12000,
       minIterationTime: 10000,
       waitTime: 1000,
+      watchDog: 0,
       handler: async (watchdog, state) => {
-        watchdog.stop();
-        return 1;
+        await testing.sleep(100);
       },
     });
 
+    let iterFailed = false;
+    i.on('iteration-failure', () => { iterFailed = true; });
+
     i.start();
+    await testing.sleep(300);
+    i.stop();
 
-    i.on('error', err => {
-      debug('correctly getting expired watchdog timer');
-      i.stop();
-      assume(i.keepGoing).is.not.ok();
-      done();
-    });
+    assume(iterFailed).is.ok();
   });
 
-  test('should emit error after too many failures', done => {
+  test('should emit error after too many failures', async function() {
     let i = new subject({
       maxIterationTime: 12000,
       maxFailures: 1,
       waitTime: 1000,
       handler: async (watchdog, state) => {
-        return new Promise((res, rej) => {
-          rej(new Error('hi'));
-        });
+        throw new Error('uhoh');
       },
     });
 
-    i.start();
+    let err = null;
+    i.on('error', e => { err = e; });
 
-    i.on('error', err => {
-      i.stop();
-      assume(i.keepGoing).is.not.ok();
-      done();
-    });
+    i.start();
+    await testing.sleep(1500);
+    i.stop();
 
+    assume(i.keepGoing).is.not.ok();
+    // yep, err is an Array..
+    assume(err.length).equals(1);
+    assume(err[0]).matches(/uhoh/);
   });
 
   test('should cause uncaughtException when error event is unhandled', done => {
@@ -309,72 +297,47 @@ suite(testing.suiteName(), () => {
 
   });
 
-  test('should share state between iterations', done => {
+  test('should share state between iterations', async function() {
     let iterations = 0;
-    let v = {a: 1};
 
     let i = new subject({
       maxIterationTime: 3000,
-      waitTime: 1000,
-      maxIterations: 2,
+      waitTime: 10,
+      maxIterations: 20,
       maxFailures: 1,
+      watchDog: 0,
       handler: async (watchdog, state) => {
-        watchdog.on('expired', () => {
-          done(new Error('incremental watch dog expiration'));
-        });
-
-        if (iterations === 0) {
-          assume(state).deeply.equals({});
-          state.v = v;
-        } else if (iterations === 1) {
-          assume(state.v).deeply.equals(v);
-          assume(state.v).equals(v);
-        } else {
-          done(new Error('too many iterations'));
+        if (!state.v) {
+          state.v = {count: 0};
         }
-
-        debug('iterate!');
-        iterations++;
-        return 1;
+        assume(state.v.count === iterations);
+        state.v.count += 1;
+        iterations += 1;
       },
     });
 
-    i.on('error', err => {
-      done(err);
-    });
-
-    i.on('iteration-error', err => {
-      done(err);
-    });
+    let iterFailed = false;
+    i.on('iteration-failure', () => { iterFailed = true; });
 
     i.start();
+    await testing.sleep(3000);
 
-    i.on('completed', () => {
-      assume(iterations).equals(2);
-      assume(i.keepGoing).is.not.ok();
-      i.stop();
-      assume(i.keepGoing).is.not.ok();
-      done();
-    });
+    assume(iterations).equals(20);
+    assume(i.keepGoing).is.not.ok();
+    i.stop();
+    assume(i.keepGoing).is.not.ok();
+    assume(iterFailed).is.not.ok();
   });
 
-  test('should emit correct events for single iteration', done => {
-    let iterations = 0; //eslint-disable-line no-unused-vars
-
+  test('should emit correct events for single iteration', async function() {
     let i = new subject({
       maxIterationTime: 3000,
       waitTime: 1000,
       handler: async (watchdog, state) => {
         debug('iterate!');
-        iterations++;
-        return 1;
       },
     });
 
-    i.on('error', err => {
-      done(err);
-    });
-
     let events = new IterateEvents(i, [
       'started',
       'iteration-start',
@@ -383,35 +346,27 @@ suite(testing.suiteName(), () => {
       'stopped',
     ]);
 
-    i.on('stopped', () => {
-      events.assert(done);
-    });
+    i.start();
 
-    i.on('started', () => {
+    // stop and wait for the stop to take effect..
+    await new Promise(resolve => {
+      i.on('stopped', resolve);
       i.stop();
     });
 
-    i.start();
+    events.assert();
   });
 
-  test('should emit correct events with maxIterations', done => {
-    let iterations = 0; //eslint-disable-line no-unused-vars
-
+  test('should emit correct events with maxIterations', async function() {
     let i = new subject({
       maxIterationTime: 3000,
       maxIterations: 1,
       waitTime: 1000,
       handler: async (watchdog, state) => {
         debug('iterate!');
-        iterations++;
-        return 1;
       },
     });
 
-    i.on('error', err => {
-      done(err);
-    });
-
     let events = new IterateEvents(i, [
       'started',
       'iteration-start',
@@ -421,24 +376,20 @@ suite(testing.suiteName(), () => {
       'stopped',
     ]);
 
-    i.on('error', err => {
-      done(err);
-    });
-
-    i.on('stopped', () => {
-      events.assert(done);
-    });
+    i.start();
 
-    i.on('started', () => {
+    // stop and wait for the stop to take effect..
+    await new Promise(resolve => {
+      i.on('stopped', resolve);
       i.stop();
     });
 
-    i.start();
+    events.assert();
   });
 
   suite('event emission ordering', () => {
 
-    test('should be correct with maxFailures and maxIterations', done => {
+    test('should be correct with maxFailures and maxIterations', async function() {
       let i = new subject({
         maxIterationTime: 3000,
         maxIterations: 1,
@@ -460,18 +411,19 @@ suite(testing.suiteName(), () => {
         'error',
       ]);
 
-      i.on('error', () => {
-        events.assert(done);
-      });
-
       i.on('started', () => {
         i.stop();
       });
 
-      i.start();
+      await new Promise(resolve => {
+        i.on('error', resolve);
+        i.start();
+      });
+
+      events.assert();
     });
 
-    test('should be correct with maxFailures only', done => {
+    test('should be correct with maxFailures only', async function() {
       let i = new subject({
         maxIterationTime: 3000,
         maxFailures: 1,
@@ -491,18 +443,19 @@ suite(testing.suiteName(), () => {
         'error',
       ]);
 
-      i.on('error', () => {
-        events.assert(done);
-      });
-
       i.on('started', () => {
         i.stop();
       });
 
-      i.start();
+      await new Promise(resolve => {
+        i.on('error', resolve);
+        i.start();
+      });
+
+      events.assert();
     });
 
-    test('should be correct when handler takes too little time', done => {
+    test('should be correct when handler takes too little time', async function() {
       let i = new subject({
         maxIterationTime: 3000,
         minIterationTime: 100000,
@@ -522,27 +475,26 @@ suite(testing.suiteName(), () => {
         'error',
       ]);
 
-      i.on('error', () => {
-        events.assert(done);
-      });
-
       i.on('started', () => {
         i.stop();
       });
 
-      i.start();
+      await new Promise(resolve => {
+        i.on('error', resolve);
+        i.start();
+      });
+
+      events.assert();
     });
 
-    test('should be correct when handler takes too long (incremental watchdog)', done => {
+    test('should be correct when handler takes too long (incremental watchdog)', async function() {
       let i = new subject({
         maxIterationTime: 5000,
         maxFailures: 1,
         watchdogTime: 100,
         waitTime: 1000,
         handler: async (watchdog, state) => {
-          return new Promise((res, rej) => {
-            setTimeout(res, 3000);
-          });
+          await testing.sleep(3000);
         },
       });
 
@@ -555,26 +507,25 @@ suite(testing.suiteName(), () => {
         'error',
       ]);
 
-      i.on('error', () => {
-        events.assert(done);
-      });
-
       i.on('started', () => {
         i.stop();
       });
 
-      i.start();
+      await new Promise(resolve => {
+        i.on('error', resolve);
+        i.start();
+      });
+
+      events.assert();
     });
 
-    test('should be correct when handler takes too long (overall time)', done => {
+    test('should be correct when handler takes too long (overall time)', async function() {
       let i = new subject({
         maxIterationTime: 3000,
         maxFailures: 1,
         waitTime: 1000,
         handler: async (watchdog, state) => {
-          return new Promise((res, rej) => {
-            setTimeout(res, 6000);
-          });
+          await testing.sleep(6000);
         },
       });
 
@@ -587,18 +538,19 @@ suite(testing.suiteName(), () => {
         'error',
       ]);
 
-      i.on('error', () => {
-        events.assert(done);
-      });
-
       i.on('started', () => {
         i.stop();
       });
 
-      i.start();
+      await new Promise(resolve => {
+        i.on('error', resolve);
+        i.start();
+      });
+
+      events.assert();
     });
 
-    test('should be correct with mixed results', done => {
+    test('should be correct with mixed results', async function() {
       let iterations = 0;
 
       let i = new subject({
@@ -639,15 +591,14 @@ suite(testing.suiteName(), () => {
         'stopped',
       ]);
 
-      i.on('stopped', () => {
-        events.assert(done);
-      });
+      i.start();
 
-      i.on('error', err => {
-        done(err);
+      // stop and wait for the stop to take effect..
+      await new Promise(resolve => {
+        i.on('stopped', resolve);
       });
 
-      i.start();
+      events.assert();
     });
   });
 });

From a981828a50d1de0c97a69e82298d4379216f3ac3 Mon Sep 17 00:00:00 2001
From: "Dustin J. Mitchell" <dustin@mozilla.com>
Date: Tue, 2 Apr 2019 15:09:50 -0400
Subject: [PATCH 04/12] Run taskcluster-lib-iterate tests with zurvan

---
 libraries/iterate/test/iterate_test.js  | 68 ++++++++++++-------------
 libraries/iterate/test/watchdog_test.js | 57 +++++++++------------
 2 files changed, 59 insertions(+), 66 deletions(-)

diff --git a/libraries/iterate/test/iterate_test.js b/libraries/iterate/test/iterate_test.js
index bd0aa478160..3a939d6110f 100644
--- a/libraries/iterate/test/iterate_test.js
+++ b/libraries/iterate/test/iterate_test.js
@@ -66,7 +66,7 @@ suite(testing.suiteName(), () => {
     sandbox.restore();
   });
 
-  test('should be able to start and stop', async function() {
+  test('should be able to start and stop', testing.runWithFakeTime(async function() {
     let iterations = 0;
 
     let i = new subject({
@@ -98,9 +98,9 @@ suite(testing.suiteName(), () => {
     manager.messages.forEach(message => {
       assume(message.Fields.status).equals('success');
     });
-  });
+  }));
 
-  test('should stop after correct number of iterations', async function() {
+  test('should stop after correct number of iterations', testing.runWithFakeTime(async function() {
     let iterations = 0;
 
     let i = new subject({
@@ -124,9 +124,9 @@ suite(testing.suiteName(), () => {
     assume(iterations).equals(5);
     assume(i.keepGoing).is.not.ok();
     i.stop();
-  });
+  }));
 
-  test('should emit error when iteration watchdog expires', async function() {
+  test('should emit error when iteration watchdog expires', testing.runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 5000,
       watchdogTime: 1000,
@@ -146,9 +146,9 @@ suite(testing.suiteName(), () => {
 
     assume(i.keepGoing).is.not.ok();
     assume(err).matches(/watchdog exceeded/);
-  });
+  }));
 
-  test('should emit error when overall iteration limit is hit', async function() {
+  test('should emit error when overall iteration limit is hit', testing.runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 1000,
       waitTime: 1000,
@@ -172,9 +172,9 @@ suite(testing.suiteName(), () => {
 
     assume(i.keepGoing).is.not.ok();
     assume(err).matches(/Iteration exceeded maximum time allowed/);
-  });
+  }));
 
-  test('should emit iteration-failure when async handler fails', async function() {
+  test('should emit iteration-failure when async handler fails', testing.runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 1000,
       waitTime: 1000,
@@ -193,9 +193,9 @@ suite(testing.suiteName(), () => {
 
     assume(i.keepGoing).is.not.ok();
     assume(sawEvent).is.ok();
-  });
+  }));
 
-  test('should emit iteration-failure when sync handler fails', async function() {
+  test('should emit iteration-failure when sync handler fails', testing.runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 1000,
       waitTime: 1000,
@@ -214,9 +214,9 @@ suite(testing.suiteName(), () => {
 
     assume(i.keepGoing).is.not.ok();
     assume(sawEvent).is.ok();
-  });
+  }));
 
-  test('should emit iteration-failure when iteration is too quick', async function() {
+  test('should emit iteration-failure when iteration is too quick', testing.runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 12000,
       minIterationTime: 10000,
@@ -235,9 +235,9 @@ suite(testing.suiteName(), () => {
     i.stop();
 
     assume(iterFailed).is.ok();
-  });
+  }));
 
-  test('should emit error after too many failures', async function() {
+  test('should emit error after too many failures', testing.runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 12000,
       maxFailures: 1,
@@ -258,7 +258,7 @@ suite(testing.suiteName(), () => {
     // yep, err is an Array..
     assume(err.length).equals(1);
     assume(err[0]).matches(/uhoh/);
-  });
+  }));
 
   test('should cause uncaughtException when error event is unhandled', done => {
     // NOTE: Mocha has it's own uncaught exception listener.  If we were to
@@ -297,7 +297,7 @@ suite(testing.suiteName(), () => {
 
   });
 
-  test('should share state between iterations', async function() {
+  test('should share state between iterations', testing.runWithFakeTime(async function() {
     let iterations = 0;
 
     let i = new subject({
@@ -327,9 +327,9 @@ suite(testing.suiteName(), () => {
     i.stop();
     assume(i.keepGoing).is.not.ok();
     assume(iterFailed).is.not.ok();
-  });
+  }));
 
-  test('should emit correct events for single iteration', async function() {
+  test('should emit correct events for single iteration', testing.runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 3000,
       waitTime: 1000,
@@ -355,9 +355,9 @@ suite(testing.suiteName(), () => {
     });
 
     events.assert();
-  });
+  }));
 
-  test('should emit correct events with maxIterations', async function() {
+  test('should emit correct events with maxIterations', testing.runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 3000,
       maxIterations: 1,
@@ -385,11 +385,11 @@ suite(testing.suiteName(), () => {
     });
 
     events.assert();
-  });
+  }));
 
   suite('event emission ordering', () => {
 
-    test('should be correct with maxFailures and maxIterations', async function() {
+    test('should be correct with maxFailures and maxIterations', testing.runWithFakeTime(async function() {
       let i = new subject({
         maxIterationTime: 3000,
         maxIterations: 1,
@@ -421,9 +421,9 @@ suite(testing.suiteName(), () => {
       });
 
       events.assert();
-    });
+    }));
 
-    test('should be correct with maxFailures only', async function() {
+    test('should be correct with maxFailures only', testing.runWithFakeTime(async function() {
       let i = new subject({
         maxIterationTime: 3000,
         maxFailures: 1,
@@ -453,9 +453,9 @@ suite(testing.suiteName(), () => {
       });
 
       events.assert();
-    });
+    }));
 
-    test('should be correct when handler takes too little time', async function() {
+    test('should be correct when handler takes too little time', testing.runWithFakeTime(async function() {
       let i = new subject({
         maxIterationTime: 3000,
         minIterationTime: 100000,
@@ -485,9 +485,9 @@ suite(testing.suiteName(), () => {
       });
 
       events.assert();
-    });
+    }));
 
-    test('should be correct when handler takes too long (incremental watchdog)', async function() {
+    test('should be correct when handler takes too long (incremental watchdog)', testing.runWithFakeTime(async function() {
       let i = new subject({
         maxIterationTime: 5000,
         maxFailures: 1,
@@ -517,9 +517,9 @@ suite(testing.suiteName(), () => {
       });
 
       events.assert();
-    });
+    }));
 
-    test('should be correct when handler takes too long (overall time)', async function() {
+    test('should be correct when handler takes too long (overall time)', testing.runWithFakeTime(async function() {
       let i = new subject({
         maxIterationTime: 3000,
         maxFailures: 1,
@@ -548,9 +548,9 @@ suite(testing.suiteName(), () => {
       });
 
       events.assert();
-    });
+    }));
 
-    test('should be correct with mixed results', async function() {
+    test('should be correct with mixed results', testing.runWithFakeTime(async function() {
       let iterations = 0;
 
       let i = new subject({
@@ -599,6 +599,6 @@ suite(testing.suiteName(), () => {
       });
 
       events.assert();
-    });
+    }));
   });
 });
diff --git a/libraries/iterate/test/watchdog_test.js b/libraries/iterate/test/watchdog_test.js
index 9a9b649c466..9a4392ce20a 100644
--- a/libraries/iterate/test/watchdog_test.js
+++ b/libraries/iterate/test/watchdog_test.js
@@ -1,84 +1,77 @@
 let subject = require('../src/watchdog');
-let sinon = require('sinon');
 let assume = require('assume');
-const testing = require('taskcluster-lib-testing');
+let testing = require('taskcluster-lib-testing');
 
 suite(testing.suiteName(), function() {
-  let events, start;
+  let events;
 
-  setup(function() {
-    this.clock = sinon.useFakeTimers();
-    start = new Date();
-  });
-
-  teardown(function() {
-    this.clock.restore();
-  });
+  const runWithFakeTime = fn => {
+    return testing.runWithFakeTime(fn, {
+      systemTime: 0,
+    });
+  };
 
   const listen = w => {
     events = [];
-    w.on('expired', () => events.push(['expired', new Date() - start]));
+    w.on('expired', () => events.push(['expired', +new Date()]));
   };
 
-  test('should emit expired event', function() {
+  test('should emit expired event', runWithFakeTime(async function() {
     let w = new subject(1 * 1000);
     listen(w);
     w.start();
-    this.clock.tick(1000);
+    await testing.sleep(1000);
     assume(events).to.deeply.equal([
       ['expired', 1000],
     ]);
-  });
+  }));
 
-  test('should not expire early', function() {
+  test('should not expire early', runWithFakeTime(async function() {
     let w = new subject(1 * 1000);
     listen(w);
     w.start();
-    this.clock.tick(999);
+    await testing.sleep(999);
     w.stop();
     assume(events).to.deeply.equal([]);
-  });
+  }));
 
-  test('should expire on time', function() {
+  test('should expire on time', runWithFakeTime(async function() {
     let w = new subject(1 * 1000);
     listen(w);
     w.start();
-    this.clock.tick(1000);
+    await testing.sleep(1000);
     w.stop();
     assume(events).to.deeply.equal([
       ['expired', 1000],
     ]);
-  });
+  }));
 
-  test('should not expire twice', function() {
+  test('should not expire twice', runWithFakeTime(async function() {
     let w = new subject(1 * 1000);
     listen(w);
     w.start();
-    this.clock.tick(1000);
-    this.clock.tick(1000);
-    this.clock.tick(1000);
+    await testing.sleep(3000);
     w.stop();
     assume(events).to.deeply.equal([
       ['expired', 1000],
     ]);
-  });
+  }));
 
-  test('touching should reset timer', function() {
+  test('touching should reset timer', runWithFakeTime(async function() {
     let w = new subject(1 * 1000);
     listen(w);
     w.start();
     // We do this three times to ensure that the
     // time period stays constant and doesn't grow
     // or shrink over time
-    this.clock.tick(999);
+    await testing.sleep(999);
     w.touch();
-    this.clock.tick(999);
+    await testing.sleep(999);
     w.touch();
-    this.clock.tick(999);
-    this.clock.tick(1);
+    await testing.sleep(1000);
     w.stop();
     assume(events).to.deeply.equal([
       ['expired', 2998],
     ]);
-  });
+  }));
 });

From 792c074209c1cfb9178b5158cdf48cca3e36bd4e Mon Sep 17 00:00:00 2001
From: "Dustin J. Mitchell" <dustin@mozilla.com>
Date: Tue, 2 Apr 2019 18:13:43 -0400
Subject: [PATCH 05/12] Make iterate.stop() wait until stopped

Specifically, until any running iteration is complete, but interrupting
any waitTime delay.
---
 libraries/iterate/README.md                |  11 +-
 libraries/iterate/src/index.js             |  41 ++++--
 libraries/iterate/test/iterate_test.js     | 160 ++++++++++-----------
 services/queue/src/claimresolver.js        |   5 +-
 services/queue/src/deadlineresolver.js     |   5 +-
 services/queue/src/dependencyresolver.js   |   7 +-
 services/worker-manager/src/provisioner.js |   5 +-
 7 files changed, 118 insertions(+), 116 deletions(-)

diff --git a/libraries/iterate/README.md b/libraries/iterate/README.md
index 3d5f4800f25..b253b86675d 100644
--- a/libraries/iterate/README.md
+++ b/libraries/iterate/README.md
@@ -57,7 +57,10 @@ All times are in milliseconds.
   If this value is omitted or zero, the watchdog is disabled.
 
 The main function of the `Iterate` instance is to call `handler` repeatedly.
-This is an async function, receiving two parameters -- `(watchdog, state)`.
+This begins after a call to the `Iterate` instance's `start()` method (which returns immediately).
+To stop iteration, call the `stop()` method; this returns a Promise that resolves when any ongoing iterations are complete.
+
+The handler is an async function, receiving two parameters -- `(watchdog, state)`.
 
 The `watchdog` parameter is basically a ticking timebomb that must be defused frequently by calling its `.touch()` method.
 It has methods `.start()`, `.stop()` and `.touch()` and emits `expired` when it expires.
@@ -77,10 +80,8 @@ Iterate is an event emitter.  When relevant events occur, the following events
 are emitted.  If the `error` event does not have a listener, the process will
 exit with a non-zero exit code when it would otherwise be emitted.
 
-* `started`: when overall iteration starts
-* `stopped`: when overall iteration is finished
-* `completed`: only when we have a max number of iterations, when we
-  finish the last iteration
+* `started`: when Iterate instance starts
+* `stopped`: when Iterate instance has stopped
 * `iteration-start`: when an individual iteration starts
 * `iteration-success`: when an individual iteration completes successfully
 * `iteration-failure`: when an individual iteration fails
diff --git a/libraries/iterate/src/index.js b/libraries/iterate/src/index.js
index 70e029b9bcf..52d68d215a0 100644
--- a/libraries/iterate/src/index.js
+++ b/libraries/iterate/src/index.js
@@ -14,8 +14,8 @@ class Iterate extends events.EventEmitter {
     // Set default values
     opts = Object.assign({}, {
       watchdogTime: 0,
-      maxIterations: 0,
       maxFailures: 7,
+      maxIterations: 0,
       minIterationTime: 0,
     }, opts);
 
@@ -62,6 +62,12 @@ class Iterate extends events.EventEmitter {
     // Decide whether iteration should continue
     this.keepGoing = false;
 
+    // Called when stop is called (used to break out of waitTime sleep)
+    this.onStopCall = null
+
+    // Fires when stopped, only set when started
+    this.stopPromise = null;
+
     // We want to be able to share state between iterations
     this.sharedState = {};
 
@@ -149,27 +155,32 @@ class Iterate extends events.EventEmitter {
       // When we reach the end of a set number of iterations, we'll stop
       if (this.maxIterations > 0 && currentIteration >= this.maxIterations) {
         debug(`reached max iterations of ${this.maxIterations}`);
-        this.stop();
-        this.emit('completed');
-        // fall through to also send 'stopped'
+        this.keepGoing = false;
       }
 
       if (failures.length >= this.maxFailures) {
         this.__emitFatalError(failures);
-        return;
+        break;
       } else if (!this.keepGoing) {
-        this.stop();
-        this.emit('stopped');
-        return;
+        break;
       }
 
       if (this.waitTime > 0) {
-        debug('waiting for next iteration');
-        await new Promise(resolve => {
+        debug('waiting for next iteration or stop');
+        const stopPromise = new Promise(resolve => {
+          this.onStopCall = resolve;
+        });
+        const waitTimePromise = new Promise(resolve => {
           this.currentTimeout = setTimeout(resolve, this.waitTime);
         });
+        await Promise.race([stopPromise, waitTimePromise]);
+        this.onStopCall = null;
+        if (!this.keepGoing) {
+          break;
+        }
       }
     }
+    this.emit('stopped');
   }
 
   /**
@@ -181,8 +192,6 @@ class Iterate extends events.EventEmitter {
     if (this.currentTimeout) {
       clearTimeout(this.currentTimeout);
     }
-    this.stop();
-    this.emit('stopped');
     if (this.monitor) {
       let err = new Error('Fatal iteration error');
       err.failures = failures;
@@ -204,6 +213,9 @@ class Iterate extends events.EventEmitter {
 
   start() {
     debug('starting');
+    this.stoppedPromise = new Promise(resolve => {
+      this.on('stopped', resolve);
+    });
     this.keepGoing = true;
 
     // Two reasons we call it this way:
@@ -222,7 +234,10 @@ class Iterate extends events.EventEmitter {
 
   stop() {
     this.keepGoing = false;
-    debug('stopped');
+    if (this.onStopCall) {
+      this.onStopCall();
+    }
+    return this.stoppedPromise;
   }
 }
 
diff --git a/libraries/iterate/test/iterate_test.js b/libraries/iterate/test/iterate_test.js
index 3a939d6110f..5559e3c1b25 100644
--- a/libraries/iterate/test/iterate_test.js
+++ b/libraries/iterate/test/iterate_test.js
@@ -66,7 +66,13 @@ suite(testing.suiteName(), () => {
     sandbox.restore();
   });
 
-  test('should be able to start and stop', testing.runWithFakeTime(async function() {
+  const runWithFakeTime = fn => {
+    return testing.runWithFakeTime(fn, {
+      systemTime: 0,
+    });
+  };
+
+  test('should be able to start and stop', runWithFakeTime(async function() {
     let iterations = 0;
 
     let i = new subject({
@@ -92,7 +98,7 @@ suite(testing.suiteName(), () => {
     assume(err).to.equal(null);
     assume(iterations).equals(5);
     assume(i.keepGoing).is.ok();
-    i.stop();
+    await i.stop();
     assume(i.keepGoing).is.not.ok();
     assume(manager.messages.length).equals(5);
     manager.messages.forEach(message => {
@@ -100,7 +106,42 @@ suite(testing.suiteName(), () => {
     });
   }));
 
-  test('should stop after correct number of iterations', testing.runWithFakeTime(async function() {
+  test('should stop after current iteration completes', runWithFakeTime(async function() {
+    let i = new subject({
+      maxIterationTime: 3000,
+      waitTime: 10,
+      maxIterations: 5,
+      watchDog: 0,
+      handler: async (watchdog, state) => {
+        await testing.sleep(500);
+      },
+    });
+
+    i.start();
+    await i.stop();
+
+    assume(+new Date()).atleast(500);
+  }));
+
+  test('should stop in the midst of waitTime', runWithFakeTime(async function() {
+    let i = new subject({
+      maxIterationTime: 3000,
+      waitTime: 1000,
+      maxIterations: 2,
+      watchDog: 0,
+      handler: async (watchdog, state) => {
+        await testing.sleep(500);
+      },
+    });
+
+    i.start();
+    await testing.sleep(1000);
+    await i.stop();
+
+    assume(+new Date()).between(1000, 1100);
+  }));
+
+  test('should stop after correct number of iterations', runWithFakeTime(async function() {
     let iterations = 0;
 
     let i = new subject({
@@ -118,15 +159,14 @@ suite(testing.suiteName(), () => {
     i.start();
 
     await new Promise(resolve => {
-      i.on('completed', resolve);
+      i.on('stopped', resolve);
     });
 
     assume(iterations).equals(5);
     assume(i.keepGoing).is.not.ok();
-    i.stop();
   }));
 
-  test('should emit error when iteration watchdog expires', testing.runWithFakeTime(async function() {
+  test('should emit error when iteration watchdog expires', runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 5000,
       watchdogTime: 1000,
@@ -148,7 +188,7 @@ suite(testing.suiteName(), () => {
     assume(err).matches(/watchdog exceeded/);
   }));
 
-  test('should emit error when overall iteration limit is hit', testing.runWithFakeTime(async function() {
+  test('should emit error when overall iteration limit is hit', runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 1000,
       waitTime: 1000,
@@ -174,7 +214,7 @@ suite(testing.suiteName(), () => {
     assume(err).matches(/Iteration exceeded maximum time allowed/);
   }));
 
-  test('should emit iteration-failure when async handler fails', testing.runWithFakeTime(async function() {
+  test('should emit iteration-failure when async handler fails', runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 1000,
       waitTime: 1000,
@@ -195,7 +235,7 @@ suite(testing.suiteName(), () => {
     assume(sawEvent).is.ok();
   }));
 
-  test('should emit iteration-failure when sync handler fails', testing.runWithFakeTime(async function() {
+  test('should emit iteration-failure when sync handler fails', runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 1000,
       waitTime: 1000,
@@ -216,7 +256,7 @@ suite(testing.suiteName(), () => {
     assume(sawEvent).is.ok();
   }));
 
-  test('should emit iteration-failure when iteration is too quick', testing.runWithFakeTime(async function() {
+  test('should emit iteration-failure when iteration is too quick', runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 12000,
       minIterationTime: 10000,
@@ -237,7 +277,7 @@ suite(testing.suiteName(), () => {
     assume(iterFailed).is.ok();
   }));
 
-  test('should emit error after too many failures', testing.runWithFakeTime(async function() {
+  test('should emit error after too many failures', runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 12000,
       maxFailures: 1,
@@ -297,7 +337,7 @@ suite(testing.suiteName(), () => {
 
   });
 
-  test('should share state between iterations', testing.runWithFakeTime(async function() {
+  test('should share state between iterations', runWithFakeTime(async function() {
     let iterations = 0;
 
     let i = new subject({
@@ -329,7 +369,7 @@ suite(testing.suiteName(), () => {
     assume(iterFailed).is.not.ok();
   }));
 
-  test('should emit correct events for single iteration', testing.runWithFakeTime(async function() {
+  test('should emit correct events for single iteration', runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 3000,
       waitTime: 1000,
@@ -347,17 +387,12 @@ suite(testing.suiteName(), () => {
     ]);
 
     i.start();
-
-    // stop and wait for the stop to take effect..
-    await new Promise(resolve => {
-      i.on('stopped', resolve);
-      i.stop();
-    });
+    await i.stop();
 
     events.assert();
   }));
 
-  test('should emit correct events with maxIterations', testing.runWithFakeTime(async function() {
+  test('should emit correct events with maxIterations', runWithFakeTime(async function() {
     let i = new subject({
       maxIterationTime: 3000,
       maxIterations: 1,
@@ -372,24 +407,19 @@ suite(testing.suiteName(), () => {
       'iteration-start',
       'iteration-success',
       'iteration-complete',
-      'completed',
       'stopped',
     ]);
 
     i.start();
-
-    // stop and wait for the stop to take effect..
-    await new Promise(resolve => {
-      i.on('stopped', resolve);
-      i.stop();
-    });
+    await testing.sleep(2000);
+    await i.stop();
 
     events.assert();
   }));
 
   suite('event emission ordering', () => {
 
-    test('should be correct with maxFailures and maxIterations', testing.runWithFakeTime(async function() {
+    test('should be correct with maxFailures and maxIterations', runWithFakeTime(async function() {
       let i = new subject({
         maxIterationTime: 3000,
         maxIterations: 1,
@@ -406,24 +436,17 @@ suite(testing.suiteName(), () => {
         'iteration-start',
         'iteration-failure',
         'iteration-complete',
-        'completed',
-        'stopped',
         'error',
+        'stopped',
       ]);
 
-      i.on('started', () => {
-        i.stop();
-      });
-
-      await new Promise(resolve => {
-        i.on('error', resolve);
-        i.start();
-      });
+      i.start();
+      await i.stop();
 
       events.assert();
     }));
 
-    test('should be correct with maxFailures only', testing.runWithFakeTime(async function() {
+    test('should be correct with maxFailures only', runWithFakeTime(async function() {
       let i = new subject({
         maxIterationTime: 3000,
         maxFailures: 1,
@@ -439,23 +462,17 @@ suite(testing.suiteName(), () => {
         'iteration-start',
         'iteration-failure',
         'iteration-complete',
-        'stopped',
         'error',
+        'stopped',
       ]);
 
-      i.on('started', () => {
-        i.stop();
-      });
-
-      await new Promise(resolve => {
-        i.on('error', resolve);
-        i.start();
-      });
+      i.start();
+      await i.stop();
 
       events.assert();
     }));
 
-    test('should be correct when handler takes too little time', testing.runWithFakeTime(async function() {
+    test('should be correct when handler takes too little time', runWithFakeTime(async function() {
       let i = new subject({
         maxIterationTime: 3000,
         minIterationTime: 100000,
@@ -471,23 +488,17 @@ suite(testing.suiteName(), () => {
         'iteration-start',
         'iteration-failure',
         'iteration-complete',
-        'stopped',
         'error',
+        'stopped',
       ]);
 
-      i.on('started', () => {
-        i.stop();
-      });
-
-      await new Promise(resolve => {
-        i.on('error', resolve);
-        i.start();
-      });
+      i.start();
+      await i.stop();
 
       events.assert();
     }));
 
-    test('should be correct when handler takes too long (incremental watchdog)', testing.runWithFakeTime(async function() {
+    test('should be correct when handler takes too long (incremental watchdog)', runWithFakeTime(async function() {
       let i = new subject({
         maxIterationTime: 5000,
         maxFailures: 1,
@@ -503,23 +514,17 @@ suite(testing.suiteName(), () => {
         'iteration-start',
         'iteration-failure',
         'iteration-complete',
-        'stopped',
         'error',
+        'stopped',
       ]);
 
-      i.on('started', () => {
-        i.stop();
-      });
-
-      await new Promise(resolve => {
-        i.on('error', resolve);
-        i.start();
-      });
+      i.start();
+      await i.stop();
 
       events.assert();
     }));
 
-    test('should be correct when handler takes too long (overall time)', testing.runWithFakeTime(async function() {
+    test('should be correct when handler takes too long (overall time)', runWithFakeTime(async function() {
       let i = new subject({
         maxIterationTime: 3000,
         maxFailures: 1,
@@ -534,23 +539,17 @@ suite(testing.suiteName(), () => {
         'iteration-start',
         'iteration-failure',
         'iteration-complete',
-        'stopped',
         'error',
+        'stopped',
       ]);
 
-      i.on('started', () => {
-        i.stop();
-      });
-
-      await new Promise(resolve => {
-        i.on('error', resolve);
-        i.start();
-      });
+      i.start();
+      await i.stop();
 
       events.assert();
     }));
 
-    test('should be correct with mixed results', testing.runWithFakeTime(async function() {
+    test('should be correct with mixed results', runWithFakeTime(async function() {
       let iterations = 0;
 
       let i = new subject({
@@ -587,13 +586,12 @@ suite(testing.suiteName(), () => {
         'iteration-start',
         'iteration-success',
         'iteration-complete',
-        'completed',
         'stopped',
       ]);
 
       i.start();
 
-      // stop and wait for the stop to take effect..
+      // wait for the maxIterations to take effect..
       await new Promise(resolve => {
         i.on('stopped', resolve);
       });
diff --git a/services/queue/src/claimresolver.js b/services/queue/src/claimresolver.js
index b4ef358464f..59b7aa417e0 100644
--- a/services/queue/src/claimresolver.js
+++ b/services/queue/src/claimresolver.js
@@ -78,10 +78,7 @@ class ClaimResolver {
 
   /** Terminate iteration, returns promise that polling is stopped */
   terminate() {
-    return new Promise((res, rej) => {
-      this.iterator.once('stopped', res);
-      this.iterator.stop();
-    });
+    return this.iterator.stop();
   }
 
   /** Poll for messages and handle them in a loop */
diff --git a/services/queue/src/deadlineresolver.js b/services/queue/src/deadlineresolver.js
index e938bb0bc2a..45a68802c54 100644
--- a/services/queue/src/deadlineresolver.js
+++ b/services/queue/src/deadlineresolver.js
@@ -89,10 +89,7 @@ class DeadlineResolver {
 
   /** Terminate iteration, returns promise that polling is stopped */
   terminate() {
-    return new Promise((res, rej) => {
-      this.iterator.once('stopped', res);
-      this.iterator.stop();
-    });
+    return this.iterator.stop();
   }
 
   /** Poll for messages and handle them in a loop */
diff --git a/services/queue/src/dependencyresolver.js b/services/queue/src/dependencyresolver.js
index a3b369d557c..9ed63dc60b1 100644
--- a/services/queue/src/dependencyresolver.js
+++ b/services/queue/src/dependencyresolver.js
@@ -63,11 +63,8 @@ class DependencyResolver {
   }
 
   /** Terminate iteration, returns promise that polling is stopped */
-  async terminate() {
-    return new Promise((res, rej) => {
-      this.iterator.once('stopped', res);
-      this.iterator.stop();
-    });
+  terminate() {
+    return this.iterator.stop();
   }
 
   /** Poll for messages and handle them in a loop */
diff --git a/services/worker-manager/src/provisioner.js b/services/worker-manager/src/provisioner.js
index a1bf3e9a31a..2ff492ae2bf 100644
--- a/services/worker-manager/src/provisioner.js
+++ b/services/worker-manager/src/provisioner.js
@@ -82,10 +82,7 @@ class Provisioner extends WMObject {
       Array.from(this.providers.values()).map(x => x.terminate()),
       Array.from(this.biddingStrategies.values()).map(x => x.terminate()),
     ].flat());
-    await new Promise(resolve => {
-      this.iterate.once('stopped', resolve);
-      this.iterate.stop();
-    });
+    await this.iterate.stop();
   }
 
   /**

From 7927c4ce064c6b689dc66ae86e47dfb839b433fe Mon Sep 17 00:00:00 2001
From: "Dustin J. Mitchell" <dustin@mozilla.com>
Date: Tue, 2 Apr 2019 18:19:41 -0400
Subject: [PATCH 06/12] Make iterate.start() wait until started

---
 libraries/iterate/README.md                |  2 +-
 libraries/iterate/src/index.js             | 30 ++++++++++------------
 services/queue/src/claimresolver.js        |  5 +---
 services/queue/src/deadlineresolver.js     |  5 +---
 services/queue/src/dependencyresolver.js   |  5 +---
 services/worker-manager/src/provisioner.js |  5 +---
 6 files changed, 18 insertions(+), 34 deletions(-)

diff --git a/libraries/iterate/README.md b/libraries/iterate/README.md
index b253b86675d..421a5e56fc6 100644
--- a/libraries/iterate/README.md
+++ b/libraries/iterate/README.md
@@ -57,7 +57,7 @@ All times are in milliseconds.
   If this value is omitted or zero, the watchdog is disabled.
 
 The main function of the `Iterate` instance is to call `handler` repeatedly.
-This begins after a call to the `Iterate` instance's `start()` method (which returns immediately).
+This begins after a call to the `Iterate` instance's `start()` method, which returns a Promise that resolves once the first iteration begins (on the next tick).
 To stop iteration, call the `stop()` method; this returns a Promise that resolves when any ongoing iterations are complete.
 
 The handler is an async function, receiving two parameters -- `(watchdog, state)`.
diff --git a/libraries/iterate/src/index.js b/libraries/iterate/src/index.js
index 52d68d215a0..57d210274a7 100644
--- a/libraries/iterate/src/index.js
+++ b/libraries/iterate/src/index.js
@@ -63,7 +63,7 @@ class Iterate extends events.EventEmitter {
     this.keepGoing = false;
 
     // Called when stop is called (used to break out of waitTime sleep)
-    this.onStopCall = null
+    this.onStopCall = null;
 
     // Fires when stopped, only set when started
     this.stopPromise = null;
@@ -116,6 +116,9 @@ class Iterate extends events.EventEmitter {
   async iterate() {
     let currentIteration = 0;
     let failures = [];
+
+    this.emit('started');
+
     while (true) {
       currentIteration++;
       let iterError;
@@ -170,11 +173,15 @@ class Iterate extends events.EventEmitter {
         const stopPromise = new Promise(resolve => {
           this.onStopCall = resolve;
         });
+        let waitTimeTimeout;
         const waitTimePromise = new Promise(resolve => {
-          this.currentTimeout = setTimeout(resolve, this.waitTime);
+          waitTimeTimeout = setTimeout(resolve, this.waitTime);
         });
         await Promise.race([stopPromise, waitTimePromise]);
+
         this.onStopCall = null;
+        clearTimeout(waitTimeTimeout);
+
         if (!this.keepGoing) {
           break;
         }
@@ -189,9 +196,6 @@ class Iterate extends events.EventEmitter {
    * further.
    */
   __emitFatalError(failures) {
-    if (this.currentTimeout) {
-      clearTimeout(this.currentTimeout);
-    }
     if (this.monitor) {
       let err = new Error('Fatal iteration error');
       err.failures = failures;
@@ -218,18 +222,10 @@ class Iterate extends events.EventEmitter {
     });
     this.keepGoing = true;
 
-    // Two reasons we call it this way:
-    //   1. first call should have same exec env as following
-    //   2. start should return immediately
-    this.currentTimeout = setTimeout(async () => {
-      debug('starting iteration');
-      this.emit('started');
-      try {
-        await this.iterate();
-      } catch (err) {
-        console.error(err.stack || err);
-      }
-    }, 0);
+    return new Promise(resolve => {
+      this.once('started', resolve);
+      process.nextTick(() => this.iterate());
+    });
   }
 
   stop() {
diff --git a/services/queue/src/claimresolver.js b/services/queue/src/claimresolver.js
index 59b7aa417e0..b2c0b50ba68 100644
--- a/services/queue/src/claimresolver.js
+++ b/services/queue/src/claimresolver.js
@@ -70,10 +70,7 @@ class ClaimResolver {
 
   /** Start polling */
   async start() {
-    return new Promise((res, rej) => {
-      this.iterator.once('started', res);
-      this.iterator.start();
-    });
+    return this.iterator.start();
   }
 
   /** Terminate iteration, returns promise that polling is stopped */
diff --git a/services/queue/src/deadlineresolver.js b/services/queue/src/deadlineresolver.js
index 45a68802c54..7b46bfcadff 100644
--- a/services/queue/src/deadlineresolver.js
+++ b/services/queue/src/deadlineresolver.js
@@ -81,10 +81,7 @@ class DeadlineResolver {
 
   /** Start polling */
   async start() {
-    return new Promise((res, rej) => {
-      this.iterator.once('started', res);
-      this.iterator.start();
-    });
+    return this.iterator.start();
   }
 
   /** Terminate iteration, returns promise that polling is stopped */
diff --git a/services/queue/src/dependencyresolver.js b/services/queue/src/dependencyresolver.js
index 9ed63dc60b1..735e773e470 100644
--- a/services/queue/src/dependencyresolver.js
+++ b/services/queue/src/dependencyresolver.js
@@ -56,10 +56,7 @@ class DependencyResolver {
 
   /** Start polling for resolved-task messages */
   async start() {
-    return new Promise((res, rej) => {
-      this.iterator.once('started', res);
-      this.iterator.start();
-    });
+    return this.iterator.start();
   }
 
   /** Terminate iteration, returns promise that polling is stopped */
diff --git a/services/worker-manager/src/provisioner.js b/services/worker-manager/src/provisioner.js
index 2ff492ae2bf..5b459d63b22 100644
--- a/services/worker-manager/src/provisioner.js
+++ b/services/worker-manager/src/provisioner.js
@@ -68,10 +68,7 @@ class Provisioner extends WMObject {
       Array.from(this.providers.values()).map(x => x.initiate()),
       Array.from(this.biddingStrategies.values()).map(x => x.initiate()),
     ].flat());
-    await new Promise(resolve => {
-      this.iterate.once('started', resolve);
-      this.iterate.start();
-    });
+    await this.iterate.start();
   }
 
   /**

From 5b0f81dbaf9f9c93f68ae6e682ff507d50447eda Mon Sep 17 00:00:00 2001
From: "Dustin J. Mitchell" <dustin@mozilla.com>
Date: Tue, 2 Apr 2019 19:36:18 -0400
Subject: [PATCH 07/12] handle repeated iteration failures

---
 libraries/iterate/README.md                | 20 ++++++++---
 libraries/iterate/src/index.js             | 37 ++++---------------
 libraries/iterate/test/iterate_test.js     | 41 +---------------------
 services/queue/src/claimresolver.js        |  5 +++
 services/queue/src/deadlineresolver.js     |  5 +++
 services/queue/src/dependencyresolver.js   |  5 +++
 services/worker-manager/src/provisioner.js |  9 +++--
 7 files changed, 44 insertions(+), 78 deletions(-)

diff --git a/libraries/iterate/README.md b/libraries/iterate/README.md
index 421a5e56fc6..e28787d8c9d 100644
--- a/libraries/iterate/README.md
+++ b/libraries/iterate/README.md
@@ -51,8 +51,8 @@ All times are in milliseconds.
 * `waitTime`: the time to wait between finishing an iteration and beginning the next.
 * `maxIterations` (optional, default infinite): Complete up to this many
   iterations and then successfully exit.  Failed iterations count.
-* `maxFailures` (optional, default 7): number of failures to tolerate before considering the iteration loop a failure by emitting an `error` event.
-  This provides a balance between quick recovery from transient errors and the crashing the process for persistent errors.
+* `maxFailures` (optional, default 0): number of consecutive failures to tolerate before considering the iteration loop a failure by emitting an `error` event.
+  Disabled if set to 0.
 * `watchdogTime`: this is the time within which `watchdog.touch` must be called or the iteration is considered a failure.
   If this value is omitted or zero, the watchdog is disabled.
 
@@ -74,6 +74,17 @@ It allows each iteration to accumulate data and use on following iterations.
 Because this object is passed in by reference, changes to properties on the object are saved, but reassignment of the state variable will not be saved.
 In other words, do `state.data = {count: 1}` and not `state = {count:1}`.
 
+If `maxFailures` is set, then the `Iterate` instance will emit an `error` event when the specified number of iteration failures have occurred with out intervening successful iterations.
+This provides an escape from the situation where an application is "wedged" and some external action is required to restart it.
+Typically, this entails exiting the process and allowing the hosting environment to automatically restart it.
+Since all of the intervening failures were logged, this can be as simple as:
+
+```js
+iterator.on('error', () => {
+  process.exit(1);
+});
+```
+
 ## Events
 
 Iterate is an event emitter.  When relevant events occur, the following events
@@ -86,6 +97,5 @@ exit with a non-zero exit code when it would otherwise be emitted.
 * `iteration-success`: when an individual iteration completes successfully
 * `iteration-failure`: when an individual iteration fails
 * `iteration-complete`: when an iteration completes, regardless of outcome
-* `error`: when the iteration is considered to be concluded and provides
-  list of iteration errors.  If there are no handlers and this event is
-  emitted, an exception will be thrown in a process.nextTick callback.
+* `error`: when the Iterate instance has failed (due to reaching maxFailures),
+  containing the most recent error.
diff --git a/libraries/iterate/src/index.js b/libraries/iterate/src/index.js
index 57d210274a7..0dc729ea281 100644
--- a/libraries/iterate/src/index.js
+++ b/libraries/iterate/src/index.js
@@ -14,7 +14,7 @@ class Iterate extends events.EventEmitter {
     // Set default values
     opts = Object.assign({}, {
       watchdogTime: 0,
-      maxFailures: 7,
+      maxFailures: 0,
       maxIterations: 0,
       minIterationTime: 0,
     }, opts);
@@ -86,6 +86,7 @@ class Iterate extends events.EventEmitter {
     // times out or the maxIterationTimeTimer expires
     let timeoutRejector = new Promise((resolve, reject) => {
       watchdog.on('expired', () => {
+        debug('watchdog expired');
         reject(new Error('watchdog exceeded'));
       });
 
@@ -161,10 +162,11 @@ class Iterate extends events.EventEmitter {
         this.keepGoing = false;
       }
 
-      if (failures.length >= this.maxFailures) {
-        this.__emitFatalError(failures);
-        break;
-      } else if (!this.keepGoing) {
+      if (this.maxFailures > 0 && failures.length >= this.maxFailures) {
+        this.emit('error', failures[failures.length - 1]);
+      }
+
+      if (!this.keepGoing) {
         break;
       }
 
@@ -190,31 +192,6 @@ class Iterate extends events.EventEmitter {
     this.emit('stopped');
   }
 
-  /**
-   * Special function which knows how to emit the final error and then throw an
-   * unhandled exception where appropriate.  Also stop trying to iterate
-   * further.
-   */
-  __emitFatalError(failures) {
-    if (this.monitor) {
-      let err = new Error('Fatal iteration error');
-      err.failures = failures;
-      this.monitor.reportError(err);
-    }
-    if (this.listeners('error').length > 0) {
-      this.emit('error', failures);
-    } else {
-      debug('fatal error:');
-      for (let x of failures) {
-        debug(`  * ${x.stack || x}`);
-      }
-      debug('trying to crash process');
-      process.nextTick(() => {
-        throw new Error(`Errors:\n=====\n${failures.map(x => x.stack || x).join('=====\n')}`);
-      });
-    }
-  }
-
   start() {
     debug('starting');
     this.stoppedPromise = new Promise(resolve => {
diff --git a/libraries/iterate/test/iterate_test.js b/libraries/iterate/test/iterate_test.js
index 5559e3c1b25..1ad746534d3 100644
--- a/libraries/iterate/test/iterate_test.js
+++ b/libraries/iterate/test/iterate_test.js
@@ -295,48 +295,9 @@ suite(testing.suiteName(), () => {
     i.stop();
 
     assume(i.keepGoing).is.not.ok();
-    // yep, err is an Array..
-    assume(err.length).equals(1);
-    assume(err[0]).matches(/uhoh/);
+    assume(err).matches(/uhoh/);
   }));
 
-  test('should cause uncaughtException when error event is unhandled', done => {
-    // NOTE: Mocha has it's own uncaught exception listener.  If we were to
-    // leave it in force during this test, we'd end up getting two results from
-    // the test.  One failure from the mocha handler and one pass from our own
-    // handler.  This is obviously not ideal, and it's sort of a risk that we
-    // mess up the uncaught exception handling for future tests
-
-    let origListeners = process.listeners('uncaughtException');
-    process.removeAllListeners('uncaughtException');
-
-    let uncaughtHandler = function(err) {
-      process.removeAllListeners('uncaughtException');
-      for (let listener of origListeners) {
-        process.on('uncaughtException', listener);
-      }
-      i.stop();
-      assume(i.keepGoing).is.not.ok();
-      done();
-    };
-
-    process.on('uncaughtException', uncaughtHandler);
-
-    let i = new subject({
-      maxIterationTime: 12000,
-      maxFailures: 1,
-      waitTime: 1000,
-      handler: async (watchdog, state) => {
-        return new Promise((res, rej) => {
-          rej(new Error('hi'));
-        });
-      },
-    });
-
-    i.start();
-
-  });
-
   test('should share state between iterations', runWithFakeTime(async function() {
     let iterations = 0;
 
diff --git a/services/queue/src/claimresolver.js b/services/queue/src/claimresolver.js
index b2c0b50ba68..0c7bdf2cad3 100644
--- a/services/queue/src/claimresolver.js
+++ b/services/queue/src/claimresolver.js
@@ -66,6 +66,11 @@ class ClaimResolver {
         await Promise.all(loops);
       },
     });
+
+    this.iterator.on('error', () => {
+      this.monitor.alert('iteration failed repeatedly; terminating process');
+      process.exit(1);
+    });
   }
 
   /** Start polling */
diff --git a/services/queue/src/deadlineresolver.js b/services/queue/src/deadlineresolver.js
index 7b46bfcadff..5c96bebef6d 100644
--- a/services/queue/src/deadlineresolver.js
+++ b/services/queue/src/deadlineresolver.js
@@ -77,6 +77,11 @@ class DeadlineResolver {
         await Promise.all(loops);
       },
     });
+
+    this.iterator.on('error', () => {
+      this.monitor.alert('iteration failed repeatedly; terminating process');
+      process.exit(1);
+    });
   }
 
   /** Start polling */
diff --git a/services/queue/src/dependencyresolver.js b/services/queue/src/dependencyresolver.js
index 735e773e470..b01f3d70978 100644
--- a/services/queue/src/dependencyresolver.js
+++ b/services/queue/src/dependencyresolver.js
@@ -52,6 +52,11 @@ class DependencyResolver {
         await Promise.all(loops);
       },
     });
+
+    this.iterator.on('error', () => {
+      this.monitor.alert('iteration failed repeatedly; terminating process');
+      process.exit(1);
+    });
   }
 
   /** Start polling for resolved-task messages */
diff --git a/services/worker-manager/src/provisioner.js b/services/worker-manager/src/provisioner.js
index 5b459d63b22..cd325dda335 100644
--- a/services/worker-manager/src/provisioner.js
+++ b/services/worker-manager/src/provisioner.js
@@ -49,15 +49,18 @@ class Provisioner extends WMObject {
     this.iterationGap = iterationGap;
 
     this.iterate = new Iterate({
-      maxFailures: 30, // We really don't want it to crash
+      maxFailures: 10,
       maxIterationTime: 300000,
       waitTime: iterationGap,
       handler: async () => {
         await this.provision();
       },
     });
-    // TODO:
-    // make sure that iteration failures have a handler that's useful
+
+    this.iterate.on('error', () => {
+      console.log('iteration failed repeatedly; terminating process');
+      process.exit(1);
+    });
   }
 
   /**

From c09cfb696d2e86d4ced04409f91a68e19be1007e Mon Sep 17 00:00:00 2001
From: "Dustin J. Mitchell" <dustin@mozilla.com>
Date: Tue, 2 Apr 2019 19:39:42 -0400
Subject: [PATCH 08/12] use const instead of let

---
 libraries/iterate/src/index.js          | 14 ++---
 libraries/iterate/src/watchdog.js       |  2 +-
 libraries/iterate/test/iterate_test.js  | 72 ++++++++++++-------------
 libraries/iterate/test/watchdog_test.js | 16 +++---
 4 files changed, 52 insertions(+), 52 deletions(-)

diff --git a/libraries/iterate/src/index.js b/libraries/iterate/src/index.js
index 0dc729ea281..29213a8463d 100644
--- a/libraries/iterate/src/index.js
+++ b/libraries/iterate/src/index.js
@@ -1,6 +1,6 @@
-let WatchDog = require('./watchdog');
-let debug = require('debug')('iterate');
-let events = require('events');
+const WatchDog = require('./watchdog');
+const debug = require('debug')('iterate');
+const events = require('events');
 
 /**
  * The Iterate Class.  See README.md for explanation of constructor
@@ -78,13 +78,13 @@ class Iterate extends events.EventEmitter {
 
   async single_iteration() {
     debug('running handler');
-    let start = new Date();
-    let watchdog = new WatchDog(this.watchdogTime);
+    const start = new Date();
+    const watchdog = new WatchDog(this.watchdogTime);
     let maxIterationTimeTimer;
 
     // build a promise that will reject when either the watchdog
     // times out or the maxIterationTimeTimer expires
-    let timeoutRejector = new Promise((resolve, reject) => {
+    const timeoutRejector = new Promise((resolve, reject) => {
       watchdog.on('expired', () => {
         debug('watchdog expired');
         reject(new Error('watchdog exceeded'));
@@ -107,7 +107,7 @@ class Iterate extends events.EventEmitter {
       watchdog.stop();
     }
 
-    let duration = new Date() - start;
+    const duration = new Date() - start;
     if (this.minIterationTime > 0 && duration < this.minIterationTime) {
       throw new Error('Handler duration was less than minIterationTime');
     }
diff --git a/libraries/iterate/src/watchdog.js b/libraries/iterate/src/watchdog.js
index 75df657f065..ad3830d6423 100644
--- a/libraries/iterate/src/watchdog.js
+++ b/libraries/iterate/src/watchdog.js
@@ -1,4 +1,4 @@
-let events = require('events');
+const events = require('events');
 
 /**
  * This is a watch dog timer.  Think of it as a ticking timebomb which will
diff --git a/libraries/iterate/test/iterate_test.js b/libraries/iterate/test/iterate_test.js
index 1ad746534d3..b8ebc22a527 100644
--- a/libraries/iterate/test/iterate_test.js
+++ b/libraries/iterate/test/iterate_test.js
@@ -1,11 +1,11 @@
-let subject = require('../');
-let sandbox = require('sinon').createSandbox();
-let assume = require('assume');
-let debug = require('debug')('iterate-test');
-let MonitorManager = require('taskcluster-lib-monitor');
-let testing = require('taskcluster-lib-testing');
-
-let possibleEvents = [
+const subject = require('../');
+const sandbox = require('sinon').createSandbox();
+const assume = require('assume');
+const debug = require('debug')('iterate-test');
+const MonitorManager = require('taskcluster-lib-monitor');
+const testing = require('taskcluster-lib-testing');
+
+const possibleEvents = [
   'started',
   'stopped',
   'completed',
@@ -29,7 +29,7 @@ class IterateEvents {
   }
 
   assert() {
-    let dl = () => { // dl === dump list
+    const dl = () => { // dl === dump list
       return `\nExpected: ${JSON.stringify(this.expectedOrder, null, 2)}` +
         `\nActual: ${JSON.stringify(this.orderOfEmission, null, 2)}`;
     };
@@ -75,7 +75,7 @@ suite(testing.suiteName(), () => {
   test('should be able to start and stop', runWithFakeTime(async function() {
     let iterations = 0;
 
-    let i = new subject({
+    const i = new subject({
       maxIterationTime: 3000,
       waitTime: 1000,
       watchDog: 0,
@@ -107,7 +107,7 @@ suite(testing.suiteName(), () => {
   }));
 
   test('should stop after current iteration completes', runWithFakeTime(async function() {
-    let i = new subject({
+    const i = new subject({
       maxIterationTime: 3000,
       waitTime: 10,
       maxIterations: 5,
@@ -124,7 +124,7 @@ suite(testing.suiteName(), () => {
   }));
 
   test('should stop in the midst of waitTime', runWithFakeTime(async function() {
-    let i = new subject({
+    const i = new subject({
       maxIterationTime: 3000,
       waitTime: 1000,
       maxIterations: 2,
@@ -144,7 +144,7 @@ suite(testing.suiteName(), () => {
   test('should stop after correct number of iterations', runWithFakeTime(async function() {
     let iterations = 0;
 
-    let i = new subject({
+    const i = new subject({
       maxIterationTime: 3000,
       waitTime: 10,
       maxIterations: 5,
@@ -167,7 +167,7 @@ suite(testing.suiteName(), () => {
   }));
 
   test('should emit error when iteration watchdog expires', runWithFakeTime(async function() {
-    let i = new subject({
+    const i = new subject({
       maxIterationTime: 5000,
       watchdogTime: 1000,
       waitTime: 1000,
@@ -189,7 +189,7 @@ suite(testing.suiteName(), () => {
   }));
 
   test('should emit error when overall iteration limit is hit', runWithFakeTime(async function() {
-    let i = new subject({
+    const i = new subject({
       maxIterationTime: 1000,
       waitTime: 1000,
       maxFailures: 1,
@@ -215,7 +215,7 @@ suite(testing.suiteName(), () => {
   }));
 
   test('should emit iteration-failure when async handler fails', runWithFakeTime(async function() {
-    let i = new subject({
+    const i = new subject({
       maxIterationTime: 1000,
       waitTime: 1000,
       maxFailures: 100,
@@ -236,7 +236,7 @@ suite(testing.suiteName(), () => {
   }));
 
   test('should emit iteration-failure when sync handler fails', runWithFakeTime(async function() {
-    let i = new subject({
+    const i = new subject({
       maxIterationTime: 1000,
       waitTime: 1000,
       maxFailures: 100,
@@ -257,7 +257,7 @@ suite(testing.suiteName(), () => {
   }));
 
   test('should emit iteration-failure when iteration is too quick', runWithFakeTime(async function() {
-    let i = new subject({
+    const i = new subject({
       maxIterationTime: 12000,
       minIterationTime: 10000,
       waitTime: 1000,
@@ -278,7 +278,7 @@ suite(testing.suiteName(), () => {
   }));
 
   test('should emit error after too many failures', runWithFakeTime(async function() {
-    let i = new subject({
+    const i = new subject({
       maxIterationTime: 12000,
       maxFailures: 1,
       waitTime: 1000,
@@ -301,7 +301,7 @@ suite(testing.suiteName(), () => {
   test('should share state between iterations', runWithFakeTime(async function() {
     let iterations = 0;
 
-    let i = new subject({
+    const i = new subject({
       maxIterationTime: 3000,
       waitTime: 10,
       maxIterations: 20,
@@ -331,7 +331,7 @@ suite(testing.suiteName(), () => {
   }));
 
   test('should emit correct events for single iteration', runWithFakeTime(async function() {
-    let i = new subject({
+    const i = new subject({
       maxIterationTime: 3000,
       waitTime: 1000,
       handler: async (watchdog, state) => {
@@ -339,7 +339,7 @@ suite(testing.suiteName(), () => {
       },
     });
 
-    let events = new IterateEvents(i, [
+    const events = new IterateEvents(i, [
       'started',
       'iteration-start',
       'iteration-success',
@@ -354,7 +354,7 @@ suite(testing.suiteName(), () => {
   }));
 
   test('should emit correct events with maxIterations', runWithFakeTime(async function() {
-    let i = new subject({
+    const i = new subject({
       maxIterationTime: 3000,
       maxIterations: 1,
       waitTime: 1000,
@@ -363,7 +363,7 @@ suite(testing.suiteName(), () => {
       },
     });
 
-    let events = new IterateEvents(i, [
+    const events = new IterateEvents(i, [
       'started',
       'iteration-start',
       'iteration-success',
@@ -381,7 +381,7 @@ suite(testing.suiteName(), () => {
   suite('event emission ordering', () => {
 
     test('should be correct with maxFailures and maxIterations', runWithFakeTime(async function() {
-      let i = new subject({
+      const i = new subject({
         maxIterationTime: 3000,
         maxIterations: 1,
         maxFailures: 1,
@@ -392,7 +392,7 @@ suite(testing.suiteName(), () => {
         },
       });
 
-      let events = new IterateEvents(i, [
+      const events = new IterateEvents(i, [
         'started',
         'iteration-start',
         'iteration-failure',
@@ -408,7 +408,7 @@ suite(testing.suiteName(), () => {
     }));
 
     test('should be correct with maxFailures only', runWithFakeTime(async function() {
-      let i = new subject({
+      const i = new subject({
         maxIterationTime: 3000,
         maxFailures: 1,
         waitTime: 1000,
@@ -418,7 +418,7 @@ suite(testing.suiteName(), () => {
         },
       });
 
-      let events = new IterateEvents(i, [
+      const events = new IterateEvents(i, [
         'started',
         'iteration-start',
         'iteration-failure',
@@ -434,7 +434,7 @@ suite(testing.suiteName(), () => {
     }));
 
     test('should be correct when handler takes too little time', runWithFakeTime(async function() {
-      let i = new subject({
+      const i = new subject({
         maxIterationTime: 3000,
         minIterationTime: 100000,
         maxFailures: 1,
@@ -444,7 +444,7 @@ suite(testing.suiteName(), () => {
         },
       });
 
-      let events = new IterateEvents(i, [
+      const events = new IterateEvents(i, [
         'started',
         'iteration-start',
         'iteration-failure',
@@ -460,7 +460,7 @@ suite(testing.suiteName(), () => {
     }));
 
     test('should be correct when handler takes too long (incremental watchdog)', runWithFakeTime(async function() {
-      let i = new subject({
+      const i = new subject({
         maxIterationTime: 5000,
         maxFailures: 1,
         watchdogTime: 100,
@@ -470,7 +470,7 @@ suite(testing.suiteName(), () => {
         },
       });
 
-      let events = new IterateEvents(i, [
+      const events = new IterateEvents(i, [
         'started',
         'iteration-start',
         'iteration-failure',
@@ -486,7 +486,7 @@ suite(testing.suiteName(), () => {
     }));
 
     test('should be correct when handler takes too long (overall time)', runWithFakeTime(async function() {
-      let i = new subject({
+      const i = new subject({
         maxIterationTime: 3000,
         maxFailures: 1,
         waitTime: 1000,
@@ -495,7 +495,7 @@ suite(testing.suiteName(), () => {
         },
       });
 
-      let events = new IterateEvents(i, [
+      const events = new IterateEvents(i, [
         'started',
         'iteration-start',
         'iteration-failure',
@@ -513,7 +513,7 @@ suite(testing.suiteName(), () => {
     test('should be correct with mixed results', runWithFakeTime(async function() {
       let iterations = 0;
 
-      let i = new subject({
+      const i = new subject({
         maxIterationTime: 3000,
         maxIterations: 6,
         maxFailures: 5,
@@ -527,7 +527,7 @@ suite(testing.suiteName(), () => {
         },
       });
 
-      let events = new IterateEvents(i, [
+      const events = new IterateEvents(i, [
         'started',
         'iteration-start',
         'iteration-failure',
diff --git a/libraries/iterate/test/watchdog_test.js b/libraries/iterate/test/watchdog_test.js
index 9a4392ce20a..cbf39c258a4 100644
--- a/libraries/iterate/test/watchdog_test.js
+++ b/libraries/iterate/test/watchdog_test.js
@@ -1,6 +1,6 @@
-let subject = require('../src/watchdog');
-let assume = require('assume');
-let testing = require('taskcluster-lib-testing');
+const subject = require('../src/watchdog');
+const assume = require('assume');
+const testing = require('taskcluster-lib-testing');
 
 suite(testing.suiteName(), function() {
   let events;
@@ -17,7 +17,7 @@ suite(testing.suiteName(), function() {
   };
 
   test('should emit expired event', runWithFakeTime(async function() {
-    let w = new subject(1 * 1000);
+    const w = new subject(1 * 1000);
     listen(w);
     w.start();
     await testing.sleep(1000);
@@ -27,7 +27,7 @@ suite(testing.suiteName(), function() {
   }));
 
   test('should not expire early', runWithFakeTime(async function() {
-    let w = new subject(1 * 1000);
+    const w = new subject(1 * 1000);
     listen(w);
     w.start();
     await testing.sleep(999);
@@ -36,7 +36,7 @@ suite(testing.suiteName(), function() {
   }));
 
   test('should expire on time', runWithFakeTime(async function() {
-    let w = new subject(1 * 1000);
+    const w = new subject(1 * 1000);
     listen(w);
     w.start();
     await testing.sleep(1000);
@@ -47,7 +47,7 @@ suite(testing.suiteName(), function() {
   }));
 
   test('should not expire twice', runWithFakeTime(async function() {
-    let w = new subject(1 * 1000);
+    const w = new subject(1 * 1000);
     listen(w);
     w.start();
     await testing.sleep(3000);
@@ -58,7 +58,7 @@ suite(testing.suiteName(), function() {
   }));
 
   test('touching should reset timer', runWithFakeTime(async function() {
-    let w = new subject(1 * 1000);
+    const w = new subject(1 * 1000);
     listen(w);
     w.start();
     // We do this three times to ensure that the

From da0a0bbc3f4d60967d7eac035b6dc81987f59396 Mon Sep 17 00:00:00 2001
From: "Dustin J. Mitchell" <dustin@mozilla.com>
Date: Tue, 2 Apr 2019 19:40:09 -0400
Subject: [PATCH 09/12] remove use of sinon

---
 libraries/iterate/test/iterate_test.js | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/libraries/iterate/test/iterate_test.js b/libraries/iterate/test/iterate_test.js
index b8ebc22a527..22a26f57fd1 100644
--- a/libraries/iterate/test/iterate_test.js
+++ b/libraries/iterate/test/iterate_test.js
@@ -1,5 +1,4 @@
 const subject = require('../');
-const sandbox = require('sinon').createSandbox();
 const assume = require('assume');
 const debug = require('debug')('iterate-test');
 const MonitorManager = require('taskcluster-lib-monitor');
@@ -62,10 +61,6 @@ suite(testing.suiteName(), () => {
     manager.terminate();
   });
 
-  teardown(() => {
-    sandbox.restore();
-  });
-
   const runWithFakeTime = fn => {
     return testing.runWithFakeTime(fn, {
       systemTime: 0,

From f1cb4089ac5e25bfdeef91e23c6e815b73732827 Mon Sep 17 00:00:00 2001
From: "Dustin J. Mitchell" <dustin@mozilla.com>
Date: Tue, 2 Apr 2019 19:43:19 -0400
Subject: [PATCH 10/12] drop support for state in tc-lib-iterate

---
 libraries/iterate/README.md            | 16 ++----
 libraries/iterate/src/index.js         |  5 +-
 libraries/iterate/test/iterate_test.js | 68 +++++++-------------------
 3 files changed, 24 insertions(+), 65 deletions(-)

diff --git a/libraries/iterate/README.md b/libraries/iterate/README.md
index e28787d8c9d..9c1947eae07 100644
--- a/libraries/iterate/README.md
+++ b/libraries/iterate/README.md
@@ -17,7 +17,7 @@ i = new Iterate({
   maxIterationTime: 10000,
   watchdogTime: 5000,
   waitTime: 2000,
-  handler: async (watchdog, state) => {
+  handler: async watchdog => {
     await doSomeWork();
     watchdog.touch();  // tell Iterate that we`re doing work still
     await doMoreWork();
@@ -39,7 +39,7 @@ i.on(`stopped`, () => {
 The constructor for the `Iterate` class takes an options object, with the following properties.
 All times are in milliseconds.
 
-* `handler`: the async function to call repeatedly, called as `await handler(watchdog, state)`.
+* `handler`: the async function to call repeatedly, called as `await handler(watchdog)`.
   See details below.
 * `monitor` (optional): instance of a `taskcluster-lib-monitor` instance with a name appropriate for this iterate instance.
   This is used to report errors.
@@ -60,20 +60,14 @@ The main function of the `Iterate` instance is to call `handler` repeatedly.
 This begins after a call to the `Iterate` instance's `start()` method, which returns a Promise that resolves once the first iteration begins (on the next tick).
 To stop iteration, call the `stop()` method; this returns a Promise that resolves when any ongoing iterations are complete.
 
-The handler is an async function, receiving two parameters -- `(watchdog, state)`.
-
-The `watchdog` parameter is basically a ticking timebomb that must be defused frequently by calling its `.touch()` method.
+The handler is an async function, receiving one parameter -- `watchdog`.
+This is basically a ticking timebomb that must be defused frequently by calling its `.touch()` method (unless it is not enabled).
 It has methods `.start()`, `.stop()` and `.touch()` and emits `expired` when it expires.
-What it allows an implementor is the abilty to say that while the absolute maximum iteration interval (`maxIterationTime`), incremental progress should be made.
+What it allows an implementor is the abilty to say that within the absolute maximum iteration interval (`maxIterationTime`), incremental progress should be made.
 The idea here is that after each chunk of work in the handler, you run `.touch()`.
 If the `watchdogTime` duration elapses without a touch, then the iteration is considered faild.
 This way, you can have a handler that can be marked as failing without waiting the full `maxIterationTime`.
 
-The `state` parameter is an object that is passed in to the handler function.
-It allows each iteration to accumulate data and use on following iterations.
-Because this object is passed in by reference, changes to properties on the object are saved, but reassignment of the state variable will not be saved.
-In other words, do `state.data = {count: 1}` and not `state = {count:1}`.
-
 If `maxFailures` is set, then the `Iterate` instance will emit an `error` event when the specified number of iteration failures have occurred with out intervening successful iterations.
 This provides an escape from the situation where an application is "wedged" and some external action is required to restart it.
 Typically, this entails exiting the process and allowing the hosting environment to automatically restart it.
diff --git a/libraries/iterate/src/index.js b/libraries/iterate/src/index.js
index 29213a8463d..c3ad38a7c7f 100644
--- a/libraries/iterate/src/index.js
+++ b/libraries/iterate/src/index.js
@@ -68,9 +68,6 @@ class Iterate extends events.EventEmitter {
     // Fires when stopped, only set when started
     this.stopPromise = null;
 
-    // We want to be able to share state between iterations
-    this.sharedState = {};
-
     // Store the iteration timeout so that a `.stop()` call during an iteration
     // inhibits a handler from running
     this.currentTimeout = null;
@@ -99,7 +96,7 @@ class Iterate extends events.EventEmitter {
       watchdog.start();
       await Promise.race([
         timeoutRejector,
-        Promise.resolve(this.handler(watchdog, this.sharedState)),
+        Promise.resolve(this.handler(watchdog)),
       ]);
     } finally {
       // stop the timers regardless of success or failure
diff --git a/libraries/iterate/test/iterate_test.js b/libraries/iterate/test/iterate_test.js
index 22a26f57fd1..579a694e463 100644
--- a/libraries/iterate/test/iterate_test.js
+++ b/libraries/iterate/test/iterate_test.js
@@ -74,7 +74,7 @@ suite(testing.suiteName(), () => {
       maxIterationTime: 3000,
       waitTime: 1000,
       watchDog: 0,
-      handler: async (watchdog, state) => {
+      handler: async watchdog => {
         debug('iterate!');
         iterations++;
         return 1;
@@ -107,7 +107,7 @@ suite(testing.suiteName(), () => {
       waitTime: 10,
       maxIterations: 5,
       watchDog: 0,
-      handler: async (watchdog, state) => {
+      handler: async watchdog => {
         await testing.sleep(500);
       },
     });
@@ -124,7 +124,7 @@ suite(testing.suiteName(), () => {
       waitTime: 1000,
       maxIterations: 2,
       watchDog: 0,
-      handler: async (watchdog, state) => {
+      handler: async watchdog => {
         await testing.sleep(500);
       },
     });
@@ -144,7 +144,7 @@ suite(testing.suiteName(), () => {
       waitTime: 10,
       maxIterations: 5,
       watchDog: 0,
-      handler: async (watchdog, state) => {
+      handler: async watchdog => {
         debug('iterate!');
         iterations++;
         return 1;
@@ -167,7 +167,7 @@ suite(testing.suiteName(), () => {
       watchdogTime: 1000,
       waitTime: 1000,
       maxFailures: 1,
-      handler: async (watchdog, state) => {
+      handler: async watchdog => {
         return testing.sleep(2000);
       },
     });
@@ -188,7 +188,7 @@ suite(testing.suiteName(), () => {
       maxIterationTime: 1000,
       waitTime: 1000,
       maxFailures: 1,
-      handler: async (watchdog, state) => {
+      handler: async watchdog => {
         watchdog.stop();
         return new Promise((res, rej) => {
           setTimeout(() => {
@@ -214,7 +214,7 @@ suite(testing.suiteName(), () => {
       maxIterationTime: 1000,
       waitTime: 1000,
       maxFailures: 100,
-      handler: async (watchdog, state) => {
+      handler: async watchdog => {
         throw new Error('uhoh');
       },
     });
@@ -235,7 +235,7 @@ suite(testing.suiteName(), () => {
       maxIterationTime: 1000,
       waitTime: 1000,
       maxFailures: 100,
-      handler: (watchdog, state) => {
+      handler: watchdog => {
         throw new Error('uhoh');
       },
     });
@@ -257,7 +257,7 @@ suite(testing.suiteName(), () => {
       minIterationTime: 10000,
       waitTime: 1000,
       watchDog: 0,
-      handler: async (watchdog, state) => {
+      handler: async watchdog => {
         await testing.sleep(100);
       },
     });
@@ -277,7 +277,7 @@ suite(testing.suiteName(), () => {
       maxIterationTime: 12000,
       maxFailures: 1,
       waitTime: 1000,
-      handler: async (watchdog, state) => {
+      handler: async watchdog => {
         throw new Error('uhoh');
       },
     });
@@ -293,43 +293,11 @@ suite(testing.suiteName(), () => {
     assume(err).matches(/uhoh/);
   }));
 
-  test('should share state between iterations', runWithFakeTime(async function() {
-    let iterations = 0;
-
-    const i = new subject({
-      maxIterationTime: 3000,
-      waitTime: 10,
-      maxIterations: 20,
-      maxFailures: 1,
-      watchDog: 0,
-      handler: async (watchdog, state) => {
-        if (!state.v) {
-          state.v = {count: 0};
-        }
-        assume(state.v.count === iterations);
-        state.v.count += 1;
-        iterations += 1;
-      },
-    });
-
-    let iterFailed = false;
-    i.on('iteration-failure', () => { iterFailed = true; });
-
-    i.start();
-    await testing.sleep(3000);
-
-    assume(iterations).equals(20);
-    assume(i.keepGoing).is.not.ok();
-    i.stop();
-    assume(i.keepGoing).is.not.ok();
-    assume(iterFailed).is.not.ok();
-  }));
-
   test('should emit correct events for single iteration', runWithFakeTime(async function() {
     const i = new subject({
       maxIterationTime: 3000,
       waitTime: 1000,
-      handler: async (watchdog, state) => {
+      handler: async watchdog => {
         debug('iterate!');
       },
     });
@@ -353,7 +321,7 @@ suite(testing.suiteName(), () => {
       maxIterationTime: 3000,
       maxIterations: 1,
       waitTime: 1000,
-      handler: async (watchdog, state) => {
+      handler: async watchdog => {
         debug('iterate!');
       },
     });
@@ -381,7 +349,7 @@ suite(testing.suiteName(), () => {
         maxIterations: 1,
         maxFailures: 1,
         waitTime: 1000,
-        handler: async (watchdog, state) => {
+        handler: async watchdog => {
           debug('iterate!');
           throw new Error('hi');
         },
@@ -407,7 +375,7 @@ suite(testing.suiteName(), () => {
         maxIterationTime: 3000,
         maxFailures: 1,
         waitTime: 1000,
-        handler: async (watchdog, state) => {
+        handler: async watchdog => {
           debug('iterate!');
           throw new Error('hi');
         },
@@ -434,7 +402,7 @@ suite(testing.suiteName(), () => {
         minIterationTime: 100000,
         maxFailures: 1,
         waitTime: 1000,
-        handler: async (watchdog, state) => {
+        handler: async watchdog => {
           return true;
         },
       });
@@ -460,7 +428,7 @@ suite(testing.suiteName(), () => {
         maxFailures: 1,
         watchdogTime: 100,
         waitTime: 1000,
-        handler: async (watchdog, state) => {
+        handler: async watchdog => {
           await testing.sleep(3000);
         },
       });
@@ -485,7 +453,7 @@ suite(testing.suiteName(), () => {
         maxIterationTime: 3000,
         maxFailures: 1,
         waitTime: 1000,
-        handler: async (watchdog, state) => {
+        handler: async watchdog => {
           await testing.sleep(6000);
         },
       });
@@ -513,7 +481,7 @@ suite(testing.suiteName(), () => {
         maxIterations: 6,
         maxFailures: 5,
         waitTime: 1000,
-        handler: async (watchdog, state) => {
+        handler: async watchdog=> {
           if (iterations++ % 2 === 0) {
             throw new Error('even, so failing');
           } else {

From 230da652fe2cb880087aad5baba8d7e83cab22c9 Mon Sep 17 00:00:00 2001
From: "Dustin J. Mitchell" <dustin@mozilla.com>
Date: Wed, 3 Apr 2019 19:07:53 +0000
Subject: [PATCH 11/12] add best-practice about libraries using src/index.js

---
 dev-docs/best-practices/libraries.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dev-docs/best-practices/libraries.md b/dev-docs/best-practices/libraries.md
index 3e05a1aa16e..36e77a32507 100644
--- a/dev-docs/best-practices/libraries.md
+++ b/dev-docs/best-practices/libraries.md
@@ -12,3 +12,4 @@ This file will automatically be linked from the `README.md` in the root of the r
 
 Library source code should be in a `src` subdirectory.
 No transpilation should be used: write JS that can be interpreted directly by the Node version in use in the repository.
+The `main` property in `package.json` should point to `src/index.js`, which may then load other parts of the library.

From 700c019ed79ca7e2e19d5d1fe85897b57035e67a Mon Sep 17 00:00:00 2001
From: "Dustin J. Mitchell" <dustin@mozilla.com>
Date: Wed, 3 Apr 2019 19:36:34 +0000
Subject: [PATCH 12/12] start iteration immediately, with a just-in-case
 .catch(..)

---
 libraries/iterate/src/index.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libraries/iterate/src/index.js b/libraries/iterate/src/index.js
index c3ad38a7c7f..bda39f3ba5e 100644
--- a/libraries/iterate/src/index.js
+++ b/libraries/iterate/src/index.js
@@ -198,7 +198,9 @@ class Iterate extends events.EventEmitter {
 
     return new Promise(resolve => {
       this.once('started', resolve);
-      process.nextTick(() => this.iterate());
+      // start iteration; any failures here are a programming error in this
+      // library and so should be considered fatal
+      this.iterate().catch(err => this.emit('error', err));
     });
   }