feat(swingset): implement Meters for crank computation charges

This introduces user-visible Meter objects, and allows new dynamic vats to be
bound to a single Meter, both exposed through the vatAdmin facet. Meters are
long-term reservoirs of execution credits, denominated in "computrons". Each
bound vat will deduct credits from its meter until exhausted, at which point
the vat will be terminated. This limits the long-term CPU consumption of a
vat, in addition to the fixed per-crank computron limit applied to any
metered vat.

Meters can be refilled and queried through their API. Each meter also has a
Notifier, and a configurable notification threshold: the notifier will be
updated if/when the remaining credits drop below the threshold. This should
allow a supervisor in userspace enough time to refill the meter before the
associated vat(s) are terminated.

Some notes:
* The vatAdmin facet now offers `createMeter()`, which returns a `Meter`
object with methods to manipulate its `remaining` and `threshold` values, as
well as a `getNotifier` to subscribe to threshold-passing events.
* The vatAdmin `createVat()` call now takes a `meter: Meter` option instead
of `metered: boolean`. If a Meter is provided, two things happen:
  * Each delivery to that (XS) vat is subject to a per-crank compute limit.
  * Each delivery deducts the compute usage from the Meter.
* Dynamic vats are now *unmetered* by default, since we need a Meter object,
not just a boolean.
* When a Meter's `remaining` drops below its `threshold`, the notifier is
triggered with the current `remaining` value.
* When a vat's Meter reaches zero, the vat is terminated, just as if it had
violated the per-crank limit.
  * Currently the termination message (used to reject the control facet's
  `.done()` Promise) is different for the per-crank limit vs the Meter limit,
  but this may change.
* Meter deductions and threshold notifications are stashed in a new
'postAbortActions' record, to make sure they happen even if the crank is
aborted and all other state changes are unwound.
* The vatManager `managerOptions` still use `metered: boolean`, because the
vat manager doesn't know about Meters: it only need to know whether to apply
the per-crank limits or not.

closes #3308

packages/SwingSet/src/kernel/kernel.js

@@ -379,12 +379,42 @@ export default function buildKernel(
   }
 
   let terminationTrigger;
+  let postAbortActions;
 
   function resetDeliveryTriggers() {
     terminationTrigger = undefined;
+    postAbortActions = {
+      meterDeductions: [], // list of { meterID, compute }
+      meterNotifications: [], // list of meterID
+    };
   }
   resetDeliveryTriggers();
 
+  function notifyMeterThreshold(meterID) {
+    // tell vatAdmin that a meter has dropped below its notifyThreshold
+    const { remaining } = kernelKeeper.getMeter(meterID);
+    const args = { body: JSON.stringify([meterID, remaining]), slots: [] };
+    queueToKref(vatAdminRootKref, 'meterCrossedThreshold', args, 'logFailure');
+  }
+
+  function deductMeter(meterID, compute, firstTime) {
+    assert.typeof(compute, 'number');
+    const res = kernelKeeper.deductMeter(meterID, compute);
+    // we also recode deduction and any notification in postAbortActions,
+    // which are executed if the delivery is being rewound for any reason
+    // (syscall error, res.underflow), so their side-effects survive
+    if (firstTime) {
+      postAbortActions.meterDeductions.push({ meterID, compute });
+    }
+    if (res.notify) {
+      notifyMeterThreshold(meterID);
+      if (firstTime) {
+        postAbortActions.meterNotifications.push(meterID);
+      }
+    }
+    return res.underflow;
+  }
+
   // this is called for syscall.exit (shouldAbortCrank=false), and for any
   // vat-fatal errors (shouldAbortCrank=true)
   function setTerminationTrigger(vatID, shouldAbortCrank, shouldReject, info) {
@@ -396,12 +426,13 @@ export default function buildKernel(
     }
   }
 
-  async function deliverAndLogToVat(vatID, kd, vd) {
+  async function deliverAndLogToVat(vatID, kd, vd, useMeter) {
     // eslint-disable-next-line no-use-before-define
     assert(vatWarehouse.lookup(vatID));
     const vatKeeper = kernelKeeper.provideVatKeeper(vatID);
     const crankNum = kernelKeeper.getCrankNumber();
     const deliveryNum = vatKeeper.nextDeliveryNum(); // increments
+    const { meterID } = vatKeeper.getOptions();
     /** @typedef { any } FinishFunction TODO: static types for slog? */
     /** @type { FinishFunction } */
     const finish = kernelSlog.delivery(vatID, crankNum, deliveryNum, kd, vd);
@@ -412,11 +443,21 @@ export default function buildKernel(
       const deliveryResult = await vatWarehouse.deliverToVat(vatID, vd);
       insistVatDeliveryResult(deliveryResult);
       finish(deliveryResult);
-      const [status, problem] = deliveryResult;
+      const [status, problem, metering] = deliveryResult;
       if (status !== 'ok') {
         // probably a metering fault, or a bug in the vat's dispatch()
         console.log(`delivery problem, terminating vat ${vatID}`, problem);
         setTerminationTrigger(vatID, true, true, makeError(problem));
+        return;
+      }
+      if (useMeter && meterID) {
+        const underflow = deductMeter(meterID, metering.compute, true);
+        if (underflow) {
+          console.log(`meter ${meterID} underflow, terminating vat ${vatID}`);
+          const err = makeError('meter underflow, vat terminated');
+          setTerminationTrigger(vatID, true, true, err);
+          return;
+        }
       }
     } catch (e) {
       // log so we get a stack trace
@@ -438,7 +479,7 @@ export default function buildKernel(
       const kd = harden(['message', target, msg]);
       // eslint-disable-next-line no-use-before-define
       const vd = vatWarehouse.kernelDeliveryToVatDelivery(vatID, kd);
-      await deliverAndLogToVat(vatID, kd, vd);
+      await deliverAndLogToVat(vatID, kd, vd, true);
     }
   }
 
@@ -551,7 +592,7 @@ export default function buildKernel(
       // eslint-disable-next-line no-use-before-define
       const vd = vatWarehouse.kernelDeliveryToVatDelivery(vatID, kd);
       vatKeeper.deleteCListEntriesForKernelSlots(targets);
-      await deliverAndLogToVat(vatID, kd, vd);
+      await deliverAndLogToVat(vatID, kd, vd, true);
     }
   }
 
@@ -575,7 +616,7 @@ export default function buildKernel(
     }
     // eslint-disable-next-line no-use-before-define
     const vd = vatWarehouse.kernelDeliveryToVatDelivery(vatID, kd);
-    await deliverAndLogToVat(vatID, kd, vd);
+    await deliverAndLogToVat(vatID, kd, vd, false);
   }
 
   async function processCreateVat(message) {
@@ -687,9 +728,18 @@ export default function buildKernel(
           // errors unwind any changes the vat made
           abortCrank();
           didAbort = true;
+          // but metering deductions or underflow notifications must survive
+          const { meterDeductions, meterNotifications } = postAbortActions;
+          for (const { meterID, compute } of meterDeductions) {
+            deductMeter(meterID, compute, false);
+          }
+          for (const meterID of meterNotifications) {
+            // reads meter.remaining, so must happen after deductMeter
+            notifyMeterThreshold(meterID);
+          }
         }
-        // state changes reflecting the termination must survive, so these
-        // happen after a possible abortCrank()
+        // state changes reflecting the termination must also survive, so
+        // these happen after a possible abortCrank()
         terminateVat(vatID, shouldReject, info);
         kernelSlog.terminateVat(vatID, shouldReject, info);
         kdebug(`vat terminated: ${JSON.stringify(info)}`);
@@ -915,6 +965,13 @@ export default function buildKernel(
         return vatID;
       },
       terminate: (vatID, reason) => terminateVat(vatID, true, reason),
+      meterCreate: (remaining, threshold) =>
+        kernelKeeper.allocateMeter(remaining, threshold),
+      meterAddRemaining: (meterID, delta) =>
+        kernelKeeper.addMeterRemaining(meterID, delta),
+      meterSetThreshold: (meterID, threshold) =>
+        kernelKeeper.setMeterThreshold(meterID, threshold),
+      meterGet: meterID => kernelKeeper.getMeter(meterID),
     };
 
     // instantiate all devices

packages/SwingSet/src/kernel/loadVat.js

@@ -87,7 +87,7 @@ export function makeVatLoader(stuff) {
 
   const allowedDynamicOptions = [
     'description',
-    'metered',
+    'meterID',
     'managerType', // TODO: not sure we want vats to be able to control this
     'vatParameters',
     'enableSetup',
@@ -133,13 +133,14 @@ export function makeVatLoader(stuff) {
    *
    * @param {number} options.virtualObjectCacheSize
    *
-   * @param {boolean} [options.metered] if true,
-   *        subjects the new dynamic vat to a meter that limits
-   *        the amount of computation and allocation that can occur during any
-   *        given crank. Stack frames are limited as well. The meter is refilled
-   *        between cranks, but if the meter ever underflows, the vat is
-   *        terminated. If false, the vat is unmetered.  Defaults to true for
-   *        dynamic vats; static vats may not be metered.
+   * @param {string} [options.meter] If a meterID is provided, the new
+   *        dynamic vat is limited to a fixed amount of computation and
+   *        allocation that can occur during any given crank. Peak stack
+   *        frames are limited as well. In addition, the given meter's
+   *        "remaining" value will be reduced by the amount of computation
+   *        used by each crank. The meter will eventually underflow unless it
+   *        is topped up, at which point the vat is terminated. If undefined,
+   *        the vat is unmetered. Static vats cannot be metered.
    *
    * @param {Record<string, unknown>} [options.vatParameters] provides
    *        the contents of the second argument to
@@ -199,7 +200,7 @@ export function makeVatLoader(stuff) {
       isDynamic ? allowedDynamicOptions : allowedStaticOptions,
     );
     const {
-      metered = isDynamic,
+      meterID,
       vatParameters = {},
       managerType,
       enableSetup = false,
@@ -231,7 +232,7 @@ export function makeVatLoader(stuff) {
     const managerOptions = {
       managerType,
       bundle: vatSourceBundle,
-      metered,
+      metered: !!meterID,
       enableDisavow,
       enableSetup,
       enablePipelining,

packages/SwingSet/src/kernel/state/kernelKeeper.js

@@ -54,6 +54,7 @@ const enableKernelGC = true;
 // v$NN.t.endPosition = $NN
 // v$NN.vs.$key = string
 // v$NN.lastSnapshot = JSON({ snapshotID, startPos })
+// v$NN.meter = m$NN
 
 // m$NN.remaining = $NN // non-negative remaining capacity (in computrons)
 // m$NN.threshold = $NN // notify when .remaining first drops below this

packages/SwingSet/src/kernel/state/vatKeeper.js

@@ -102,7 +102,7 @@ export function makeVatKeeper(
   }
 
   function getOptions() {
-    const options = JSON.parse(kvStore.get(`${vatID}.options`));
+    const options = JSON.parse(kvStore.get(`${vatID}.options`) || '{}');
     return harden(options);
   }
 

packages/SwingSet/src/kernel/vatAdmin/vatAdmin-src.js

@@ -18,12 +18,33 @@
  */
 
 import { Far } from '@agoric/marshal';
+import { Nat } from '@agoric/nat';
 
 export function buildRootDeviceNode({ endowments, serialize }) {
-  const { pushCreateVatEvent, terminate: kernelTerminateVatFn } = endowments;
+  const {
+    pushCreateVatEvent,
+    terminate: kernelTerminateVatFn,
+    meterCreate,
+    meterAddRemaining,
+    meterSetThreshold,
+    meterGet,
+  } = endowments;
 
   // The Root Device Node.
   return Far('root', {
+    createMeter(remaining, threshold) {
+      return meterCreate(Nat(remaining), Nat(threshold));
+    },
+    addMeterRemaining(meterID, delta) {
+      meterAddRemaining(meterID, Nat(delta));
+    },
+    setMeterThreshold(meterID, threshold) {
+      meterSetThreshold(meterID, Nat(threshold));
+    },
+    getMeter(meterID) {
+      return meterGet(meterID);
+    },
+
     // Called by the wrapper vat to create a new vat. Gets a new ID from the
     // kernel's vat creator fn. Remember that the root object will arrive
     // separately. Clean up the outgoing and incoming arguments.

packages/SwingSet/src/kernel/vatAdmin/vatAdminWrapper.js

@@ -6,7 +6,9 @@
  * device affordances into objects that can be used by code in other vats.
  */
 import { makePromiseKit } from '@agoric/promise-kit';
+import { makeNotifierKit } from '@agoric/notifier';
 import { Far } from '@agoric/marshal';
+import { Nat } from '@agoric/nat';
 
 function producePRR() {
   const { promise, resolve, reject } = makePromiseKit();
@@ -17,6 +19,26 @@ export function buildRootObject(vatPowers) {
   const { D } = vatPowers;
   const pending = new Map(); // vatID -> { resolve, reject } for promise
   const running = new Map(); // vatID -> { resolve, reject } for doneP
+  const meterByID = new Map(); // meterID -> { meter, updater }
+  const meterIDByMeter = new WeakMap(); // meter -> meterID
+
+  function makeMeter(vatAdminNode, remaining, threshold) {
+    Nat(remaining);
+    Nat(threshold);
+    const meterID = D(vatAdminNode).createMeter(remaining, threshold);
+    const { updater, notifier } = makeNotifierKit();
+    const meter = Far('meter', {
+      addRemaining(delta) {
+        D(vatAdminNode).addMeterRemaining(meterID, Nat(delta));
+      },
+      setThreshold(newThreshold) {
+        D(vatAdminNode).setMeterThreshold(meterID, Nat(newThreshold));
+      },
+      get: () => D(vatAdminNode).getMeter(meterID), // returns Numbers
+      getNotifier: () => notifier,
+    });
+    return { meterID, meter, updater };
+  }
 
   function finishVatCreation(vatAdminNode, vatID) {
     const [promise, pendingRR] = producePRR();
@@ -39,14 +61,35 @@ export function buildRootObject(vatPowers) {
     });
   }
 
+  function convertOptions(origOptions) {
+    const options = { ...origOptions };
+    delete options.meterID;
+    delete options.meter;
+    if (origOptions.meter) {
+      const meterID = meterIDByMeter.get(origOptions.meter);
+      options.meterID = meterID;
+    }
+    return harden(options);
+  }
+
   function createVatAdminService(vatAdminNode) {
     return Far('vatAdminService', {
-      createVat(code, options) {
-        const vatID = D(vatAdminNode).create(code, options);
+      createMeter(remaining, threshold) {
+        const m = makeMeter(vatAdminNode, remaining, threshold);
+        const { meterID, meter, updater } = m;
+        meterByID.set(meterID, harden({ meter, updater }));
+        meterIDByMeter.set(meter, meterID);
+        return meter;
+      },
+      createVat(code, options = {}) {
+        const vatID = D(vatAdminNode).create(code, convertOptions(options));
         return finishVatCreation(vatAdminNode, vatID);
       },
-      createVatByName(bundleName, options) {
-        const vatID = D(vatAdminNode).createByName(bundleName, options);
+      createVatByName(bundleName, options = {}) {
+        const vatID = D(vatAdminNode).createByName(
+          bundleName,
+          convertOptions(options),
+        );
         return finishVatCreation(vatAdminNode, vatID);
       },
     });
@@ -63,6 +106,11 @@ export function buildRootObject(vatPowers) {
     }
   }
 
+  function meterCrossedThreshold(meterID, remaining) {
+    const { updater } = meterByID.get(meterID);
+    updater.updateState(remaining);
+  }
+
   // the kernel sends this when the vat halts
   function vatTerminated(vatID, shouldReject, info) {
     if (!running.has(vatID)) {
@@ -85,5 +133,6 @@ export function buildRootObject(vatPowers) {
     createVatAdminService,
     newVatCallback,
     vatTerminated,
+    meterCrossedThreshold,
   });
 }

packages/SwingSet/test/gc-dead-vat/bootstrap.js

@@ -15,7 +15,7 @@ export function buildRootObject() {
   return Far('root', {
     async bootstrap(vats, devices) {
       const vatMaker = E(vats.vatAdmin).createVatAdminService(devices.vatAdmin);
-      vat = await E(vatMaker).createVatByName('doomed', { metered: false });
+      vat = await E(vatMaker).createVatByName('doomed');
       doomedRoot = vat.root;
       await sendExport(doomedRoot);
       const doomedExport1Presence = await E(doomedRoot).getDoomedExport1();