feat(vow): Support reliable retry-send

packages/vow/package.json

@@ -31,8 +31,11 @@
   },
   "devDependencies": {
     "@agoric/internal": "^0.3.2",
+    "@agoric/swingset-liveslots": "^0.10.2",
+    "@agoric/zone": "^0.2.2",
     "@endo/far": "^1.1.5",
     "@endo/init": "^1.1.4",
+    "@endo/ses-ava": "^1.2.5",
     "ava": "^5.3.0",
     "tsd": "^0.31.1"
   },

packages/vow/src/retrier.js

@@ -0,0 +1,140 @@
+import { M } from '@endo/patterns';
+import { PromiseWatcherI } from '@agoric/base-zone';
+import { isUpgradeDisconnection } from '@agoric/internal/src/upgrade-api.js';
+import { heapVowE, prepareVowTools } from '../vat.js';
+import { VowShape, toPassableCap } from './vow-utils.js';
+
+/**
+ * @import { Zone } from '@agoric/base-zone'
+ * @import {VowTools} from './tools.js'
+ */
+
+const RetrierI = M.interface('Retrier', {
+  retry: M.call().returns(),
+  getVow: M.call().returns(VowShape),
+  cancel: M.call(M.error()).returns(),
+});
+
+const RetrierShape = M.remotable('retrier');
+
+const RetrierAdminI = M.interface('RetrierAdmin', {
+  // modeled on getFlowForOutcomeVow
+  getRetrierForOutcomeVow: M.call(VowShape).returns(M.opt(RetrierShape)),
+});
+
+/**
+ * @param {Zone} zone
+ * @param {VowTools} [vowTools]
+ */
+export const prepareRetrierTools = (zone, vowTools = prepareVowTools(zone)) => {
+  const { makeVowKit, watch } = vowTools;
+  const retrierForOutcomeVowKey = zone.mapStore('retrierForOutcomeVow', {
+    keyShape: M.remotable('toPassableCap'),
+    valueShape: RetrierShape,
+  });
+
+  const makeRetrierKit = zone.exoClassKit(
+    'Retrier',
+    {
+      retrier: RetrierI,
+      watcher: PromiseWatcherI,
+    },
+    (target, optVerb, args) => {
+      const { vow, resolver } = makeVowKit();
+      return harden({
+        target,
+        optVerb,
+        args,
+        vow,
+        optResolver: resolver,
+      });
+    },
+    {
+      retrier: {
+        retry() {
+          const { state, facets } = this;
+          const { target, optVerb, args, optResolver } = state;
+          const { watcher } = facets;
+
+          if (optResolver === undefined) {
+            return;
+          }
+          // TODO `heapVowE` is likely too fragile under upgrade.
+          const p =
+            optVerb === undefined
+              ? heapVowE(target)(...args)
+              : heapVowE(target)[optVerb](...args);
+          watch(p, watcher);
+        },
+        getVow() {
+          const { state } = this;
+          const { vow } = state;
+          return vow;
+        },
+        cancel(reason) {
+          const { state } = this;
+          const { vow } = state;
+          if (state.optResolver === undefined) {
+            return;
+          }
+          state.optResolver.reject(reason);
+          state.optResolver = undefined;
+          retrierForOutcomeVowKey.delete(toPassableCap(vow));
+        },
+      },
+      watcher: {
+        onFulfilled(value) {
+          const { state } = this;
+
+          if (state.optResolver === undefined) {
+            return;
+          }
+          state.optResolver.resolve(value);
+          state.optResolver = undefined;
+        },
+        onRejected(reason) {
+          const { state, facets } = this;
+          const { retrier } = facets;
+
+          if (state.optResolver === undefined) {
+            return;
+          }
+          if (isUpgradeDisconnection(reason)) {
+            // TODO do I need to wait for a new incarnation
+            // using isRetryableReason instead?
+            retrier.retry();
+            return;
+          }
+          state.optResolver.reject(reason);
+          state.optResolver = undefined;
+        },
+      },
+    },
+    {
+      finish({ state, facets }) {
+        const { vow } = state;
+        const { retrier } = facets;
+        retrier.retry();
+        retrierForOutcomeVowKey.init(toPassableCap(vow), retrier);
+      },
+    },
+  );
+
+  const retrierAdmin = zone.exo('retrierAdmin', RetrierAdminI, {
+    getRetrierForOutcomeVow(vow) {
+      return retrierForOutcomeVowKey.get(toPassableCap(vow));
+    },
+  });
+
+  const retry = (target, optVerb, args) => {
+    const { retrier } = makeRetrierKit(target, optVerb, args);
+    return retrier.getVow();
+  };
+
+  return harden({
+    makeRetrierKit,
+    retrierAdmin,
+    retry,
+  });
+};
+harden(prepareRetrierTools);

packages/vow/test/retrier.test.js

@@ -0,0 +1,130 @@
+import '@agoric/swingset-liveslots/tools/prepare-test-env.js';
+import test from '@endo/ses-ava/prepare-endo.js';
+
+import { Fail } from '@endo/errors';
+import { passStyleOf } from '@endo/pass-style';
+import { eventLoopIteration } from '@agoric/internal/src/testing-utils.js';
+import { makeHeapZone } from '@agoric/zone/heap.js';
+import { makeVirtualZone } from '@agoric/zone/virtual.js';
+import { makeDurableZone } from '@agoric/zone/durable.js';
+import { reincarnate } from '@agoric/swingset-liveslots/tools/setup-vat-data.js';
+import { makeUpgradeDisconnection } from '@agoric/internal/src/upgrade-api.js';
+
+import { prepareVowTools } from '../vat.js';
+import { prepareRetrierTools } from '../src/retrier.js';
+import { isVow } from '../src/vow-utils.js';
+
+/**
+ * @import {Zone} from '@agoric/base-zone'
+ */
+
+/** @type {ReturnType<typeof reincarnate>} */
+let incarnation;
+
+const annihilate = () => {
+  incarnation = reincarnate({ relaxDurabilityRules: false });
+};
+
+const getBaggage = () => {
+  return incarnation.fakeVomKit.cm.provideBaggage();
+};
+
+const nextLife = () => {
+  incarnation = reincarnate(incarnation);
+};
+
+/**
+ * @param {any} t
+ * @param {Zone} zone
+ */
+const retrierPlay1 = async (t, zone) => {
+  const vowTools = prepareVowTools(zone);
+  const { retry, retrierAdmin } = prepareRetrierTools(zone, vowTools);
+
+  const makeBob = zone.exoClass('Bob', undefined, count => ({ count }), {
+    foo(carol) {
+      const { state } = this;
+      state.count += 1;
+      carol.ping(state.count);
+      if (state.count < 102) {
+        throw makeUpgradeDisconnection('emulated upgrade1', state.count);
+      } else {
+        t.log('postponed at', state.count);
+        return new Promise(() => {}); // never resolves
+      }
+    },
+  });
+  const bob = makeBob(100);
+  const carol = zone.exo('carol', undefined, {
+    ping(count) {
+      t.log('ping at', count);
+    },
+  });
+  const v = zone.makeOnce('v', () => retry(bob, 'foo', [carol]));
+  t.true(isVow(v));
+  t.is(passStyleOf(retrierAdmin.getRetrierForOutcomeVow(v)), 'remotable');
+};
+
+/**
+ * @param {any} t
+ * @param {Zone} zone
+ */
+const retrierPlay2 = async (t, zone) => {
+  const vowTools = prepareVowTools(zone);
+  const { when } = vowTools;
+  const { retrierAdmin } = prepareRetrierTools(zone, vowTools);
+
+  zone.exoClass('Bob', undefined, count => ({ count }), {
+    foo(carol) {
+      const { state } = this;
+      t.true(state.count >= 102);
+      state.count += 1;
+      carol.ping(state.count);
+      if (state.count < 104) {
+        throw makeUpgradeDisconnection('emulated upgrade2', state.count);
+      } else {
+        return carol;
+      }
+    },
+  });
+  const carol = zone.exo('carol', undefined, {
+    ping(count) {
+      t.log('ping at', count);
+    },
+  });
+  const v = zone.makeOnce('v', () => Fail`need v`);
+  t.true(isVow(v));
+  const retrier = retrierAdmin.getRetrierForOutcomeVow(v);
+
+  // Emulate waking up after upgrade
+  // Should only be needed because of low fidelity of this
+  // lightweight upgrade testing framework.
+  // TODO remove once ported to a higher fidelity upgrade testing framework.
+  // See https://github.com/Agoric/agoric-sdk/issues/9303
+  retrier.retry();
+  t.is(await when(v), carol);
+  t.log('carol finally returned');
+};
+
+test.serial('test heap retrier', async t => {
+  const zone = makeHeapZone('heapRoot');
+  return retrierPlay1(t, zone);
+});
+
+test.serial('test virtual retrier', async t => {
+  annihilate();
+  const zone = makeVirtualZone('virtualRoot');
+  return retrierPlay1(t, zone);
+});
+
+test.serial('test durable retrier', async t => {
+  annihilate();
+  const zone1 = makeDurableZone(getBaggage(), 'durableRoot');
+  await retrierPlay1(t, zone1);
+
+  await eventLoopIteration();
+
+  nextLife();
+  const zone2 = makeDurableZone(getBaggage(), 'durableRoot');
+  await retrierPlay2(t, zone2);
+});