feat: robustify allManagersDo (#9833)

closes: #9706 

## Description

make `allManagersDo` robust to errors thrown by one vaultManager

### Security Considerations

Prevent issues in one vaultManager from blocking progress in others. This would affect liquidation and locking oracle prices for upcoming liquidation auctions.

### Scaling Considerations

Doesn't add work.

### Documentation Considerations

Not user visible

### Testing Considerations

added a tiny unit test.

### Upgrade Considerations

added robustness shouldn't impact upgrade.

packages/inter-protocol/src/vaultFactory/vaultDirector.js

@@ -82,6 +82,21 @@ const trace = makeTracer('VD', true);
 
 const shortfallInvitationKey = 'shortfallInvitation';
 
+// If one manager/token fails, we don't want that to block possible success for
+// others, so we .catch() and log separately.
+//
+// exported for testing
+export const makeAllManagersDo = (collateralManagers, vaultManagers) => {
+  /** @param {(vm: VaultManager) => void} fn */
+  return fn => {
+    for (const managerIndex of collateralManagers.values()) {
+      Promise.resolve(vaultManagers.get(managerIndex).self)
+        .then(vm => fn(vm))
+        .catch(e => trace('🚨ERROR: allManagersDo', e));
+    }
+  };
+};
+
 /**
  * @param {import('@agoric/swingset-liveslots').Baggage} baggage
  * @param {import('./vaultFactory.js').VaultFactoryZCF} zcf
@@ -263,13 +278,7 @@ const prepareVaultDirector = (
     metrics: makeRecorderTopic('Vault Factory metrics', metricsKit),
   });
 
-  /** @param {(vm: VaultManager) => void} fn */
-  const allManagersDo = fn => {
-    for (const managerIndex of collateralManagers.values()) {
-      const vm = vaultManagers.get(managerIndex).self;
-      fn(vm);
-    }
-  };
+  const allManagersDo = makeAllManagersDo(collateralManagers, vaultManagers);
 
   const makeWaker = (name, func) => {
     return Far(name, {

packages/inter-protocol/test/vaultFactory/director-allMgrsDo.test.js

@@ -0,0 +1,63 @@
+import { Far } from '@endo/marshal';
+import { test } from '@agoric/zoe/tools/prepare-test-env-ava.js';
+
+import { waitUntilQuiescent } from '@agoric/swingset-liveslots/test/waitUntilQuiescent.js';
+import { makeAllManagersDo } from '../../src/vaultFactory/vaultDirector.js';
+
+const makeFakeVM = (name, thrower) => {
+  let okay = true;
+  const fakeVM = Far('fakeVM', {
+    doIt: () => {
+      if (thrower) {
+        okay = false;
+        throw Error('whatever', name);
+      }
+      return name;
+    },
+    okay: () => okay,
+  });
+  return { self: fakeVM };
+};
+
+const makeFakeVMgrKits = () => {
+  const kits = [];
+  return {
+    add: k => kits.push(k),
+    get: i => kits[i],
+  };
+};
+
+test(`AllManagersDo no throw`, async t => {
+  const collMgr = [0, 1, 2];
+  const kits = makeFakeVMgrKits();
+  kits.add(makeFakeVM('A', false));
+  kits.add(makeFakeVM('B', false));
+  kits.add(makeFakeVM('C', false));
+
+  const allManagersDo = makeAllManagersDo(collMgr, kits);
+  // @ts-expect-error It's a fake
+  const result = allManagersDo(vm => vm.doIt());
+  t.is(result, undefined);
+  t.true(kits.get(0).self.okay());
+  t.true(kits.get(1).self.okay());
+  t.true(kits.get(2).self.okay());
+});
+
+test(`AllManagersDo throw`, async t => {
+  const collMgr = [0, 1, 2];
+  const kits = makeFakeVMgrKits();
+  kits.add(makeFakeVM('A', false));
+  kits.add(makeFakeVM('B', true));
+  kits.add(makeFakeVM('C', false));
+
+  const allManagersDo = makeAllManagersDo(collMgr, kits);
+  // @ts-expect-error It's a fake
+  const result = allManagersDo(vm => vm.doIt());
+  t.is(result, undefined);
+
+  await waitUntilQuiescent();
+
+  t.true(kits.get(0).self.okay());
+  t.false(kits.get(1).self.okay());
+  t.true(kits.get(2).self.okay());
+});