diff --git a/packages/SwingSet/src/kernel/liveSlots.js b/packages/SwingSet/src/kernel/liveSlots.js index 7abbbae15f5..00169af3f20 100644 --- a/packages/SwingSet/src/kernel/liveSlots.js +++ b/packages/SwingSet/src/kernel/liveSlots.js @@ -102,14 +102,19 @@ function build( const importedDevices = new Set(); // device nodes const deadSet = new Set(); // vrefs that are finalized but not yet reported - function retainExportedRemotable(vref) { + function retainExportedVref(vref) { // if the vref corresponds to a Remotable, keep a strong reference to it // until the kernel tells us to release it const { type, allocatedByVat, virtual } = parseVatSlot(vref); - if (type === 'object' && allocatedByVat && !virtual) { - const remotable = slotToVal.get(vref).deref(); - assert(remotable, X`somehow lost Remotable for ${vref}`); - exportedRemotables.add(remotable); + if (type === 'object' && allocatedByVat) { + if (virtual) { + // eslint-disable-next-line no-use-before-define + vom.setExported(vref, true); + } else { + const remotable = slotToVal.get(vref).deref(); + assert(remotable, X`somehow lost Remotable for ${vref}`); + exportedRemotables.add(remotable); + } } } @@ -179,11 +184,6 @@ function build( } const droppedRegistry = new FinalizationRegistry(finalizeDroppedImport); - function processDroppedRepresentative(_vref) { - // no-op, to be implemented by virtual object manager - return false; - } - function processDeadSet() { let doMore = false; const [importsToDrop, importsToRetire, exportsToRetire] = [[], [], []]; @@ -193,17 +193,18 @@ function build( assert(type === 'object', `unprepared to track ${type}`); if (virtual) { // Representative: send nothing, but perform refcount checking - doMore = doMore || processDroppedRepresentative(vref); + // eslint-disable-next-line no-use-before-define + doMore = doMore || vom.possibleVirtualObjectDeath(vref); } else if (allocatedByVat) { // Remotable: send retireExport exportsToRetire.push(vref); } else { // Presence: send dropImport unless reachable by VOM // eslint-disable-next-line no-lonely-if, no-use-before-define - if (!isVrefReachable(vref)) { + if (!vom.isVrefReachable(vref)) { importsToDrop.push(vref); // eslint-disable-next-line no-use-before-define - if (!isVrefRecognizable(vref)) { + if (!vom.isVrefRecognizable(vref)) { importsToRetire.push(vref); } } @@ -412,15 +413,7 @@ function build( return wr && wr.deref(); } - const { - makeVirtualObjectRepresentative, - makeWeakStore, - makeKind, - VirtualObjectAwareWeakMap, - VirtualObjectAwareWeakSet, - isVrefReachable, - isVrefRecognizable, - } = makeVirtualObjectManager( + const vom = makeVirtualObjectManager( syscall, allocateExportID, getSlotForVal, @@ -506,20 +499,13 @@ function build( // detect reanimation by playing games inside their instanceKitMaker to // try to observe when new representatives are created (e.g., by // counting calls or squirreling things away in hidden WeakMaps). - makeVirtualObjectRepresentative(slot, true); // N.b.: throwing away the result + vom.makeVirtualObjectRepresentative(slot, true); // N.b.: throwing away the result } return val; } if (virtual) { - // Virtual objects should never be put in the slotToVal table, as their - // entire raison d'etre is to be absent from memory when they're not being - // used. They *do* get put in the valToSlot table, which is OK because - // it's a WeakMap, but they don't get put there here. Instead, they are - // put there by makeVirtualObjectRepresentative, who already has to do - // this anyway in the cases of creating virtual objects in the first place - // and swapping them in from disk. assert.equal(type, 'object'); - val = makeVirtualObjectRepresentative(slot, false); + val = vom.makeVirtualObjectRepresentative(slot, false); } else { assert(!allocatedByVat, X`I don't remember allocating ${slot}`); if (type === 'object') { @@ -577,7 +563,7 @@ function build( function collect(promiseID, rejected, value) { doneResolutions.add(promiseID); const valueSer = m.serialize(value); - valueSer.slots.map(retainExportedRemotable); + valueSer.slots.map(retainExportedVref); resolutions.push([promiseID, rejected, valueSer]); scanSlots(valueSer.slots); } @@ -609,7 +595,7 @@ function build( } const serArgs = m.serialize(harden(args)); - serArgs.slots.map(retainExportedRemotable); + serArgs.slots.map(retainExportedVref); const resultVPID = allocatePromiseID(); lsdebug(`Promise allocation ${forVatID}:${resultVPID} in queueMessage`); // create a Promise which callers follow for the result, give it a @@ -669,7 +655,7 @@ function build( } return (...args) => { const serArgs = m.serialize(harden(args)); - serArgs.slots.map(retainExportedRemotable); + serArgs.slots.map(retainExportedVref); forbidPromises(serArgs); const ret = syscall.callNow(slot, prop, serArgs); insistCapData(ret); @@ -849,6 +835,10 @@ function build( if (o) { exportedRemotables.delete(o); } + const { virtual } = parseVatSlot(vref); + if (virtual) { + vom.setExported(vref, false); + } } } @@ -909,13 +899,13 @@ function build( function exitVat(completion) { const args = m.serialize(harden(completion)); - args.slots.map(retainExportedRemotable); + args.slots.map(retainExportedVref); syscall.exit(false, args); } function exitVatWithFailure(reason) { const args = m.serialize(harden(reason)); - args.slots.map(retainExportedRemotable); + args.slots.map(retainExportedVref); syscall.exit(true, args); } @@ -937,13 +927,13 @@ function build( } const vatGlobals = harden({ - makeWeakStore, - makeKind, + makeWeakStore: vom.makeWeakStore, + makeKind: vom.makeKind, }); const inescapableGlobalProperties = harden({ - WeakMap: VirtualObjectAwareWeakMap, - WeakSet: VirtualObjectAwareWeakSet, + WeakMap: vom.VirtualObjectAwareWeakMap, + WeakSet: vom.VirtualObjectAwareWeakSet, }); function setBuildRootObject(buildRootObject) { @@ -989,7 +979,7 @@ function build( const rootSlot = makeVatSlot('object', true, BigInt(0)); valToSlot.set(rootObject, rootSlot); slotToVal.set(rootSlot, new WeakRef(rootObject)); - retainExportedRemotable(rootSlot); + retainExportedVref(rootSlot); // we do not use droppedRegistry for exports } diff --git a/packages/SwingSet/src/kernel/virtualObjectManager.js b/packages/SwingSet/src/kernel/virtualObjectManager.js index 5baba09e534..8c910553668 100644 --- a/packages/SwingSet/src/kernel/virtualObjectManager.js +++ b/packages/SwingSet/src/kernel/virtualObjectManager.js @@ -1,4 +1,7 @@ +/* eslint-disable no-use-before-define */ + import { assert, details as X, quote as q } from '@agoric/assert'; +import { Nat } from '@agoric/nat'; import { parseVatSlot } from '../parseVatSlots.js'; // import { kdebug } from './kdebug.js'; @@ -50,7 +53,10 @@ export function makeCache(size, fetch, store) { } else { lruHead = undefined; } + const deadEntry = lruTail; lruTail = lruTail.prev; + deadEntry.next = undefined; + deadEntry.prev = undefined; } }, flush() { @@ -148,7 +154,7 @@ export function makeCache(size, fetch, store) { * instances, writing any changed state to the persistent store. This * provided for testing; it otherwise has little use. * - * - `makeVirtualObjectRepresentation` will provide a useeable, in-memory + * - `makeVirtualObjectRepresentative` will provide a useeable, in-memory * version of a virtual object, given its vat slot ID. This is used when * deserializing a reference to an object that has been received in a message * or is part of the persistent state of another virtual object that is being @@ -188,6 +194,65 @@ export function makeVirtualObjectManager( syscall.vatstoreSet(`vom.${vobjID}`, JSON.stringify(rawData)); } + function possibleVirtualObjectDeath(vobjID) { + if (!isVrefReachable(vobjID) && !getValForSlot(vobjID)) { + const [exported, refCount] = getRefCounts(vobjID); + if (exported === 0 && refCount === 0) { + // TODO: decrement refcounts on vrefs in the virtualized data being deleted + syscall.vatstoreDelete(`vom.${vobjID}`); + syscall.vatstoreDelete(`vom.${vobjID}.refCount`); + } + } + } + + function getRefCounts(vobjID) { + const rawCounts = syscall.vatstoreGet(`vom.${vobjID}.refCount`); + if (rawCounts) { + return rawCounts.split(' ').map(Number); + } else { + return [0, 0]; + } + } + + function setRefCounts(vobjID, exported, count) { + syscall.vatstoreSet( + `vom.${vobjID}.refCount`, + `${Nat(exported)} ${Nat(count)}`, + ); + if (exported === 0 && count === 0) { + possibleVirtualObjectDeath(vobjID); + } + } + + function setExported(vobjID, newSetting) { + const [wasExported, refCount] = getRefCounts(vobjID); + const isNowExported = Number(newSetting); + if (wasExported !== isNowExported) { + setRefCounts(vobjID, isNowExported, refCount); + } + } + + function incRefCount(vobjID) { + const [exported, oldCount] = getRefCounts(vobjID); + if (oldCount === 0) { + // TODO: right now we are not tracking actual refcounts, so for now a + // refcount of 0 means never referenced and a refcount of 1 means + // referenced at least once in the past. Once actual refcounts are + // working (notably including calling decref at the appropriate times), + // take out the above if. + setRefCounts(vobjID, exported, oldCount + 1); + } + } + + // TODO: reenable once used; it's commented out just to make eslint shut up + /* + function decRefCount(vobjID) { + const [exported, oldCount] = getRefCounts(vobjID); + assert(oldCount > 0, `attempt to decref ${vobjID} below 0`); + setRefCounts(vobjID, exported, oldCount - 1); + } + */ + const cache = makeCache(cacheSize, fetch, store); /** @@ -212,8 +277,9 @@ export function makeVirtualObjectManager( // We track imports, to preserve their vrefs against syscall.dropImport // when the Presence goes away. function addReachablePresenceRef(vref) { - const { type, allocatedByVat } = parseVatSlot(vref); - if (type === 'object' && !allocatedByVat) { + // XXX TODO including virtual objects gives lie to the name, but this hack should go away with VO refcounts + const { type, allocatedByVat, virtual } = parseVatSlot(vref); + if (type === 'object' && (!allocatedByVat || virtual)) { reachableVrefs.add(vref); } } @@ -264,12 +330,16 @@ export function makeVirtualObjectManager( const reachableRemotables = new Set(); function addReachableRemotableRef(vref) { const { type, virtual, allocatedByVat } = parseVatSlot(vref); - if (type === 'object' && !virtual && allocatedByVat) { - // exported non-virtual object: Remotable - const remotable = getValForSlot(vref); - assert(remotable, X`no remotable for ${vref}`); - // console.log(`adding ${vref} to reachableRemotables`); - reachableRemotables.add(remotable); + if (type === 'object' && allocatedByVat) { + if (virtual) { + incRefCount(vref); + } else { + // exported non-virtual object: Remotable + const remotable = getValForSlot(vref); + assert(remotable, X`no remotable for ${vref}`); + // console.log(`adding ${vref} to reachableRemotables`); + reachableRemotables.add(remotable); + } } } @@ -542,29 +612,35 @@ export function makeVirtualObjectManager( * selves, and state data. * * A representative is the manifestation of a virtual object that vat code has - * direct access to. A given virtual object can have multiple - * representatives: one is created when the instance is initially made and - * another is generated each time the instance's virtual object ID is - * deserialized, either when delivered as part of an incoming message or read - * as part of another virtual object's state. These representatives are not - * === but do obey the `sameKey` equivalence relation. In particular, methods - * invoked on them all operate on the same underyling virtual object state. A - * representative is garbage collectable once it becomes unreferenced in the - * vat. + * direct access to. A given virtual object can have at most one + * representative, which will be created as needed. This will happen when the + * instance is initially made, and can also happen (if it does not already + * exist) when the instance's virtual object ID is deserialized, either when + * delivered as part of an incoming message or read as part of another virtual + * object's state. A representative will be kept alive in memory as long as + * there is a variable somewhere that references it directly or indirectly. + * However, if a representative becomes unreferenced in memory it is subject + * to garbage collection, leaving the representation that is kept in the vat + * store as the record of its state from which a mew representative can be + * reconsituted at need. Since only one representative exists at a time, + * references to them may be compared with the equality operator (===). + * Although the identity of a representative can change over time, this is + * never visible to code running in the vat. Methods invoked on a + * representative always operate on the underyling virtual object state. * * The inner self represents the in-memory information about an object, aside * from its state. There is an inner self for each virtual object that is * currently resident in memory; that is, there is an inner self for each - * virtual object for which there is currently at least one representative - * present somewhere in the vat. The inner self maintains two pieces of - * information: its corresponding virtual object's virtual object ID, and a - * pointer to the virtual object's state in memory if the virtual object's - * state is, in fact, currently resident in memory. If the state is not in - * memory, the inner self's pointer to the state is null. In addition, the - * virtual object manager maintains an LRU cache of inner selves. Inner - * selves that are in the cache are not necessarily referenced by any existing - * representative, but are available to be used should such a representative - * be needed. How this all works will be explained in a moment. + * virtual object for which there is currently a representative present + * somewhere in the vat. The inner self maintains two pieces of information: + * its corresponding virtual object's virtual object ID, and a pointer to the + * virtual object's state in memory if the virtual object's state is, in fact, + * currently resident in memory. If the state is not in memory, the inner + * self's pointer to the state is null. In addition, the virtual object + * manager maintains an LRU cache of inner selves. Inner selves that are in + * the cache are not necessarily referenced by any existing representative, + * but are available to be used should such a representative be needed. How + * this all works will be explained in a moment. * * The state of a virtual object is a collection of mutable properties, each * of whose values is itself immutable and serializable. The methods of a @@ -585,10 +661,10 @@ export function makeVirtualObjectManager( * corresponding inner self is made to point at it, and then the inner self is * placed at the head of the LRU cache (causing the least recently used inner * self to fall off the end of the cache). If it *is* in memory, it is - * promoted to the head of the LRU cache but the contents of the cache remains - * unchanged. When an inner self falls off the end of the LRU, its reference - * to the state is nulled out and the object holding the state becomes garbage - * collectable. + * promoted to the head of the LRU cache but the overall contents of the cache + * remain unchanged. When an inner self falls off the end of the LRU, its + * reference to the state is nulled out and the object holding the state + * becomes garbage collectable. */ function makeKind(instanceKitMaker) { const kindID = `${allocateExportID()}`; @@ -712,7 +788,9 @@ export function makeVirtualObjectManager( VirtualObjectAwareWeakSet, isVrefReachable, isVrefRecognizable, + setExported, flushCache: cache.flush, makeVirtualObjectRepresentative, + possibleVirtualObjectDeath, }); } diff --git a/packages/SwingSet/test/virtualObjects/test-representatives.js b/packages/SwingSet/test/virtualObjects/test-representatives.js index 33d316bdc81..d930de78d76 100644 --- a/packages/SwingSet/test/virtualObjects/test-representatives.js +++ b/packages/SwingSet/test/virtualObjects/test-representatives.js @@ -279,29 +279,67 @@ test('exercise cache', async t => { // init cache - [] await make('thing1', true, T1); // make t1 - [t1] + t.deepEqual(log.shift(), ['get', `${thingID(1)}.refCount`, undefined]); + t.deepEqual(log.shift(), ['set', `${thingID(1)}.refCount`, '1 0']); + t.deepEqual(log, []); + await make('thing2', false, T2); // make t2 - [t2 t1] + t.deepEqual(log.shift(), ['get', `${thingID(2)}.refCount`, undefined]); + t.deepEqual(log.shift(), ['set', `${thingID(2)}.refCount`, '1 0']); + t.deepEqual(log, []); + await read(T1, 'thing1'); // refresh t1 - [t1 t2] await read(T2, 'thing2'); // refresh t2 - [t2 t1] await readHeld('thing1'); // refresh t1 - [t1 t2] await make('thing3', false, T3); // make t3 - [t3 t1 t2] + t.deepEqual(log.shift(), ['get', `${thingID(3)}.refCount`, undefined]); + t.deepEqual(log.shift(), ['set', `${thingID(3)}.refCount`, '1 0']); + t.deepEqual(log, []); + await make('thing4', false, T4); // make t4 - [t4 t3 t1 t2] + t.deepEqual(log.shift(), ['get', `${thingID(4)}.refCount`, undefined]); + t.deepEqual(log.shift(), ['set', `${thingID(4)}.refCount`, '1 0']); t.deepEqual(log, []); + await make('thing5', false, T5); // evict t2, make t5 - [t5 t4 t3 t1] + t.deepEqual(log.shift(), ['get', `${thingID(5)}.refCount`, undefined]); + t.deepEqual(log.shift(), ['get', `${thingID(2)}.refCount`, '1 0']); t.deepEqual(log.shift(), ['set', thingID(2), thingVal('thing2')]); + t.deepEqual(log.shift(), ['set', `${thingID(5)}.refCount`, '1 0']); + t.deepEqual(log, []); + await make('thing6', false, T6); // evict t1, make t6 - [t6 t5 t4 t3] + t.deepEqual(log.shift(), ['get', `${thingID(6)}.refCount`, undefined]); t.deepEqual(log.shift(), ['set', thingID(1), thingVal('thing1')]); + t.deepEqual(log.shift(), ['set', `${thingID(6)}.refCount`, '1 0']); + t.deepEqual(log, []); + await make('thing7', false, T7); // evict t3, make t7 - [t7 t6 t5 t4] + t.deepEqual(log.shift(), ['get', `${thingID(7)}.refCount`, undefined]); + t.deepEqual(log.shift(), ['get', `${thingID(3)}.refCount`, '1 0']); t.deepEqual(log.shift(), ['set', thingID(3), thingVal('thing3')]); + t.deepEqual(log.shift(), ['set', `${thingID(7)}.refCount`, '1 0']); + t.deepEqual(log, []); + await make('thing8', false, T8); // evict t4, make t8 - [t8 t7 t6 t5] + t.deepEqual(log.shift(), ['get', `${thingID(8)}.refCount`, undefined]); + t.deepEqual(log.shift(), ['get', `${thingID(4)}.refCount`, '1 0']); t.deepEqual(log.shift(), ['set', thingID(4), thingVal('thing4')]); + t.deepEqual(log.shift(), ['set', `${thingID(8)}.refCount`, '1 0']); + t.deepEqual(log, []); await read(T2, 'thing2'); // reanimate t2, evict t5 - [t2 t8 t7 t6] t.deepEqual(log.shift(), ['get', thingID(2), thingVal('thing2')]); + t.deepEqual(log.shift(), ['get', `${thingID(5)}.refCount`, '1 0']); t.deepEqual(log.shift(), ['set', thingID(5), thingVal('thing5')]); + t.deepEqual(log, []); + await readHeld('thing1'); // reanimate t1, evict t6 - [t1 t2 t8 t7] t.deepEqual(log.shift(), ['get', thingID(1), thingVal('thing1')]); + t.deepEqual(log.shift(), ['get', `${thingID(6)}.refCount`, '1 0']); t.deepEqual(log.shift(), ['set', thingID(6), thingVal('thing6')]); + t.deepEqual(log, []); await write(T2, 'thing2 updated'); // refresh t2 - [t2 t1 t8 t7] await writeHeld('thing1 updated'); // refresh t1 - [t1 t2 t8 t7] @@ -309,30 +347,131 @@ test('exercise cache', async t => { await read(T8, 'thing8'); // refresh t8 - [t8 t1 t2 t7] await read(T7, 'thing7'); // refresh t7 - [t7 t8 t1 t2] t.deepEqual(log, []); + await read(T6, 'thing6'); // reanimate t6, evict t2 - [t6 t7 t8 t1] t.deepEqual(log.shift(), ['get', thingID(6), thingVal('thing6')]); + t.deepEqual(log.shift(), ['get', `${thingID(2)}.refCount`, '1 0']); t.deepEqual(log.shift(), ['set', thingID(2), thingVal('thing2 updated')]); + t.deepEqual(log, []); + await read(T5, 'thing5'); // reanimate t5, evict t1 - [t5 t6 t7 t8] t.deepEqual(log.shift(), ['get', thingID(5), thingVal('thing5')]); t.deepEqual(log.shift(), ['set', thingID(1), thingVal('thing1 updated')]); + t.deepEqual(log, []); + await read(T4, 'thing4'); // reanimate t4, evict t8 - [t4 t5 t6 t7] t.deepEqual(log.shift(), ['get', thingID(4), thingVal('thing4')]); t.deepEqual(log.shift(), ['set', thingID(8), thingVal('thing8')]); + t.deepEqual(log, []); + await read(T3, 'thing3'); // reanimate t3, evict t7 - [t3 t4 t5 t6] t.deepEqual(log.shift(), ['get', thingID(3), thingVal('thing3')]); + t.deepEqual(log.shift(), ['get', `${thingID(7)}.refCount`, '1 0']); t.deepEqual(log.shift(), ['set', thingID(7), thingVal('thing7')]); + t.deepEqual(log, []); await read(T2, 'thing2 updated'); // reanimate t2, evict t6 - [t2 t3 t4 t5] t.deepEqual(log.shift(), ['get', thingID(2), thingVal('thing2 updated')]); + t.deepEqual(log.shift(), ['get', `${thingID(6)}.refCount`, '1 0']); + t.deepEqual(log, []); + await readHeld('thing1 updated'); // reanimate t1, evict t5 - [t1 t2 t3 t4] t.deepEqual(log.shift(), ['get', thingID(1), thingVal('thing1 updated')]); + t.deepEqual(log.shift(), ['get', `${thingID(5)}.refCount`, '1 0']); + t.deepEqual(log, []); await forgetHeld(); // cache unchanged - [t1 t2 t3 t4] + t.deepEqual(log.shift(), ['get', `${thingID(1)}.refCount`, '1 0']); + t.deepEqual(log, []); + await hold(T8); // cache unchanged - [t1 t2 t3 t4] + t.deepEqual(log.shift(), ['get', `${thingID(4)}.refCount`, '1 0']); t.deepEqual(log, []); + await read(T7, 'thing7'); // reanimate t7, evict t4 - [t7 t1 t2 t3] t.deepEqual(log.shift(), ['get', thingID(7), thingVal('thing7')]); + t.deepEqual(log.shift(), ['get', `${thingID(3)}.refCount`, '1 0']); + t.deepEqual(log, []); + await writeHeld('thing8 updated'); // reanimate t8, evict t3 - [t8 t7 t1 t2] t.deepEqual(log.shift(), ['get', thingID(8), thingVal('thing8')]); t.deepEqual(log, []); }); + +test('virtual object gc', async t => { + /* + With respect to any given case, we really only have two variables to fiddle + with: whether a reference to the VO is exported, and whether a reference to + the VO is retained in memory in the vat. + + For the first variable, there are three possible states: not exported, + exported and then retained externally, exported and then dropped externally. + + For the second variable, there are two possible states: retained internally, + and dropped internally. + + That would seem like 6 cases, but that's not quite true, since a reference + that is dropped internally before being exported can't ever be + exported. (Also if a VO has both an exported reference and an internal + reference and both are dropped, we still have to worry about what order + they're dropped in, but there's another test that worries about the order of + droppage issue). + + In this test, Bob generates 9 VOs of the same kind, with vrefs o+1/1 through + o+1/9. Bob dispenses some of these to Bootstrap, drops some of them, + retires some, etc. The breakdown is as follows: + + # Exported? Export Dropped? Local dropped? Delete? + 1 yes yes yes yes (dropped both locally and by export) + 2 yes yes no no (retained in local variable) + 3 yes no yes no (retained by export) + 4 no n/a yes yes (dropped before use) + 5,6,7 no n/a yes yes (dropped at end) + 8,9 no n/a no no (retained unused) + + Things 5, 6, and 7 are essentially the same as 4, but they are dropped by a + loop at the end, to make sure that that works. + + Things 8 and 9 are both the same. They are never used and so are retained + where they were originally stashed on creation + */ + + const config = { + bootstrap: 'bootstrap', + defaultManagerType: 'xs-worker', + vats: { + bob: { + sourceSpec: path.resolve(__dirname, 'vat-vom-gc-bob.js'), + creationOptions: { + virtualObjectCacheSize: 3, + }, + }, + bootstrap: { + sourceSpec: path.resolve(__dirname, 'vat-vom-gc-bootstrap.js'), + }, + }, + }; + + const hostStorage = provideHostStorage(); + + const c = await buildVatController(config, [], { hostStorage }); + c.pinVatRoot('bootstrap'); + + await c.run(); + t.deepEqual( + c.kpResolution(c.bootstrapResult), + capargs({ '@qclass': 'undefined' }), + ); + const remainingVOs = {}; + for (const key of hostStorage.kvStore.getKeys('v1.vs.', 'v1.vs/')) { + remainingVOs[key] = hostStorage.kvStore.get(key); + } + t.deepEqual(remainingVOs, { + 'v1.vs.vom.o+1/2': '{"label":{"body":"\\"thing #2\\"","slots":[]}}', + 'v1.vs.vom.o+1/2.refCount': '0 0', + 'v1.vs.vom.o+1/3': '{"label":{"body":"\\"thing #3\\"","slots":[]}}', + 'v1.vs.vom.o+1/3.refCount': '1 0', + 'v1.vs.vom.o+1/8': '{"label":{"body":"\\"thing #8\\"","slots":[]}}', + 'v1.vs.vom.o+1/9': '{"label":{"body":"\\"thing #9\\"","slots":[]}}', + }); +}); diff --git a/packages/SwingSet/test/virtualObjects/test-virtualObjectManager.js b/packages/SwingSet/test/virtualObjects/test-virtualObjectManager.js index dce45169fdb..ec741dc0c0e 100644 --- a/packages/SwingSet/test/virtualObjects/test-virtualObjectManager.js +++ b/packages/SwingSet/test/virtualObjects/test-virtualObjectManager.js @@ -46,6 +46,23 @@ function thingVal(counter, label, resetCounter) { }); } +function minThing(label) { + return thingVal(0, label, 0); +} + +function makeRefInstance(state) { + return { + init(value) { + state.value = value; + }, + self: Far('ref', { + setVal(value) { + state.value = value; + }, + }), + }; +} + function makeZotInstance(state) { return { init(arbitrary = 47, name = 'Bob', tag = 'say what?') { @@ -297,6 +314,211 @@ test('virtual object operations', t => { ]); }); +test('virtual object gc', t => { + const log = []; + const { + makeKind, + dumpStore, + setExported, + deleteEntry, + possibleVirtualObjectDeath, + } = makeFakeVirtualObjectManager({ cacheSize: 3, log }); + + const thingMaker = makeKind(makeThingInstance); + const refMaker = makeKind(makeRefInstance); + + // make a bunch of things which we'll use + // all virtual objects are born locally ref'd + const things = []; + for (let i = 1; i <= 9; i += 1) { + things.push(thingMaker(`thing #${i}`)); + } + t.is(log.shift(), `set vom.o+1/1 ${minThing('thing #1')}`); + t.is(log.shift(), `set vom.o+1/2 ${minThing('thing #2')}`); + t.is(log.shift(), `set vom.o+1/3 ${minThing('thing #3')}`); + t.is(log.shift(), `set vom.o+1/4 ${minThing('thing #4')}`); + t.is(log.shift(), `set vom.o+1/5 ${minThing('thing #5')}`); + t.deepEqual(log, []); + t.deepEqual(dumpStore(), [ + ['vom.o+1/1', minThing('thing #1')], + ['vom.o+1/2', minThing('thing #2')], + ['vom.o+1/3', minThing('thing #3')], + ['vom.o+1/4', minThing('thing #4')], + ['vom.o+1/5', minThing('thing #5')], + ]); + + // This is what the finalizer would do if the local reference was dropped and GC'd + function pretendGC(vref) { + deleteEntry(vref); + possibleVirtualObjectDeath(vref); + } + + // case 1: export, drop local ref, drop export + // export + setExported('o+1/1', true); + t.is(log.shift(), `get vom.o+1/1.refCount => undefined`); + t.is(log.shift(), `set vom.o+1/1.refCount 1 0`); + t.deepEqual(log, []); + // drop local ref -- should not delete because exported + pretendGC('o+1/1'); + t.is(log.shift(), `get vom.o+1/1.refCount => 1 0`); + t.deepEqual(log, []); + t.deepEqual(dumpStore(), [ + ['vom.o+1/1', minThing('thing #1')], + ['vom.o+1/1.refCount', '1 0'], + ['vom.o+1/2', minThing('thing #2')], + ['vom.o+1/3', minThing('thing #3')], + ['vom.o+1/4', minThing('thing #4')], + ['vom.o+1/5', minThing('thing #5')], + ]); + // drop export -- should delete + setExported('o+1/1', false); + t.is(log.shift(), `get vom.o+1/1.refCount => 1 0`); + t.is(log.shift(), `set vom.o+1/1.refCount 0 0`); + t.is(log.shift(), `get vom.o+1/1.refCount => 0 0`); + t.is(log.shift(), `delete vom.o+1/1`); + t.is(log.shift(), `delete vom.o+1/1.refCount`); + t.deepEqual(log, []); + t.deepEqual(dumpStore(), [ + ['vom.o+1/2', minThing('thing #2')], + ['vom.o+1/3', minThing('thing #3')], + ['vom.o+1/4', minThing('thing #4')], + ['vom.o+1/5', minThing('thing #5')], + ]); + + // case 2: export, drop export, drop local ref + // export + setExported('o+1/2', true); + t.is(log.shift(), `get vom.o+1/2.refCount => undefined`); + t.is(log.shift(), `set vom.o+1/2.refCount 1 0`); + t.deepEqual(log, []); + // drop export -- should not delete because ref'd locally + setExported('o+1/2', false); + t.is(log.shift(), `get vom.o+1/2.refCount => 1 0`); + t.is(log.shift(), `set vom.o+1/2.refCount 0 0`); + t.deepEqual(log, []); + t.deepEqual(dumpStore(), [ + ['vom.o+1/2', minThing('thing #2')], + ['vom.o+1/2.refCount', '0 0'], + ['vom.o+1/3', minThing('thing #3')], + ['vom.o+1/4', minThing('thing #4')], + ['vom.o+1/5', minThing('thing #5')], + ]); + // drop local ref -- should delete + pretendGC('o+1/2'); + t.is(log.shift(), `get vom.o+1/2.refCount => 0 0`); + t.is(log.shift(), `delete vom.o+1/2`); + t.is(log.shift(), `delete vom.o+1/2.refCount`); + t.deepEqual(log, []); + t.deepEqual(dumpStore(), [ + ['vom.o+1/3', minThing('thing #3')], + ['vom.o+1/4', minThing('thing #4')], + ['vom.o+1/5', minThing('thing #5')], + ]); + + // case 3: drop local ref with no prior export + // drop local ref -- should delete + pretendGC('o+1/3'); + t.is(log.shift(), `get vom.o+1/3.refCount => undefined`); + t.is(log.shift(), `delete vom.o+1/3`); + t.is(log.shift(), `delete vom.o+1/3.refCount`); + t.deepEqual(log, []); + t.deepEqual(dumpStore(), [ + ['vom.o+1/4', minThing('thing #4')], + ['vom.o+1/5', minThing('thing #5')], + ]); + + // case 4: ref virtually, export, drop local ref, drop export + // ref virtually + // eslint-disable-next-line no-unused-vars + const ref1 = refMaker(things[3]); + t.is(log.shift(), `set vom.o+1/6 ${minThing('thing #6')}`); + t.is(log.shift(), `get vom.o+1/4.refCount => undefined`); + t.is(log.shift(), `set vom.o+1/4.refCount 0 1`); + t.deepEqual(log, []); + t.deepEqual(dumpStore(), [ + ['vom.o+1/4', minThing('thing #4')], + ['vom.o+1/4.refCount', '0 1'], + ['vom.o+1/5', minThing('thing #5')], + ['vom.o+1/6', minThing('thing #6')], + ]); + // export + setExported('o+1/4', true); + t.is(log.shift(), `get vom.o+1/4.refCount => 0 1`); + t.is(log.shift(), `set vom.o+1/4.refCount 1 1`); + t.deepEqual(log, []); + // drop local ref -- should not delete because ref'd virtually AND exported + pretendGC('o+1/4'); + t.deepEqual(log, []); + // drop export -- should not delete because ref'd virtually + setExported('o+1/4', false); + t.is(log.shift(), `get vom.o+1/4.refCount => 1 1`); + t.is(log.shift(), `set vom.o+1/4.refCount 0 1`); + t.deepEqual(log, []); + + // case 5: export, ref virtually, drop local ref, drop export + // export + setExported('o+1/5', true); + t.is(log.shift(), `get vom.o+1/5.refCount => undefined`); + t.is(log.shift(), `set vom.o+1/5.refCount 1 0`); + t.deepEqual(log, []); + // ref virtually + // eslint-disable-next-line no-unused-vars + const ref2 = refMaker(things[4]); + t.is(log.shift(), `set vom.o+1/7 ${minThing('thing #7')}`); + t.is(log.shift(), `get vom.o+1/5.refCount => 1 0`); + t.is(log.shift(), `set vom.o+1/5.refCount 1 1`); + t.deepEqual(log, []); + t.deepEqual(dumpStore(), [ + ['vom.o+1/4', minThing('thing #4')], + ['vom.o+1/4.refCount', '0 1'], + ['vom.o+1/5', minThing('thing #5')], + ['vom.o+1/5.refCount', '1 1'], + ['vom.o+1/6', minThing('thing #6')], + ['vom.o+1/7', minThing('thing #7')], + ]); + // drop local ref -- should not delete because ref'd virtually AND exported + pretendGC('o+1/5'); + t.deepEqual(log, []); + // drop export -- should not delete because ref'd virtually + setExported('o+1/5', false); + t.is(log.shift(), `get vom.o+1/5.refCount => 1 1`); + t.is(log.shift(), `set vom.o+1/5.refCount 0 1`); + t.deepEqual(log, []); + + // case 6: ref virtually, drop local ref + // ref virtually + // eslint-disable-next-line no-unused-vars + const ref3 = refMaker(things[5]); + t.is(log.shift(), `set vom.o+1/8 ${minThing('thing #8')}`); + t.is(log.shift(), `get vom.o+1/6.refCount => undefined`); + t.is(log.shift(), `set vom.o+1/6.refCount 0 1`); + t.deepEqual(log, []); + t.deepEqual(dumpStore(), [ + ['vom.o+1/4', minThing('thing #4')], + ['vom.o+1/4.refCount', '0 1'], + ['vom.o+1/5', minThing('thing #5')], + ['vom.o+1/5.refCount', '0 1'], + ['vom.o+1/6', minThing('thing #6')], + ['vom.o+1/6.refCount', '0 1'], + ['vom.o+1/7', minThing('thing #7')], + ['vom.o+1/8', minThing('thing #8')], + ]); + // drop local ref -- should not delete because ref'd virtually + pretendGC('o+1/6'); + t.deepEqual(log, []); + t.deepEqual(dumpStore(), [ + ['vom.o+1/4', minThing('thing #4')], + ['vom.o+1/4.refCount', '0 1'], + ['vom.o+1/5', minThing('thing #5')], + ['vom.o+1/5.refCount', '0 1'], + ['vom.o+1/6', minThing('thing #6')], + ['vom.o+1/6.refCount', '0 1'], + ['vom.o+1/7', minThing('thing #7')], + ['vom.o+1/8', minThing('thing #8')], + ]); +}); + test('weak store operations', t => { const { makeWeakStore, makeKind } = makeFakeVirtualObjectManager({ cacheSize: 3, diff --git a/packages/SwingSet/test/virtualObjects/vat-vom-gc-bob.js b/packages/SwingSet/test/virtualObjects/vat-vom-gc-bob.js new file mode 100644 index 00000000000..0884bddb62c --- /dev/null +++ b/packages/SwingSet/test/virtualObjects/vat-vom-gc-bob.js @@ -0,0 +1,60 @@ +/* global makeKind */ +import { E } from '@agoric/eventual-send'; + +const things = []; + +export function buildRootObject(_vatPowers) { + function makeThingInstance(state) { + return { + init(label) { + state.label = label; + }, + self: { + getLabel() { + return state.label; + }, + }, + }; + } + + const thingMaker = makeKind(makeThingInstance); + let nextThingNumber = 0; + + return harden({ + prepare() { + things.push(null); + for (let i = 1; i <= 9; i += 1) { + things.push(thingMaker(`thing #${i}`)); + } + }, + getThing(forWhom) { + let thing; + do { + thing = things[nextThingNumber]; + nextThingNumber += 1; + } while (!thing); + + if (nextThingNumber === 3) { + thing.getLabel(); + things[4].getLabel(); + things[4] = null; // arbitrarily drop one before sending it, but don't drop the one we're sending + } else { + thing.getLabel(); + things[nextThingNumber - 1] = null; // drop the one we're sending + } + E(forWhom).deliverThing(thing); + thing = null; + }, + finish() { + while (nextThingNumber < 8) { + const deadThing = things[nextThingNumber]; + if (deadThing) { + deadThing.getLabel(); + things[nextThingNumber] = null; + } + nextThingNumber += 1; + } + console.log(`Bob finishing`); + }, + }); +} diff --git a/packages/SwingSet/test/virtualObjects/vat-vom-gc-bootstrap.js b/packages/SwingSet/test/virtualObjects/vat-vom-gc-bootstrap.js new file mode 100644 index 00000000000..256e367400e --- /dev/null +++ b/packages/SwingSet/test/virtualObjects/vat-vom-gc-bootstrap.js @@ -0,0 +1,30 @@ +import { E } from '@agoric/eventual-send'; + +export function buildRootObject(_vatPowers) { + let other; + let bob; + let me; + let goCount = 3; + return harden({ + async bootstrap(vats) { + me = vats.bootstrap; + bob = vats.bob; + E(bob).prepare(); + await E(me).go(); + }, + go() { + if (goCount > 0) { + E(bob).getThing(me); + } else { + E(bob).finish(); + } + goCount -= 1; + }, + deliverThing(thing) { + // eslint thinks 'other' is unused, but eslint is wrong. + // eslint-disable-next-line no-unused-vars + other = thing; + E(me).go(); + }, + }); +} diff --git a/packages/SwingSet/tools/fakeVirtualObjectManager.js b/packages/SwingSet/tools/fakeVirtualObjectManager.js index b69ffa9c1ee..1eee6e7a4e3 100644 --- a/packages/SwingSet/tools/fakeVirtualObjectManager.js +++ b/packages/SwingSet/tools/fakeVirtualObjectManager.js @@ -96,6 +96,9 @@ export function makeFakeVirtualObjectManager(options = {}) { } function deleteEntry(slot, val) { + if (!val) { + val = getValForSlot(slot); + } slotToVal.delete(slot); valToSlot.delete(val); } @@ -107,6 +110,8 @@ export function makeFakeVirtualObjectManager(options = {}) { VirtualObjectAwareWeakMap, VirtualObjectAwareWeakSet, isVrefReachable, + setExported, + possibleVirtualObjectDeath, flushCache, } = makeVirtualObjectManager( fakeSyscall, @@ -124,6 +129,8 @@ export function makeFakeVirtualObjectManager(options = {}) { VirtualObjectAwareWeakMap, VirtualObjectAwareWeakSet, isVrefReachable, + setExported, + possibleVirtualObjectDeath, }; const debugTools = { diff --git a/packages/swingset-runner/demo/vatStore3/bootstrap.js b/packages/swingset-runner/demo/vatStore3/bootstrap.js new file mode 100644 index 00000000000..256e367400e --- /dev/null +++ b/packages/swingset-runner/demo/vatStore3/bootstrap.js @@ -0,0 +1,30 @@ +import { E } from '@agoric/eventual-send'; + +export function buildRootObject(_vatPowers) { + let other; + let bob; + let me; + let goCount = 3; + return harden({ + async bootstrap(vats) { + me = vats.bootstrap; + bob = vats.bob; + E(bob).prepare(); + await E(me).go(); + }, + go() { + if (goCount > 0) { + E(bob).getThing(me); + } else { + E(bob).finish(); + } + goCount -= 1; + }, + deliverThing(thing) { + // eslint thinks 'other' is unused, but eslint is wrong. + // eslint-disable-next-line no-unused-vars + other = thing; + E(me).go(); + }, + }); +} diff --git a/packages/swingset-runner/demo/vatStore3/vat-bob.js b/packages/swingset-runner/demo/vatStore3/vat-bob.js new file mode 100644 index 00000000000..0ce6d7db616 --- /dev/null +++ b/packages/swingset-runner/demo/vatStore3/vat-bob.js @@ -0,0 +1,59 @@ +/* global makeKind */ +import { E } from '@agoric/eventual-send'; + +const things = []; + +export function buildRootObject(_vatPowers) { + function makeThingInstance(state) { + return { + init(label) { + state.label = label; + }, + self: { + getLabel() { + return state.label; + }, + }, + }; + } + + const thingMaker = makeKind(makeThingInstance); + let nextThingNumber = 0; + + return harden({ + prepare() { + for (let i = 1; i <= 9; i += 1) { + things.push(thingMaker(`thing #${i}`)); + } + }, + getThing(forWhom) { + let thing; + do { + thing = things[nextThingNumber]; + nextThingNumber += 1; + } while (!thing); + + if (nextThingNumber === 2) { + console.log(`not nulling ${thing.getLabel()}`); + console.log(`nulling ${things[3].getLabel()} instead`); + things[3] = null; // arbitrarily drop one before sending it, but don't drop the one we're sending + } else { + console.log(`nulling ${thing.getLabel()}`); + things[nextThingNumber - 1] = null; // drop the one we're sending + } + E(forWhom).deliverThing(thing); + thing = null; + }, + finish() { + while (nextThingNumber < 7) { + const deadThing = things[nextThingNumber]; + if (deadThing) { + console.log(`final nulling ${deadThing.getLabel()}`); + things[nextThingNumber] = null; + } + nextThingNumber += 1; + } + console.log(`Bob finishing`); + }, + }); +} diff --git a/packages/swingset-runner/src/main.js b/packages/swingset-runner/src/main.js index 808ca5281cf..a6ae9ba1325 100644 --- a/packages/swingset-runner/src/main.js +++ b/packages/swingset-runner/src/main.js @@ -51,6 +51,7 @@ FLAGS may be: --initonly - initialize the swingset but exit without running it --lmdb - runs using LMDB as the data store (default) --memdb - runs using the non-persistent in-memory data store + --usexs - run vats using the the XS engine --dbdir DIR - specify where the data store should go (default BASEDIR) --dbsize SIZE - set the LMDB size limit to SIZE megabytes (default 2GB) --blockmode - run in block mode (checkpoint every BLOCKSIZE blocks) @@ -180,6 +181,7 @@ export async function main() { let dbDir = null; let dbSize = 0; let initOnly = false; + let useXS = false; while (argv[0] && argv[0].startsWith('-')) { const flag = argv.shift(); @@ -273,6 +275,10 @@ export async function main() { case '--lmdb': dbMode = flag; break; + case '--usexs': + case '--useXS': + useXS = true; + break; case '-v': case '--verbose': verbose = true; @@ -332,6 +338,9 @@ export async function main() { delete config.loopboxSenders; deviceEndowments.loopbox = { ...loopboxEndowments }; } + if (useXS) { + config.defaultManagerType = 'xs-worker'; + } if (launchIndirectly) { config = generateIndirectConfig(config); } @@ -530,6 +539,7 @@ export async function main() { if (statLogger) { statLogger.close(); } + controller.shutdown(); function getCrankNumber() { return Number(swingStore.kvStore.get('crankNumber')); diff --git a/packages/swingset-runner/src/slogulator.js b/packages/swingset-runner/src/slogulator.js index 52d0dc7069e..7222e08cd70 100644 --- a/packages/swingset-runner/src/slogulator.js +++ b/packages/swingset-runner/src/slogulator.js @@ -371,6 +371,11 @@ export function main() { } } + function doDeliverDropRetire(delivery, prefix = '') { + // prettier-ignore + p(`${prefix}recv-${delivery[0]}: [${delivery[1].map(r => pref(r)).join(' ')}]`); + } + function doDeliver(delivery, prefix) { switch (delivery[0]) { case 'message': @@ -379,6 +384,11 @@ export function main() { case 'notify': doDeliverNotify(delivery, prefix); break; + case 'dropExports': + case 'retireExports': + case 'retireImports': + doDeliverDropRetire(delivery, prefix); + break; default: p(`deliver: unknown deliver type "${delivery[0]}"`); break; @@ -456,6 +466,10 @@ export function main() { p(`${tag}: ${key} := '${value}'`); } + function doSyscallDropRetire(tag, entry) { + p(`send-${tag}: [${entry[1].map(r => pref(r)).join(' ')}]`); + } + function doSyscallExit(tag, entry) { const failure = kernelSpace ? entry[2] : entry[1]; const value = kernelSpace ? entry[3] : entry[2]; @@ -488,6 +502,11 @@ export function main() { case 'vatstoreSet': doSyscallVatstoreSet(tag, syscall); break; + case 'dropImports': + case 'retireExports': + case 'retireImports': + doSyscallDropRetire(tag, syscall); + break; default: p(`syscall: unknown syscall ${currentSyscallName}`); break; @@ -617,6 +636,9 @@ export function main() { case 'subscribe': case 'vatstoreDelete': case 'vatstoreSet': + case 'dropImports': + case 'retireExports': + case 'retireImports': if (value !== null) { p(`${tag}: unexpected value ${value}`); }