feat(swing-store): budget-limited deletion of snapshot and transcripts

Both `snapStore.deleteVatSnapshots()` and
`transcriptStore.deleteVatTranscripts()` now take a numeric `budget=`
argument, which will limit the number of snapshots or transcript spans
deleted in each call. Both return a `{ done, cleanups }` record so the
caller knows when to stop calling.

This enables the slow deletion of large vats (lots of transcript spans
or snapshots), a small number of items at a time. Recommended budget
is 5, which (given SwingSet's `snapInterval=200` default) will cause
the deletion of 1000 rows from the `transcriptItems` table each call,
which shouldn't take more than 100ms.

Without this, the kernel's attempt to slowly delete a terminated vat
would succeed in slowly draining the kvStore, but would trigger a
gigantic SQL transaction at the end, as it deleted every transcript
item in the vat's history. The worst-case example I found would be the
mainnet chain's v43-walletFactory, which (as of apr-2024) has 8.2M
transcript items in 40k spans. A fast machine takes two seconds just
to count all the items, and deletion took 22 *minutes*, with a
`swingstore.wal` file that peaked at 27 GiB. This would cause an
enormous chain stall at some surprising point in time weeks or months
after the vat was first terminated. In addition, both the transcript
spans and the snapshot records are shadowed into IAVL (via
`export-data`) for integrity, and deleting 40k+40k=80k IAVL records in
a single block might cause some significant churn too.

refs #8928

packages/swing-store/src/snapStore.js

@@ -37,7 +37,7 @@ import { buffer } from './util.js';
  *   loadSnapshot: (vatID: string) => AsyncIterableIterator<Uint8Array>,
  *   saveSnapshot: (vatID: string, snapPos: number, snapshotStream: AsyncIterable<Uint8Array>) => Promise<SnapshotResult>,
  *   deleteAllUnusedSnapshots: () => void,
- *   deleteVatSnapshots: (vatID: string) => void,
+ *   deleteVatSnapshots: (vatID: string, budget?: number) => { done: boolean, cleanups: number },
  *   stopUsingLastSnapshot: (vatID: string) => void,
  *   getSnapshotInfo: (vatID: string) => SnapshotInfo,
  * }} SnapStore
@@ -352,6 +352,11 @@ export function makeSnapStore(
     WHERE vatID = ?
   `);
 
+  const sqlDeleteOneVatSnapshot = db.prepare(`
+    DELETE FROM snapshots
+    WHERE vatID = ? AND snapPos = ?
+  `);
+
   const sqlGetSnapshotList = db.prepare(`
     SELECT snapPos
     FROM snapshots
@@ -360,20 +365,93 @@ export function makeSnapStore(
   `);
   sqlGetSnapshotList.pluck(true);
 
+  const sqlGetSnapshotListLimited = db.prepare(`
+    SELECT snapPos
+    FROM snapshots
+    WHERE vatID = ?
+    ORDER BY snapPos ASC
+    LIMIT ?
+  `);
+  sqlGetSnapshotListLimited.pluck(true);
+
+  /**
+   * @param {string} vatID
+   * @returns {boolean}
+   */
+  function hasSnapshots(vatID) {
+    return !!sqlGetSnapshotListLimited.all(vatID, 1).length;
+  }
+
   /**
-   * Delete all snapshots for a given vat (for use when, e.g., a vat is terminated)
    *
    * @param {string} vatID
+   * @param {number} budget
+   * @returns {{ done: boolean, cleanups: number }}
    */
-  function deleteVatSnapshots(vatID) {
+  function deleteSomeVatSnapshots(vatID, budget) {
+    // Unlike transcripts, here we delete the oldest snapshots first,
+    // to simplify the logic: we delete the only inUse=1 snapshot
+    // last, and then immediately delete the .current record, at which
+    // point we're done. This has a side-effect of keeping the unused
+    // snapshot in the export artifacts longer, but it doesn't seem
+    // worth fixing.
     ensureTxn();
-    const deletions = sqlGetSnapshotList.all(vatID);
+    assert(budget >= 1);
+    let cleanups = 0;
+    const deletions = sqlGetSnapshotListLimited.all(vatID, budget);
+    if (!deletions.length) {
+      return { done: true, cleanups };
+    }
     for (const snapPos of deletions) {
       const exportRec = snapshotRec(vatID, snapPos, undefined);
       noteExport(snapshotMetadataKey(exportRec), undefined);
+      cleanups += 1;
+      sqlDeleteOneVatSnapshot.run(vatID, snapPos);
+    }
+    if (hasSnapshots(vatID)) {
+      // if any snapshots remain, even the inUse=1, ask to keep going
+      return { done: false, cleanups };
     }
+    // if we reach here, the last sqlDeleteOneVatSnapshot() in that
+    // loop had deleted the inUse=1 snapshot and the corresponding
+    // snapshotMetadataKey, so now it is time to delete the .current
+    // record and inform the kernel that we're done
     noteExport(currentSnapshotMetadataKey({ vatID }), undefined);
+    return { done: true, cleanups };
+  }
+
+  /**
+   *
+   * @param {string} vatID
+   */
+  function deleteAllVatSnapshots(vatID) {
+    ensureTxn();
+    const deletions = sqlGetSnapshotList.all(vatID);
+    for (const snapPos of deletions) {
+      const exportRec = snapshotRec(vatID, snapPos, undefined);
+      noteExport(snapshotMetadataKey(exportRec), undefined);
+    }
+    // fastest to delete them all in a single DB statement
     sqlDeleteVatSnapshots.run(vatID);
+    noteExport(currentSnapshotMetadataKey({ vatID }), undefined);
+  }
+
+  /**
+   * Delete some or all snapshots for a given vat (for use when, e.g.,
+   * a vat is terminated)
+   *
+   * @param {string} vatID
+   * @param {number} [budget]
+   * @returns {{ done: boolean, cleanups: number }}
+   */
+  function deleteVatSnapshots(vatID, budget = undefined) {
+    if (budget) {
+      return deleteSomeVatSnapshots(vatID, budget);
+    } else {
+      deleteAllVatSnapshots(vatID);
+      // if you didn't set a budget, you won't be counting deletions
+      return { done: true, cleanups: 0 };
+    }
   }
 
   const sqlGetSnapshotInfo = db.prepare(`

packages/swing-store/src/transcriptStore.js

@@ -18,7 +18,7 @@ import { createSHA256 } from './hasher.js';
  *   rolloverSpan: (vatID: string) => number,
  *   rolloverIncarnation: (vatID: string) => number,
  *   getCurrentSpanBounds: (vatID: string) => { startPos: number, endPos: number, hash: string, incarnation: number },
- *   deleteVatTranscripts: (vatID: string) => void,
+ *   deleteVatTranscripts: (vatID: string, budget?: number) => { done: boolean, cleanups: number },
  *   addItem: (vatID: string, item: string) => void,
  *   readSpan: (vatID: string, startPos?: number) => IterableIterator<string>,
  * }} TranscriptStore
@@ -314,12 +314,64 @@ export function makeTranscriptStore(
     ORDER BY startPos
   `);
 
+  const sqlGetSomeVatSpans = db.prepare(`
+    SELECT vatID, startPos, endPos, isCurrent
+    FROM transcriptSpans
+    WHERE vatID = ?
+    ORDER BY startPos DESC
+    LIMIT ?
+  `);
+
+  const sqlDeleteVatSpan = db.prepare(`
+    DELETE FROM transcriptSpans
+    WHERE vatID = ? AND startPos = ?
+  `);
+
+  const sqlDeleteSomeItems = db.prepare(`
+    DELETE FROM transcriptItems
+    WHERE vatID = ? AND position >= ? AND position < ?
+  `);
+
   /**
-   * Delete all transcript data for a given vat (for use when, e.g., a vat is terminated)
    *
    * @param {string} vatID
+   * @returns {boolean}
    */
-  function deleteVatTranscripts(vatID) {
+  function hasSpans(vatID) {
+    const spans = sqlGetSomeVatSpans.all(vatID, 1);
+    return !!spans.length;
+  }
+
+  /**
+   *
+   * @param {string} vatID
+   * @param {number} budget
+   * @returns {{ done: boolean, cleanups: number }}
+   */
+  function deleteSomeVatTranscripts(vatID, budget) {
+    ensureTxn();
+    assert(budget >= 1);
+    let cleanups = 0;
+    // this query is ORDER BY startPos DESC, so we delete the
+    // isCurrent=1 span first, which causes export to ignore the
+    // entire vat (good, since it's deleted)
+    const deletions = sqlGetSomeVatSpans.all(vatID, budget);
+    if (!deletions.length) {
+      return { done: true, cleanups };
+    }
+    for (const rec of deletions) {
+      noteExport(spanMetadataKey(rec), undefined);
+      sqlDeleteVatSpan.run(vatID, rec.startPos);
+      sqlDeleteSomeItems.run(vatID, rec.startPos, rec.endPos);
+      cleanups += 1;
+    }
+    if (hasSpans(vatID)) {
+      return { done: false, cleanups };
+    }
+    return { done: true, cleanups };
+  }
+
+  function deleteAllVatTranscripts(vatID) {
     ensureTxn();
     const deletions = sqlGetVatSpans.all(vatID);
     for (const rec of deletions) {
@@ -329,6 +381,24 @@ export function makeTranscriptStore(
     sqlDeleteVatSpans.run(vatID);
   }
 
+  /**
+   * Delete some or all transcript data for a given vat (for use when,
+   * e.g., a vat is terminated)
+   *
+   * @param {string} vatID
+   * @param {number} [budget]
+   * @returns {{ done: boolean, cleanups: number }}
+   */
+  function deleteVatTranscripts(vatID, budget = undefined) {
+    if (budget) {
+      return deleteSomeVatTranscripts(vatID, budget);
+    } else {
+      deleteAllVatTranscripts(vatID);
+      // no budget? no accounting.
+      return { done: true, cleanups: 0 };
+    }
+  }
+
   const sqlGetAllSpanMetadata = db.prepare(`
     SELECT vatID, startPos, endPos, hash, isCurrent, incarnation
     FROM transcriptSpans
@@ -379,6 +449,12 @@ export function makeTranscriptStore(
    * The only code path which could use 'false' would be `swingstore.dump()`,
    * which takes the same flag.
    *
+   * Note that when a vat is terminated and has been partially
+   * deleted, we will retain (and return) a subset of the metadata
+   * records, because they must be deleted in-consensus and with
+   * updates to the noteExport hook. But we don't create any artifacts
+   * for the terminated vats, even for the spans that remain,
+   *
    * @yields {readonly [key: string, value: string]}
    * @returns {IterableIterator<readonly [key: string, value: string]>}
    *    An iterator over pairs of [spanMetadataKey, rec], where `rec` is a
@@ -432,9 +508,16 @@ export function makeTranscriptStore(
         }
       }
     } else if (artifactMode === 'archival') {
-      // everything
+      // every span for all vatIDs that have an isCurrent span (to
+      // ignore terminated/partially-deleted vats)
+      const vatIDs = new Set();
+      for (const { vatID } of sqlGetCurrentSpanMetadata.iterate()) {
+        vatIDs.add(vatID);
+      }
       for (const rec of sqlGetAllSpanMetadata.iterate()) {
-        yield spanArtifactName(rec);
+        if (vatIDs.has(rec.vatID)) {
+          yield spanArtifactName(rec);
+        }
       }
     } else if (artifactMode === 'debug') {
       // everything that is a complete span