Merge pull request #7659 from Agoric/7655-baggage-check

Tool for auditing dangling kindID references

packages/SwingSet/misc-tools/baggage-check-tool.js

@@ -0,0 +1,19 @@
+import process from 'process';
+import sqlite3 from 'better-sqlite3';
+import '@endo/init/debug.js';
+import { checkBaggage } from '../tools/baggage-check.js';
+
+function main() {
+  if (process.argv.length !== 4) {
+    console.error('usage: node baggage-check-tool.js VATID DBPATH');
+    process.exit(1);
+  }
+  const argv = process.argv.slice(2);
+  const vatID = argv[0];
+  const dbPath = argv[1];
+
+  const db = sqlite3(dbPath);
+  checkBaggage(db, vatID, true);
+}
+
+main();

packages/SwingSet/package.json

@@ -52,6 +52,7 @@
     "@endo/promise-kit": "^0.2.56",
     "@endo/ses-ava": "^0.2.40",
     "@endo/zip": "^0.2.31",
+    "better-sqlite3": "^8.2.0",
     "ansi-styles": "^6.2.1",
     "anylogger": "^0.21.0",
     "import-meta-resolve": "^2.2.1",

packages/SwingSet/tools/baggage-check.js

@@ -0,0 +1,194 @@
+import { makeMarshal } from '@endo/marshal';
+import { Far } from '@endo/far';
+
+/* eslint-disable no-use-before-define */
+
+function fakeSTV(slot, iface = 'Remotable') {
+  return Far(iface, {
+    getSlot: () => slot,
+  });
+}
+
+const marshaller = makeMarshal(undefined, fakeSTV, {
+  serializeBodyFormat: 'smallcaps',
+});
+const { fromCapData } = marshaller;
+
+function decodeValueString(s) {
+  const parsed = JSON.parse(s);
+  const value = fromCapData(parsed);
+  return [value, parsed.slots];
+}
+
+function extractDurableRefParts(ref) {
+  const patt = /^o\+d([0-9]+)\/([0-9]+)$/;
+  const matches = ref.match(patt);
+  if (matches) {
+    return [+matches[1], +matches[2]];
+  } else {
+    return null;
+  }
+}
+
+export function checkBaggage(db, vatID, verbose = false) {
+  const sqlKVDump = db.prepare(`
+    SELECT key, value
+    FROM kvStore
+    WHERE key GLOB ?
+  `);
+
+  const sqlKVGet = db.prepare(`
+    SELECT value
+    FROM kvStore
+    WHERE key = ?
+  `);
+  sqlKVGet.pluck(true);
+
+  const vatStoreKeyRoot = `${vatID}.vs`;
+  const vomKeyRoot = `${vatStoreKeyRoot}.vom`;
+  const dkindKeyRoot = `${vomKeyRoot}.dkind`;
+  const vcKeyRoot = `${vatStoreKeyRoot}.vc`;
+
+  // Kinds which are known to have definitions.  Initially these are just those
+  // whose definitions are built in.
+  const predefinedKinds = new Map(); // kindID -> tag
+
+  predefinedKinds.set(1, 'kindHandle');
+  const builtInKinds = JSON.parse(
+    sqlKVGet.get(`${vatStoreKeyRoot}.storeKindIDTable`),
+  );
+  for (const kindName of Object.keys(builtInKinds)) {
+    predefinedKinds.set(builtInKinds[kindName], kindName);
+  }
+  const KIND_HANDLE_BASEREF = 1;
+  const BIGMAP_BASEREF = 6;
+  const BIGSET_BASEREF = 8;
+
+  // Kinds for which we have seen a kindHandle entry in the vatStore.  These are
+  // kinds which exist and have been used at some point, though their current
+  // ontological status is unknown.
+  const extantKinds = new Map(); // kindID -> tag
+
+  // Kinds for which we have seen a kindHandle value reference in the reference
+  // graph descended from baggage.  These are kinds that are potentially
+  // definable, though we can't at this stage determine if they've actually been
+  // given definitions.
+  const knownKinds = new Set();
+
+  // Kinds that have been seen in use somewhere.  These are kinds that had
+  // better have definitions or else something Bad will happen.
+  const usedKinds = new Set();
+
+  const kindKVs = sqlKVDump.iterate(`${dkindKeyRoot}.*`);
+  for (const kv of kindKVs) {
+    const descriptorSuffix = '.descriptor';
+    if (kv.key.endsWith(descriptorSuffix)) {
+      const dkindID = kv.key
+        .substring(0, kv.key.length - descriptorSuffix.length)
+        .substring(dkindKeyRoot.length + 1);
+      const descriptor = JSON.parse(kv.value);
+      extantKinds.set(+dkindID, descriptor.tag);
+    }
+  }
+
+  let pendingCollections = new Set();
+  const knownCollections = new Set();
+  let pendingObjects = new Set();
+  const knownObjects = new Set();
+
+  pendingCollections.add(1);
+  do {
+    const workingCollections = pendingCollections;
+    pendingCollections = new Set();
+    for (const coll of workingCollections) {
+      knownCollections.add(coll);
+    }
+    for (const coll of workingCollections) {
+      scanCollection(coll);
+    }
+    const workingObjects = pendingObjects;
+    pendingObjects = new Set();
+    for (const obj of workingObjects) {
+      knownObjects.add(obj);
+    }
+    for (const obj of workingObjects) {
+      scanObject(obj);
+    }
+  } while (pendingCollections.size > 0 || pendingObjects.size > 0);
+
+  // Kinds which have been used but whose kindID handles have not been seen
+  const usedButNotKnownKinds = new Set();
+  usedKinds.forEach(v => usedButNotKnownKinds.add(v));
+  knownKinds.forEach(v => usedButNotKnownKinds.delete(v));
+
+  // Kinds which exist but whose kind handles have not been seen
+  const extantButNotSeen = new Map();
+  extantKinds.forEach((k, v) => extantButNotSeen.set(v, k));
+  knownKinds.forEach(k => extantButNotSeen.delete(k));
+
+  // Kinds which exist but are not used
+  const extantButNotUsed = new Map();
+  extantKinds.forEach((k, v) => extantButNotUsed.set(v, k));
+  usedKinds.forEach(k => extantButNotUsed.delete(k));
+
+  if (verbose || usedButNotKnownKinds.size > 0) {
+    console.log('predefinedKinds', predefinedKinds);
+    console.log('extantKinds', extantKinds);
+    console.log('knownKinds', knownKinds);
+    console.log('usedKinds', usedKinds);
+    console.log('extantButNotSeen', extantButNotSeen);
+    console.log('extantButNotUsed', extantButNotUsed);
+    console.log('usedButNotKnownKinds', usedButNotKnownKinds);
+  }
+  if (usedButNotKnownKinds.size > 0) {
+    throw Error(
+      `kind IDs used without reachable kind ID handles: ${usedButNotKnownKinds.size}`,
+    );
+  }
+
+  function scanSlots(slots) {
+    for (const slot of slots) {
+      const refParts = extractDurableRefParts(slot);
+      if (refParts) {
+        const [baseRef, subRef] = refParts;
+        if (baseRef === BIGMAP_BASEREF || baseRef === BIGSET_BASEREF) {
+          if (!knownCollections.has(subRef)) {
+            pendingCollections.add(subRef);
+          }
+        } else if (baseRef === KIND_HANDLE_BASEREF) {
+          knownKinds.add(subRef);
+        } else {
+          if (!usedKinds.has(baseRef)) {
+            usedKinds.add(baseRef);
+          }
+          if (!knownObjects.has(slot)) {
+            pendingObjects.add(slot);
+          }
+        }
+      }
+    }
+  }
+
+  function scanCollection(collectionID) {
+    const collectionKeyRoot = `${vcKeyRoot}.${collectionID}.`;
+    const keyMatch = `${collectionKeyRoot}[a-z]*`;
+    const kvPairs = sqlKVDump.iterate(keyMatch);
+    for (const kv of kvPairs) {
+      const key = kv.key.substring(collectionKeyRoot.length);
+      const [_, slots] = decodeValueString(kv.value);
+      if (key[0] === 'r') {
+        const rkey = key.split(':')[1];
+        scanSlots([rkey]);
+      }
+      scanSlots(slots);
+    }
+  }
+
+  function scanObject(objectID) {
+    const rawObj = sqlKVGet.get(`${vomKeyRoot}.${objectID}`);
+    const props = JSON.parse(rawObj);
+    for (const prop of Object.keys(props)) {
+      scanSlots(props[prop].slots);
+    }
+  }
+}

packages/swing-store/src/swingStore.js

@@ -855,6 +855,10 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
     });
   }
 
+  function getDatabase() {
+    return db;
+  }
+
   const transcriptStorePublic = {
     initTranscript: transcriptStore.initTranscript,
     rolloverSpan: transcriptStore.rolloverSpan,
@@ -903,6 +907,7 @@ function makeSwingStore(dirPath, forceReset, options = {}) {
   const debug = {
     serialize,
     dump,
+    getDatabase,
   };
   const internal = {
     snapStore,