From 0170568d66748af76f0bd24a4acdaa34b9c79cca Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Thu, 9 Nov 2023 21:21:26 +0000 Subject: [PATCH 1/2] feat(swing-store): faster import of swing-store Disable journaling and full synchronous pragma modes during import --- .../test/transcript/test-state-sync-reload.js | 8 ++++- packages/swing-store/src/importer.js | 5 +++- packages/swing-store/src/swingStore.js | 30 +++++++++++++++++-- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/packages/SwingSet/test/transcript/test-state-sync-reload.js b/packages/SwingSet/test/transcript/test-state-sync-reload.js index ec6af96d47c..b3e1f64454d 100644 --- a/packages/SwingSet/test/transcript/test-state-sync-reload.js +++ b/packages/SwingSet/test/transcript/test-state-sync-reload.js @@ -6,6 +6,7 @@ import tmp from 'tmp'; import { kunser } from '@agoric/kmarshal'; import { initSwingStore, + openSwingStore, makeSwingStoreExporter, importSwingStore, } from '@agoric/swing-store'; @@ -36,7 +37,9 @@ test.before(async t => { test('state-sync reload', async t => { const [dbDir, cleanup] = await tmpDir('testdb'); + const [importDbDir, cleanupImport] = await tmpDir('importtestdb'); t.teardown(cleanup); + t.teardown(cleanupImport); const config = { snapshotInitial: 2, @@ -106,7 +109,10 @@ test('state-sync reload', async t => { getArtifact: name => artifacts.get(name), close: () => 0, }; - const ss2 = await importSwingStore(datasetExporter); + const ssi = await importSwingStore(datasetExporter, importDbDir); + await ssi.hostStorage.commit(); + await ssi.hostStorage.close(); + const ss2 = openSwingStore(importDbDir); const c2 = await makeSwingsetController( ss2.kernelStorage, {}, diff --git a/packages/swing-store/src/importer.js b/packages/swing-store/src/importer.js index a6cedb25e5d..05600f25812 100644 --- a/packages/swing-store/src/importer.js +++ b/packages/swing-store/src/importer.js @@ -27,7 +27,10 @@ export async function importSwingStore(exporter, dirPath = null, options = {}) { const { artifactMode = 'operational', ...makeSwingStoreOptions } = options; validateArtifactMode(artifactMode); - const store = makeSwingStore(dirPath, true, makeSwingStoreOptions); + const store = makeSwingStore(dirPath, true, { + unsafeFastMode: true, + ...makeSwingStoreOptions, + }); const { kernelStorage, internal } = store; // For every exportData entry, we add a DB record. 'kv' entries are diff --git a/packages/swing-store/src/swingStore.js b/packages/swing-store/src/swingStore.js index d0f9a68b8bb..1b488273f22 100644 --- a/packages/swing-store/src/swingStore.js +++ b/packages/swing-store/src/swingStore.js @@ -203,9 +203,29 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { // mode that defers merge work for a later attempt rather than block any // potential readers or writers. See https://sqlite.org/wal.html for details. + // However we also allow opening the DB with journaling off, which is unsafe + // and doesn't support rollback, but avoids any overhead for large + // transactions like for during an import. + + function setUnsafeFastMode(enabled) { + const journalMode = enabled ? 'off' : 'wal'; + const synchronousMode = enabled ? 'normal' : 'full'; + !db.inTransaction || Fail`must not be in a transaction`; + + db.unsafeMode(!!enabled); + // The WAL mode is persistent so it's not possible to switch to a different + // mode for an existing DB. + const actualMode = db.pragma(`journal_mode=${journalMode}`, { + simple: true, + }); + actualMode === journalMode || + filePath === ':memory:' || + Fail`Couldn't set swing-store DB to ${journalMode} mode (is ${actualMode})`; + db.pragma(`synchronous=${synchronousMode}`); + } + // PRAGMAs have to happen outside a transaction - db.exec(`PRAGMA journal_mode=WAL`); - db.exec(`PRAGMA synchronous=FULL`); + setUnsafeFastMode(options.unsafeFastMode); // We use IMMEDIATE because the kernel is supposed to be the sole writer of // the DB, and if some other process is holding a write lock, we want to find @@ -481,7 +501,11 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { } /** @type {import('./internal.js').SwingStoreInternal} */ - const internal = harden({ snapStore, transcriptStore, bundleStore }); + const internal = harden({ + snapStore, + transcriptStore, + bundleStore, + }); async function repairMetadata(exporter) { return doRepairMetadata(internal, exporter); From 6a833ebda2b2ff0e72040ca8186f93ae91567add Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Fri, 10 Nov 2023 00:06:59 +0000 Subject: [PATCH 2/2] feat(swing-store): prevent SwingSet usage of imported swing-store --- .../test/transcript/test-state-sync-reload.js | 1 + packages/swing-store/docs/data-export.md | 30 ++++++----------- packages/swing-store/src/importer.js | 23 +++++++------ packages/swing-store/test/test-bundles.js | 6 +++- .../swing-store/test/test-exportImport.js | 1 + packages/swing-store/test/test-import.js | 2 ++ .../swing-store/test/test-repair-metadata.js | 32 +++++++++++++------ 7 files changed, 56 insertions(+), 39 deletions(-) diff --git a/packages/SwingSet/test/transcript/test-state-sync-reload.js b/packages/SwingSet/test/transcript/test-state-sync-reload.js index b3e1f64454d..5792c2520b6 100644 --- a/packages/SwingSet/test/transcript/test-state-sync-reload.js +++ b/packages/SwingSet/test/transcript/test-state-sync-reload.js @@ -113,6 +113,7 @@ test('state-sync reload', async t => { await ssi.hostStorage.commit(); await ssi.hostStorage.close(); const ss2 = openSwingStore(importDbDir); + t.teardown(ss2.hostStorage.close); const c2 = await makeSwingsetController( ss2.kernelStorage, {}, diff --git a/packages/swing-store/docs/data-export.md b/packages/swing-store/docs/data-export.md index 4559f1ff386..72a70eade5e 100644 --- a/packages/swing-store/docs/data-export.md +++ b/packages/swing-store/docs/data-export.md @@ -152,21 +152,9 @@ for (const name of exporter.getArtifactNames()) { ## Import -On other end of the export process is an importer. This is a new host application, which wants to start from the contents of the export, rather than initializing a brand new (empty) kernel state. +On the other end of the export process is an importer. This is used to restore kernel state, so that a new host application can simply continue mostly as if it had been previously executing. The expectation is that the import and the execution are 2 independent events, and the execution doesn't need to be aware it was imported. -When starting a brand new instance, host applications would normally call `openSwingStore(dirPath)` to create a new (empty) SwingStore, then call SwingSet's `initializeSwingset(config, .., kernelStorage)` to let the kernel initialize the DB with a config-dependent starting state: - -```js -// this is done only the first time an instance is created: - -import { openSwingStore } from '@agoric/swing-store'; -import { initializeSwingset } from '@agoric/swingset-vat'; -const dirPath = './swing-store'; -const { hostStorage, kernelStorage } = openSwingStore(dirPath); -await initializeSwingset(config, argv, kernelStorage); -``` - -Once the initial state is created, each time the application is launched, it will build a controller around the existing state: +For reference, after the initial state is created, each time the application is launched, it builds a controller around the existing state: ```js import { openSwingStore } from '@agoric/swing-store'; @@ -177,7 +165,7 @@ const controller = await makeSwingsetController(kernelStorage); // ... now do things like controller.run(), etc ``` -When cloning an existing kernel, the initialization step is replaced with `importSwingStore`. The host application should feed the importer with the export data and artifacts, by passing an object that has the same API as the SwingStore's exporter: +When cloning an existing kernel, the host application first imports and commits the restored state using `importSwingStore`. The host application should feed the importer with the export data and artifacts, by passing an object that has the same API as the SwingStore's exporter: ```js import { importSwingStore } from '@agoric/swing-store'; @@ -188,11 +176,13 @@ const exporter = { getArtifact(name) { // return blob of artifact data }, }; const { hostStorage } = importSwingStore(exporter, dirPath); -hostStorage.commit(); -// now the swingstore is fully populated +// Update any hostStorage as needed +await hostStorage.commit(); +await hostStorage.close(); +// now the populated swingstore can be re-opened using `openSwingStore`` ``` -Once the new SwingStore is fully populated with the previously-exported data, the host application can use `makeSwingsetController()` to build a kernel that will start from the exported state. +Once the new SwingStore is fully populated with the previously-exported data, the host application can update any host specific state before committing and closing the SwingStore. `importSwingStore` returns only the host facet of the SwingStore instance, as it is not suitable for immediate execution. ## Optional / Historical Data @@ -223,14 +213,14 @@ Also note that when a vat is terminated, we delete all information about it, inc When importing, the `importSwingStore()` function's options bag takes a property named `artifactMode`, with the same meanings as for export. Importing with the `operational` mode will ignore any artifacts other than those needed for current operations, and will fail unless all such artifacts were available. Importing with `replay` will ignore spans from old incarnations, but will fail unless all spans from current incarnations are present. Importing with `archival` will fail unless all spans from all incarnations are present. There is no `debug` option during import. -`importSwingStore()` returns a swingstore, which means its options bag also contains the same options as `openSwingStore()`, including the `keepTranscripts` option. This defaults to `true`, but if it were overridden to `false`, then the new swingstore will delete transcript spans as soon as they are no longer needed for operational purposes (e.g. when `transcriptStore.rolloverSpan()` is called). +While `importSwingStore()`'s options bag accepts the same options as `openSwingStore()`, since it returns only the host facet of a SwingStore, some of these options might not be meaningful, such as `keepTranscripts`. So, to avoid pruning current-incarnation historical transcript spans when exporting from one swingstore to another, you must set (or avoid overriding) the following options along the way: * the original swingstore must not be opened with `{ keepTranscripts: false }`, otherwise the old spans will be pruned immediately * the export must use `makeSwingStoreExporter(dirpath, { artifactMode: 'replay'})`, otherwise the export will omit the old spans * the import must use `importSwingStore(exporter, dirPath, { artifactMode: 'replay'})`, otherwise the import will ignore the old spans - * the `importSwingStore` call (and all subsequent `openSwingStore` calls) must not use `keepTranscripts: false`, otherwise the new swingstore will prune historical spans as new ones are created (during `rolloverSpan`). + * subsequent `openSwingStore` calls must not use `keepTranscripts: false`, otherwise the new swingstore will prune historical spans as new ones are created (during `rolloverSpan`). ## Implementation Details diff --git a/packages/swing-store/src/importer.js b/packages/swing-store/src/importer.js index 05600f25812..6bbf8a48ad8 100644 --- a/packages/swing-store/src/importer.js +++ b/packages/swing-store/src/importer.js @@ -11,14 +11,16 @@ import { assertComplete } from './assertComplete.js'; */ /** - * Function used to create a new swingStore from an object implementing the + * Function used to populate a swingStore from an object implementing the * exporter API. The exporter API may be provided by a swingStore instance, or - * implemented by a host to restore data that was previously exported. + * implemented by a host to restore data that was previously exported. The + * returned swingStore is not suitable for execution, and thus only contains + * the host facet for committing the populated swingStore. * * @param {import('./exporter').SwingStoreExporter} exporter * @param {string | null} [dirPath] * @param {ImportSwingStoreOptions} [options] - * @returns {Promise} + * @returns {Promise>} */ export async function importSwingStore(exporter, dirPath = null, options = {}) { if (dirPath && typeof dirPath !== 'string') { @@ -27,11 +29,14 @@ export async function importSwingStore(exporter, dirPath = null, options = {}) { const { artifactMode = 'operational', ...makeSwingStoreOptions } = options; validateArtifactMode(artifactMode); - const store = makeSwingStore(dirPath, true, { - unsafeFastMode: true, - ...makeSwingStoreOptions, - }); - const { kernelStorage, internal } = store; + const { hostStorage, kernelStorage, internal, debug } = makeSwingStore( + dirPath, + true, + { + unsafeFastMode: true, + ...makeSwingStoreOptions, + }, + ); // For every exportData entry, we add a DB record. 'kv' entries are // the "kvStore shadow table", and are not associated with any @@ -124,5 +129,5 @@ export async function importSwingStore(exporter, dirPath = null, options = {}) { assertComplete(internal, checkMode); await exporter.close(); - return store; + return { hostStorage, debug }; } diff --git a/packages/swing-store/test/test-bundles.js b/packages/swing-store/test/test-bundles.js index 7c2003183e6..a8267102870 100644 --- a/packages/swing-store/test/test-bundles.js +++ b/packages/swing-store/test/test-bundles.js @@ -118,7 +118,11 @@ test('b0 import', async t => { }, close: async () => undefined, }; - const { kernelStorage } = await importSwingStore(exporter); + const ss = await importSwingStore(exporter); + t.teardown(ss.hostStorage.close); + await ss.hostStorage.commit(); + const serialized = ss.debug.serialize(); + const { kernelStorage } = initSwingStore(null, { serialized }); const { bundleStore } = kernelStorage; t.truthy(bundleStore.hasBundle(idA)); t.deepEqual(bundleStore.getBundle(idA), b0A); diff --git a/packages/swing-store/test/test-exportImport.js b/packages/swing-store/test/test-exportImport.js index 521356c2f39..bf01d2271c4 100644 --- a/packages/swing-store/test/test-exportImport.js +++ b/packages/swing-store/test/test-exportImport.js @@ -322,6 +322,7 @@ async function testExportImport( } t.is(failureMode, 'none'); const ssIn = await doImport(); + t.teardown(ssIn.hostStorage.close); await ssIn.hostStorage.commit(); let dumpsShouldMatch = true; if (runMode === 'operational') { diff --git a/packages/swing-store/test/test-import.js b/packages/swing-store/test/test-import.js index 620d9b35c01..9fdbc52410c 100644 --- a/packages/swing-store/test/test-import.js +++ b/packages/swing-store/test/test-import.js @@ -46,6 +46,7 @@ test('import empty', async t => { t.teardown(cleanup); const exporter = makeExporter(new Map(), new Map()); const ss = await importSwingStore(exporter, dbDir); + t.teardown(ss.hostStorage.close); await ss.hostStorage.commit(); const data = convert(ss.debug.dump()); t.deepEqual(data, { @@ -69,6 +70,7 @@ const importTest = test.macro(async (t, mode) => { // now import const ss = await importSwingStore(exporter, dbDir, { artifactMode }); + t.teardown(ss.hostStorage.close); await ss.hostStorage.commit(); const data = convert(ss.debug.dump()); diff --git a/packages/swing-store/test/test-repair-metadata.js b/packages/swing-store/test/test-repair-metadata.js index 6fee2af4566..9bd7b1996a6 100644 --- a/packages/swing-store/test/test-repair-metadata.js +++ b/packages/swing-store/test/test-repair-metadata.js @@ -4,7 +4,7 @@ import path from 'path'; import test from 'ava'; import sqlite3 from 'better-sqlite3'; -import { importSwingStore } from '../src/index.js'; +import { importSwingStore, openSwingStore } from '../src/index.js'; import { makeExporter, buildData } from './exports.js'; import { tmpDir } from './util.js'; @@ -19,8 +19,9 @@ test('repair metadata', async t => { // then manually deleting the historical metadata entries from the // DB const exporter = makeExporter(exportData, artifacts); - const ss = await importSwingStore(exporter, dbDir); - await ss.hostStorage.commit(); + const ssi = await importSwingStore(exporter, dbDir); + await ssi.hostStorage.commit(); + await ssi.hostStorage.close(); const filePath = path.join(dbDir, 'swingstore.sqlite'); const db = sqlite3(filePath); @@ -51,6 +52,8 @@ test('repair metadata', async t => { t.deepEqual(ss2, [7]); // now fix it + const ss = openSwingStore(dbDir); + t.teardown(ss.hostStorage.close); await ss.hostStorage.repairMetadata(exporter); await ss.hostStorage.commit(); @@ -62,6 +65,7 @@ test('repair metadata', async t => { // repair should be idempotent await ss.hostStorage.repairMetadata(exporter); + await ss.hostStorage.commit(); const ts4 = getTS.all('v1'); t.deepEqual(ts4, [0, 2, 5, 8]); // still there @@ -76,11 +80,15 @@ test('repair metadata ignores kvStore entries', async t => { const { exportData, artifacts } = buildData(); const exporter = makeExporter(exportData, artifacts); - const ss = await importSwingStore(exporter, dbDir); - await ss.hostStorage.commit(); + const ssi = await importSwingStore(exporter, dbDir); + await ssi.hostStorage.commit(); + await ssi.hostStorage.close(); // perform the repair with spurious kv entries exportData.set('kv.key2', 'value2'); + + const ss = openSwingStore(dbDir); + t.teardown(ss.hostStorage.close); await ss.hostStorage.repairMetadata(exporter); await ss.hostStorage.commit(); @@ -95,14 +103,17 @@ test('repair metadata rejects mismatched snapshot entries', async t => { const { exportData, artifacts } = buildData(); const exporter = makeExporter(exportData, artifacts); - const ss = await importSwingStore(exporter, dbDir); - await ss.hostStorage.commit(); + const ssi = await importSwingStore(exporter, dbDir); + await ssi.hostStorage.commit(); + await ssi.hostStorage.close(); // perform the repair with mismatched snapshot entry const old = JSON.parse(exportData.get('snapshot.v1.4')); const wrong = { ...old, hash: 'wrong' }; exportData.set('snapshot.v1.4', JSON.stringify(wrong)); + const ss = openSwingStore(dbDir); + t.teardown(ss.hostStorage.close); await t.throwsAsync(async () => ss.hostStorage.repairMetadata(exporter), { message: /repairSnapshotRecord metadata mismatch/, }); @@ -115,14 +126,17 @@ test('repair metadata rejects mismatched transcript span', async t => { const { exportData, artifacts } = buildData(); const exporter = makeExporter(exportData, artifacts); - const ss = await importSwingStore(exporter, dbDir); - await ss.hostStorage.commit(); + const ssi = await importSwingStore(exporter, dbDir); + await ssi.hostStorage.commit(); + await ssi.hostStorage.close(); // perform the repair with mismatched transcript span entry const old = JSON.parse(exportData.get('transcript.v1.0')); const wrong = { ...old, hash: 'wrong' }; exportData.set('transcript.v1.0', JSON.stringify(wrong)); + const ss = openSwingStore(dbDir); + t.teardown(ss.hostStorage.close); await t.throwsAsync(async () => ss.hostStorage.repairMetadata(exporter), { message: /repairTranscriptSpanRecord metadata mismatch/, });