From e08e8e0ccb01afb2cc3a6bfeb4a29476b6178f5d Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Wed, 12 Jul 2023 23:10:47 +0000 Subject: [PATCH] chore(cosmic-swingset): thread snapshot options through --- .../swingset/keeper/extension_snapshotter.go | 6 +- .../keeper/swing_store_exports_handler.go | 65 ++++++++++++++++--- .../swing_store_exports_handler_test.go | 18 ++--- packages/cosmic-swingset/src/chain-main.js | 47 +++++++++++--- .../cosmic-swingset/src/export-kernel-db.js | 19 ++++++ .../cosmic-swingset/src/import-kernel-db.js | 18 +++++ 6 files changed, 143 insertions(+), 30 deletions(-) diff --git a/golang/cosmos/x/swingset/keeper/extension_snapshotter.go b/golang/cosmos/x/swingset/keeper/extension_snapshotter.go index f7902ebb4da6..e6f5e28d6666 100644 --- a/golang/cosmos/x/swingset/keeper/extension_snapshotter.go +++ b/golang/cosmos/x/swingset/keeper/extension_snapshotter.go @@ -131,7 +131,10 @@ func (snapshotter *ExtensionSnapshotter) InitiateSnapshot(height int64) error { blockHeight := uint64(height) - return snapshotter.swingStoreExportsHandler.InitiateExport(blockHeight, snapshotter) + return snapshotter.swingStoreExportsHandler.InitiateExport(blockHeight, snapshotter, SwingStoreExportOptions{ + ExportMode: SwingStoreExportModeCurrent, + IncludeExportData: false, + }) } // OnExportStarted performs the actual cosmos state-sync app snapshot. @@ -313,5 +316,6 @@ func (snapshotter *ExtensionSnapshotter) RestoreExtension(blockHeight uint64, fo return snapshotter.swingStoreExportsHandler.RestoreExport( SwingStoreExportProvider{BlockHeight: blockHeight, GetExportData: getExportData, ReadArtifact: readArtifact}, + SwingStoreRestoreOptions{IncludeHistorical: false}, ) } diff --git a/golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go b/golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go index 963bbb0a81ba..a01bcf35ff36 100644 --- a/golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go +++ b/golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go @@ -123,9 +123,10 @@ const swingStoreExportActionType = "SWING_STORE_EXPORT" const initiateRequest = "initiate" type swingStoreInitiateExportAction struct { - Type string `json:"type"` // "SWING_STORE_EXPORT" - Request string `json:"request"` // "initiate" - BlockHeight uint64 `json:"blockHeight,omitempty"` // empty if no blockHeight requested (latest) + Type string `json:"type"` // "SWING_STORE_EXPORT" + Request string `json:"request"` // "initiate" + BlockHeight uint64 `json:"blockHeight,omitempty"` // empty if no blockHeight requested (latest) + Args [1]SwingStoreExportOptions `json:"args"` } // retrieveRequest is the request type for retrieving an initiated export @@ -150,10 +151,50 @@ type swingStoreDiscardExportAction struct { const restoreRequest = "restore" type swingStoreRestoreExportAction struct { - Type string `json:"type"` // "SWING_STORE_EXPORT" - Request string `json:"request"` // "restore" - BlockHeight uint64 `json:"blockHeight,omitempty"` // empty if deferring blockHeight to the manifest - Args [1]string `json:"args"` // args[1] is the directory in which the export to restore from is located + Type string `json:"type"` // "SWING_STORE_EXPORT" + Request string `json:"request"` // "restore" + BlockHeight uint64 `json:"blockHeight,omitempty"` // empty if deferring blockHeight to the manifest + Args [1]swingStoreImportOptions `json:"args"` +} + +// SwingStoreExportModeCurrent represents the minimal set of artifacts needed +// to operate a node. +const SwingStoreExportModeCurrent = "current" + +// SwingStoreExportModeArchival represents the set of all artifacts needed to +// not lose any historical state. +const SwingStoreExportModeArchival = "archival" + +// SwingStoreExportModeDebug represents the maximal set of artifacts available +// in the JS swing-store, including any kept around for debugging purposed only +// (like previous XS heap snapshots) +const SwingStoreExportModeDebug = "debug" + +// SwingStoreExportOptions are configurable options provided to the JS swing-store export +type SwingStoreExportOptions struct { + // The export mode can be "current", "archival" or "debug" (SwingStoreExportMode* const) + // See packages/cosmic-swingset/src/export-kernel-db.js initiateSwingStoreExport and + // packages/swing-store/src/swingStore.js makeSwingStoreExporter + ExportMode string `json:"exportMode,omitempty"` + // A flag indicating whether "export data" should be part of the swing-store export + // If false, the resulting SwingStoreExportProvider's GetExportData will + // return an empty list of "export data" entries. + IncludeExportData bool `json:"includeExportData,omitempty"` +} + +// SwingStoreRestoreOptions are configurable options provided to the JS swing-store import +type SwingStoreRestoreOptions struct { + // A flag indicating whether the swing-store import should attempt to load + // all historical artifacts available from the export provider + IncludeHistorical bool `json:"includeHistorical,omitempty"` +} + +type swingStoreImportOptions struct { + // ExportDir is the directory created by RestoreExport that JS swing-store + // should import from. + ExportDir string `json:"exportDir"` + // IncludeHistorical is a copy of SwingStoreRestoreOptions.IncludeHistorical + IncludeHistorical bool `json:"includeHistorical,omitempty"` } var disallowedArtifactNameChar = regexp.MustCompile(`[^-_.a-zA-Z0-9]`) @@ -393,7 +434,7 @@ func NewSwingStoreExportsHandler(logger log.Logger, blockingSend func(action vm. // from the goroutine that initiated the export. // // Must be called by the main goroutine -func (exportsHandler SwingStoreExportsHandler) InitiateExport(blockHeight uint64, eventHandler SwingStoreExportEventHandler) error { +func (exportsHandler SwingStoreExportsHandler) InitiateExport(blockHeight uint64, eventHandler SwingStoreExportEventHandler, exportOptions SwingStoreExportOptions) error { err := checkNotActive() if err != nil { return err @@ -447,6 +488,7 @@ func (exportsHandler SwingStoreExportsHandler) InitiateExport(blockHeight uint64 Type: swingStoreExportActionType, BlockHeight: blockHeight, Request: initiateRequest, + Args: [1]SwingStoreExportOptions{exportOptions}, } // blockingSend for SWING_STORE_EXPORT action is safe to call from a goroutine @@ -645,7 +687,7 @@ func (exportsHandler SwingStoreExportsHandler) retrieveExport(onExportRetrieved // RestoreExport restores the JS swing-store using previously exported data and artifacts. // // Must be called by the main goroutine -func (exportsHandler SwingStoreExportsHandler) RestoreExport(provider SwingStoreExportProvider) error { +func (exportsHandler SwingStoreExportsHandler) RestoreExport(provider SwingStoreExportProvider, restoreOptions SwingStoreRestoreOptions) error { err := checkNotActive() if err != nil { return err @@ -759,7 +801,10 @@ func (exportsHandler SwingStoreExportsHandler) RestoreExport(provider SwingStore Type: swingStoreExportActionType, BlockHeight: blockHeight, Request: restoreRequest, - Args: [1]string{exportDir}, + Args: [1]swingStoreImportOptions{{ + ExportDir: exportDir, + IncludeHistorical: restoreOptions.IncludeHistorical, + }}, } _, err = exportsHandler.blockingSend(action, true) diff --git a/golang/cosmos/x/swingset/keeper/swing_store_exports_handler_test.go b/golang/cosmos/x/swingset/keeper/swing_store_exports_handler_test.go index 32f12a6f6d73..7396c5011578 100644 --- a/golang/cosmos/x/swingset/keeper/swing_store_exports_handler_test.go +++ b/golang/cosmos/x/swingset/keeper/swing_store_exports_handler_test.go @@ -58,7 +58,7 @@ func TestSwingStoreSnapshotterInProgress(t *testing.T) { <-ch return nil } - err := exportsHandler.InitiateExport(123, exportEventHandler) + err := exportsHandler.InitiateExport(123, exportEventHandler, SwingStoreExportOptions{}) if err != nil { t.Fatal(err) } @@ -67,12 +67,12 @@ func TestSwingStoreSnapshotterInProgress(t *testing.T) { t.Fatal(err) } - err = exportsHandler.InitiateExport(456, newTestSwingStoreEventHandler()) + err = exportsHandler.InitiateExport(456, newTestSwingStoreEventHandler(), SwingStoreExportOptions{}) if err == nil { t.Error("wanted error for export operation in progress") } - err = exportsHandler.RestoreExport(SwingStoreExportProvider{BlockHeight: 456}) + err = exportsHandler.RestoreExport(SwingStoreExportProvider{BlockHeight: 456}, SwingStoreRestoreOptions{}) if err == nil { t.Error("wanted error for export operation in progress") } @@ -82,7 +82,7 @@ func TestSwingStoreSnapshotterInProgress(t *testing.T) { if err != nil { t.Fatal(err) } - err = exportsHandler.InitiateExport(456, exportEventHandler) + err = exportsHandler.InitiateExport(456, exportEventHandler, SwingStoreExportOptions{}) if err != nil { t.Fatal(err) } @@ -108,7 +108,7 @@ func TestSwingStoreSnapshotterSecondCommit(t *testing.T) { if err != nil { t.Fatal(err) } - err = exportsHandler.InitiateExport(123, exportEventHandler) + err = exportsHandler.InitiateExport(123, exportEventHandler, SwingStoreExportOptions{}) if err != nil { t.Fatal(err) } @@ -138,7 +138,7 @@ func TestSwingStoreSnapshotterInitiateFails(t *testing.T) { return "", nil } - err := exportsHandler.InitiateExport(123, exportEventHandler) + err := exportsHandler.InitiateExport(123, exportEventHandler, SwingStoreExportOptions{}) if err != nil { t.Fatal(err) } @@ -180,7 +180,7 @@ func TestSwingStoreSnapshotterRetrievalFails(t *testing.T) { return savedErr } - err := exportsHandler.InitiateExport(123, exportEventHandler) + err := exportsHandler.InitiateExport(123, exportEventHandler, SwingStoreExportOptions{}) if err != nil { t.Fatal(err) } @@ -216,7 +216,7 @@ func TestSwingStoreSnapshotterDiscard(t *testing.T) { activeOperation.exportRetrieved = true return nil } - err := exportsHandler.InitiateExport(123, exportEventHandler) + err := exportsHandler.InitiateExport(123, exportEventHandler, SwingStoreExportOptions{}) if err != nil { t.Fatal(err) } @@ -233,7 +233,7 @@ func TestSwingStoreSnapshotterDiscard(t *testing.T) { exportEventHandler.onExportStarted = func(height uint64, retrieveExport func() error) error { return nil } - err = exportsHandler.InitiateExport(456, exportEventHandler) + err = exportsHandler.InitiateExport(456, exportEventHandler, SwingStoreExportOptions{}) if err != nil { t.Fatal(err) } diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index 3fb102df4d3f..eddb0d10d998 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -39,8 +39,14 @@ import stringify from './helpers/json-stable-stringify.js'; import { launch } from './launch-chain.js'; import { getTelemetryProviders } from './kernel-stats.js'; import { makeProcessValue } from './helpers/process-value.js'; -import { spawnSwingStoreExport } from './export-kernel-db.js'; -import { performStateSyncImport } from './import-kernel-db.js'; +import { + spawnSwingStoreExport, + validateExporterOptions, +} from './export-kernel-db.js'; +import { + performStateSyncImport, + validateImporterOptions, +} from './import-kernel-db.js'; // eslint-disable-next-line no-unused-vars let whenHellFreezesOver = null; @@ -498,26 +504,44 @@ export default async function main(progname, args, { env, homedir, agcc }) { await null; switch (request) { case 'restore': { - const exportDir = requestArgs[0]; - if (typeof exportDir !== 'string') { - throw Fail`Invalid exportDir argument ${q(exportDir)}`; - } + const requestOptions = + typeof requestArgs[0] === 'string' + ? { exportDir: requestArgs[0] } + : requestArgs[0] || {}; + const options = { + ...requestOptions, + stateDir: stateDBDir, + blockHeight, + }; + validateImporterOptions(options); !stateSyncExport || Fail`Snapshot already in progress for ${stateSyncExport.blockHeight}`; !blockingSend || Fail`Cannot restore snapshot after init`; console.info( 'Restoring SwingSet state from snapshot at block height', blockHeight, + 'with options', + JSON.stringify(requestOptions), ); - return performStateSyncImport( - { exportDir, stateDir: stateDBDir, blockHeight }, - { fs: { ...fs, ...fsPromises }, pathResolve, log: null }, - ); + return performStateSyncImport(options, { + fs: { ...fs, ...fsPromises }, + pathResolve, + log: null, + }); } case 'initiate': { !stateSyncExport || Fail`Snapshot already in progress for ${stateSyncExport.blockHeight}`; + const requestOptions = requestArgs[0] || {}; + + validateExporterOptions({ + ...requestOptions, + stateDir: stateDBDir, + exportDir: '', + blockHeight, + }); + const exportData = /** @type {Required>} */ ({ blockHeight, @@ -560,9 +584,12 @@ export default async function main(progname, args, { env, homedir, agcc }) { console.info( 'Initiating SwingSet state snapshot at block height', blockHeight, + 'with options', + JSON.stringify(requestOptions), ); exportData.exporter = spawnSwingStoreExport( { + ...requestOptions, stateDir: stateDBDir, exportDir: exportData.exportDir, blockHeight, diff --git a/packages/cosmic-swingset/src/export-kernel-db.js b/packages/cosmic-swingset/src/export-kernel-db.js index a6a3493b57c9..4dc069e00077 100755 --- a/packages/cosmic-swingset/src/export-kernel-db.js +++ b/packages/cosmic-swingset/src/export-kernel-db.js @@ -77,6 +77,25 @@ const checkExportMode = mode => { * @property {boolean} [includeExportData] whether to include an artifact for the export data in the export */ +/** + * @param {object} options + * @returns {asserts options is StateSyncExporterOptions} + */ +export const validateExporterOptions = options => { + typeof options === 'object' || Fail`options is not an object`; + typeof options.stateDir === 'string' || + Fail`required stateDir option not a string`; + typeof options.exportDir === 'string' || + Fail`required exportDir option not a string`; + options.blockHeight == null || + typeof options.blockHeight === 'number' || + Fail`optional blockHeight option not a number`; + checkExportMode(options.exportMode); + options.includeExportData == null || + typeof options.includeExportData === 'boolean' || + Fail`optional includeExportData option not a boolean`; +}; + /** * @param {StateSyncExporterOptions} options * @param {object} powers diff --git a/packages/cosmic-swingset/src/import-kernel-db.js b/packages/cosmic-swingset/src/import-kernel-db.js index 113b5415f902..ff1d3a5f9a82 100755 --- a/packages/cosmic-swingset/src/import-kernel-db.js +++ b/packages/cosmic-swingset/src/import-kernel-db.js @@ -27,6 +27,24 @@ import { ExportManifestFileName } from './export-kernel-db.js'; * @property {boolean} [includeHistorical] whether to include historical artifacts in the export */ +/** + * @param {object} options + * @returns {asserts options is StateSyncImporterOptions} + */ +export const validateImporterOptions = options => { + typeof options === 'object' || Fail`options is not an object`; + typeof options.stateDir === 'string' || + Fail`required stateDir option not a string`; + typeof options.exportDir === 'string' || + Fail`required exportDir option not a string`; + options.blockHeight == null || + typeof options.blockHeight === 'number' || + Fail`optional blockHeight option not a number`; + options.includeHistorical == null || + typeof options.includeHistorical === 'boolean' || + Fail`optional includeHistorical option not a boolean`; +}; + /** * @param {StateSyncImporterOptions} options * @param {object} powers