From 7a4fa83ce8c013be46ff0d624445184236ada53a Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Thu, 17 Aug 2023 21:52:27 +0000 Subject: [PATCH 1/7] refactor(x/swingset): explicit read/write swing-store export directory --- .../keeper/swing_store_exports_handler.go | 106 +++++++++++------- 1 file changed, 64 insertions(+), 42 deletions(-) diff --git a/golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go b/golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go index 564db5de630..2cc7e83f588 100644 --- a/golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go +++ b/golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go @@ -638,19 +638,42 @@ func (exportsHandler SwingStoreExportsHandler) retrieveExport(onExportRetrieved defer os.RemoveAll(exportDir) - rawManifest, err := os.ReadFile(filepath.Join(exportDir, ExportManifestFilename)) + provider, err := OpenSwingStoreExportDirectory(exportDir) if err != nil { return err } - var manifest exportManifest - err = json.Unmarshal(rawManifest, &manifest) + if blockHeight != 0 && provider.BlockHeight != blockHeight { + return fmt.Errorf("export manifest blockHeight (%d) doesn't match (%d)", provider.BlockHeight, blockHeight) + } + + err = onExportRetrieved(provider) if err != nil { return err } - if blockHeight != 0 && manifest.BlockHeight != blockHeight { - return fmt.Errorf("export manifest blockHeight (%d) doesn't match (%d)", manifest.BlockHeight, blockHeight) + operationDetails.logger.Info("retrieved swing-store export", "exportDir", exportDir) + + return nil +} + +// OpenSwingStoreExportDirectory creates an export provider from a swing-store +// export saved on disk in the provided directory. It expects the export manifest +// to be present in that directory. The provider's function will read the +// export's data and artifacts from disk on demand. Each artifact is using a +// dedicated file, and the export data is read from a jsonl-like file, if any. +// The export manifest filename and overall export format is common with the JS +// swing-store import/export logic. +func OpenSwingStoreExportDirectory(exportDir string) (SwingStoreExportProvider, error) { + rawManifest, err := os.ReadFile(filepath.Join(exportDir, ExportManifestFilename)) + if err != nil { + return SwingStoreExportProvider{}, err + } + + var manifest exportManifest + err = json.Unmarshal(rawManifest, &manifest) + if err != nil { + return SwingStoreExportProvider{}, err } getExportDataReader := func() (agoric.KVEntryReader, error) { @@ -689,18 +712,7 @@ func (exportsHandler SwingStoreExportsHandler) retrieveExport(onExportRetrieved return artifact, err } - err = onExportRetrieved(SwingStoreExportProvider{BlockHeight: manifest.BlockHeight, GetExportDataReader: getExportDataReader, ReadNextArtifact: readNextArtifact}) - if err != nil { - return err - } - - // if nextArtifact != len(manifest.Artifacts) { - // return errors.New("not all export artifacts were retrieved") - // } - - operationDetails.logger.Info("retrieved swing-store export", "exportDir", exportDir) - - return nil + return SwingStoreExportProvider{BlockHeight: manifest.BlockHeight, GetExportDataReader: getExportDataReader, ReadNextArtifact: readNextArtifact}, nil } // RestoreExport restores the JS swing-store using previously exported data and artifacts. @@ -739,8 +751,41 @@ func (exportsHandler SwingStoreExportsHandler) RestoreExport(provider SwingStore } defer os.RemoveAll(exportDir) - manifest := exportManifest{ + err = WriteSwingStoreExportToDirectory(provider, exportDir) + if err != nil { + return err + } + + action := &swingStoreRestoreExportAction{ + Type: swingStoreExportActionType, BlockHeight: blockHeight, + Request: restoreRequest, + Args: [1]swingStoreImportOptions{{ + ExportDir: exportDir, + ArtifactMode: restoreOptions.ArtifactMode, + ExportDataMode: restoreOptions.ExportDataMode, + }}, + } + + _, err = exportsHandler.blockingSend(action, true) + if err != nil { + return err + } + + exportsHandler.logger.Info("restored swing-store export", "exportDir", exportDir, "height", blockHeight) + + return nil +} + +// WriteSwingStoreExportToDirectory consumes a provider and saves a swing-store +// export to disk in the provided directory. It creates files for each artifact +// deriving a filename from the artifact name, and stores any "export data" in +// a jsonl-like file, before saving the export manifest linking these together. +// The export manifest filename and overall export format is common with the JS +// swing-store import/export logic. +func WriteSwingStoreExportToDirectory(provider SwingStoreExportProvider, exportDir string) error { + manifest := exportManifest{ + BlockHeight: provider.BlockHeight, } exportDataReader, err := provider.GetExportDataReader() @@ -808,28 +853,5 @@ func (exportsHandler SwingStoreExportsHandler) RestoreExport(provider SwingStore if err != nil { return err } - err = writeExportFile(ExportManifestFilename, manifestBytes) - if err != nil { - return err - } - - action := &swingStoreRestoreExportAction{ - Type: swingStoreExportActionType, - BlockHeight: blockHeight, - Request: restoreRequest, - Args: [1]swingStoreImportOptions{{ - ExportDir: exportDir, - ArtifactMode: restoreOptions.ArtifactMode, - ExportDataMode: restoreOptions.ExportDataMode, - }}, - } - - _, err = exportsHandler.blockingSend(action, true) - if err != nil { - return err - } - - exportsHandler.logger.Info("restored swing-store export", "exportDir", exportDir, "height", blockHeight) - - return nil + return writeExportFile(ExportManifestFilename, manifestBytes) } From 33c4a517f079c4ad17c30f9d1d13f181b06f112f Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Fri, 21 Jul 2023 23:43:15 +0000 Subject: [PATCH 2/7] fix(cosmic-swingset): log level for swing-store export --- packages/cosmic-swingset/src/chain-main.js | 2 +- packages/cosmic-swingset/src/export-kernel-db.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index 2c0a1eac1e0..915564f2ca4 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -584,7 +584,7 @@ export default async function main(progname, args, { env, homedir, agcc }) { ); }); - console.info( + console.warn( 'Initiating SwingSet state snapshot at block height', blockHeight, 'with options', diff --git a/packages/cosmic-swingset/src/export-kernel-db.js b/packages/cosmic-swingset/src/export-kernel-db.js index 6dca1f2fe17..2da9b2e3d94 100755 --- a/packages/cosmic-swingset/src/export-kernel-db.js +++ b/packages/cosmic-swingset/src/export-kernel-db.js @@ -351,7 +351,7 @@ export const main = async ( { fs, pathResolve, - log: verbose ? console.log : null, + log: verbose ? console.warn : null, }, ); From 592948dc2fada0a9d1f56581ccae42040bfe4a53 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Fri, 21 Jul 2023 23:43:15 +0000 Subject: [PATCH 3/7] feat(cosmos): spawn JS on export command Do not assume deamonization --- golang/cosmos/cmd/agd/main.go | 7 +++++-- golang/cosmos/daemon/cmd/root.go | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/golang/cosmos/cmd/agd/main.go b/golang/cosmos/cmd/agd/main.go index 21f3a0db1ce..174ada874f3 100644 --- a/golang/cosmos/cmd/agd/main.go +++ b/golang/cosmos/cmd/agd/main.go @@ -13,7 +13,7 @@ import ( func main() { // We need to delegate to our default app for running the actual chain. - daemoncmd.OnStartHook = func(logger log.Logger) { + launchVM := func(logger log.Logger) { args := []string{"ag-chain-cosmos", "--home", gaia.DefaultNodeHome} args = append(args, os.Args[1:]...) @@ -22,12 +22,15 @@ func main() { panic(lookErr) } - logger.Info("Start chain delegating to JS executable", "binary", binary, "args", args) + logger.Info("agd delegating to JS executable", "binary", binary, "args", args) execErr := syscall.Exec(binary, args, os.Environ()) if execErr != nil { panic(execErr) } } + daemoncmd.OnStartHook = launchVM + daemoncmd.OnExportHook = launchVM + daemon.RunWithController(nil) } diff --git a/golang/cosmos/daemon/cmd/root.go b/golang/cosmos/daemon/cmd/root.go index 8426cd6cacb..339b111ee3b 100644 --- a/golang/cosmos/daemon/cmd/root.go +++ b/golang/cosmos/daemon/cmd/root.go @@ -40,7 +40,8 @@ import ( type Sender func(needReply bool, str string) (string, error) var AppName = "agd" -var OnStartHook func(log.Logger) +var OnStartHook func(logger log.Logger) +var OnExportHook func(logger log.Logger) // NewRootCmd creates a new root command for simd. It is called once in the // main function. @@ -272,6 +273,9 @@ func (ac appCreator) appExport( jailAllowedAddrs []string, appOpts servertypes.AppOptions, ) (servertypes.ExportedApp, error) { + if OnExportHook != nil { + OnExportHook(logger) + } homePath, ok := appOpts.Get(flags.FlagHome).(string) if !ok || homePath == "" { From 3be2986059c9f007d34518deef68e31956e9b45e Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Fri, 18 Aug 2023 20:06:10 +0000 Subject: [PATCH 4/7] feat(cosmos)!: add required export-dir export cmd option --- golang/cosmos/daemon/cmd/root.go | 64 ++++++++++++++++++- packages/cosmic-swingset/test/scenario2.js | 5 +- .../agoric-upgrade-11/actions.sh | 1 - .../agoric-upgrade-11/env_setup.sh | 11 ++-- 4 files changed, 73 insertions(+), 8 deletions(-) diff --git a/golang/cosmos/daemon/cmd/root.go b/golang/cosmos/daemon/cmd/root.go index 339b111ee3b..9fe90a0b018 100644 --- a/golang/cosmos/daemon/cmd/root.go +++ b/golang/cosmos/daemon/cmd/root.go @@ -134,6 +134,14 @@ func initRootCmd(sender Sender, rootCmd *cobra.Command, encodingConfig params.En } server.AddCommands(rootCmd, gaia.DefaultNodeHome, ac.newApp, ac.appExport, addModuleInitFlags) + hasVMController := sender != nil + for _, command := range rootCmd.Commands() { + if command.Name() == "export" { + extendCosmosExportCommand(command, hasVMController) + break + } + } + // add keybase, auxiliary RPC, query, and tx child commands rootCmd.AddCommand( rpc.StatusCommand(), @@ -233,7 +241,9 @@ func (ac appCreator) newApp( panic(err) } - snapshotDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", "snapshots") + homePath := cast.ToString(appOpts.Get(flags.FlagHome)) + + snapshotDir := filepath.Join(homePath, "data", "snapshots") snapshotDB, err := sdk.NewLevelDB("metadata", snapshotDir) if err != nil { panic(err) @@ -246,7 +256,7 @@ func (ac appCreator) newApp( return gaia.NewAgoricApp( ac.sender, logger, db, traceStore, true, skipUpgradeHeights, - cast.ToString(appOpts.Get(flags.FlagHome)), + homePath, cast.ToUint(appOpts.Get(server.FlagInvCheckPeriod)), ac.encCfg, appOpts, @@ -264,6 +274,56 @@ func (ac appCreator) newApp( ) } +const ( + // FlagExportDir is the command-line flag for the "export" command specifying + // where the output of the export should be placed. + FlagExportDir = "export-dir" + // ExportedGenesisFileName is the file name used to save the genesis in the export-dir + ExportedGenesisFileName = "genesis.json" +) + +// extendCosmosExportCommand monkey-patches the "export" command added by +// cosmos-sdk to add a required "export-dir" command-line flag, and create the +// genesis export in the specified directory. +func extendCosmosExportCommand(cmd *cobra.Command, hasVMController bool) { + cmd.Flags().String(FlagExportDir, "", "The directory where to create the genesis export") + err := cmd.MarkFlagRequired(FlagExportDir) + if err != nil { + panic(err) + } + + originalRunE := cmd.RunE + + extendedRunE := func(cmd *cobra.Command, args []string) error { + exportDir, _ := cmd.Flags().GetString(FlagExportDir) + err := os.MkdirAll(exportDir, os.ModePerm) + if err != nil { + return err + } + + genesisPath := filepath.Join(exportDir, ExportedGenesisFileName) + + // This will fail is a genesis.json already exists in the export-dir + genesisFile, err := os.OpenFile(genesisPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, os.ModePerm) + if err != nil { + return err + } + defer genesisFile.Close() + + cmd.SetOut(genesisFile) + + return originalRunE(cmd, args) + } + + // Only modify the command handler when we have a VM controller to handle + // the full export logic. Otherwise, appExport will just exec the VM program + // (OnExportHook), which will result in re-entering this flow with the VM + // controller set. + if hasVMController { + cmd.RunE = extendedRunE + } +} + func (ac appCreator) appExport( logger log.Logger, db dbm.DB, diff --git a/packages/cosmic-swingset/test/scenario2.js b/packages/cosmic-swingset/test/scenario2.js index 006a5670c2e..a489195be51 100644 --- a/packages/cosmic-swingset/test/scenario2.js +++ b/packages/cosmic-swingset/test/scenario2.js @@ -81,7 +81,10 @@ export const makeScenario2 = ({ pspawnMake, pspawnAgd, log }) => { return runMake(['scenario2-run-rosetta-ci'], { stdio: onlyStderr }); }, export: () => - pspawnAgd(['export', '--home=t1/n0'], { stdio: onlyStderr }).exit, + pspawnAgd( + ['export', '--home=t1/n0', '--export-dir=t1/n0/genesis-export'], + { stdio: onlyStderr }, + ).exit, }); }; diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh index a535679ff83..7a76e3ec011 100644 --- a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh +++ b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh @@ -18,7 +18,6 @@ mv -n $HOME/.agoric/data/agoric/swing-store-historical-artifacts/* $EXPORT_DIR | mv $EXPORT_DIR/export-manifest.json $EXPORT_DIR/export-manifest-original.json cat $EXPORT_DIR/export-manifest-original.json | jq -r ".artifacts = .artifacts + [${HISTORICAL_ARTIFACTS%%,}] | del(.artifactMode)" > $EXPORT_DIR/export-manifest.json restore_swing_store_snapshot $EXPORT_DIR || fail "Couldn't restore swing-store snapshot" -rmdir $HOME/.agoric/data/agoric/swing-store-historical-artifacts rm -rf $EXPORT_DIR startAgd diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh index 0faaf3a66b5..4ab804dcd9e 100644 --- a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh +++ b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh @@ -86,14 +86,16 @@ pushPriceOnce () { } export_genesis() { - HEIGHT_ARG= + GENESIS_EXPORT_DIR="$1" + shift + GENESIS_HEIGHT_ARG= if [ -n "$1" ]; then - HEIGHT_ARG="--height $1" + GENESIS_HEIGHT_ARG="--height $1" shift fi - agd export $HEIGHT_ARG "$@" + agd export --export-dir "$GENESIS_EXPORT_DIR" $GENESIS_HEIGHT_ARG "$@" } make_swing_store_snapshot() {( set -euo pipefail @@ -108,7 +110,8 @@ make_swing_store_snapshot() {( set -euo pipefail EXPORT_MANIFEST="$(cat $EXPORT_MANIFEST_FILE)" mv "$EXPORT_DATA_FILE" "$EXPORT_DATA_UNTRUSTED_FILE" - export_genesis $EXPORT_HEIGHT | jq -cr '.app_state.swingset.swing_store_export_data[] | [.key,.value]' > "$EXPORT_DATA_FILE" + export_genesis "$EXPORT_DIR/genesis-export" $EXPORT_HEIGHT + cat $EXPORT_DIR/genesis-export/genesis.json | jq -cr '.app_state.swingset.swing_store_export_data[] | [.key,.value]' > "$EXPORT_DATA_FILE" jq -n "$EXPORT_MANIFEST | .untrustedData=\"$(basename -- "$EXPORT_DATA_UNTRUSTED_FILE")\"" > "$EXPORT_MANIFEST_FILE" From 598abf73ac555bd7284c8a91a03c205dc62d9b0c Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Fri, 21 Jul 2023 23:43:15 +0000 Subject: [PATCH 5/7] feat(x/swingset): export swing store in genesis --- golang/cosmos/app/app.go | 11 +++- golang/cosmos/daemon/cmd/root.go | 20 ++++++- golang/cosmos/x/swingset/genesis.go | 56 ++++++++++++++++--- golang/cosmos/x/swingset/module.go | 29 +++++++--- .../agoric-upgrade-11/actions.sh | 3 +- .../agoric-upgrade-11/env_setup.sh | 17 +++--- 6 files changed, 109 insertions(+), 27 deletions(-) diff --git a/golang/cosmos/app/app.go b/golang/cosmos/app/app.go index 82db55ed54f..f5932d90c95 100644 --- a/golang/cosmos/app/app.go +++ b/golang/cosmos/app/app.go @@ -126,6 +126,14 @@ import ( const appName = "agoric" +// FlagSwingStoreExportDir defines the config flag used to specify where a +// genesis swing-store export is expected. For start from genesis, the default +// value is config/swing-store in the home directory. For genesis export, the +// value is always a "swing-store" directory sibling to the exported +// genesis.json file. +// TODO: document this flag in config, likely alongside the genesis path +const FlagSwingStoreExportDir = "swing-store-export-dir" + var ( // DefaultNodeHome default home directories for the application daemon DefaultNodeHome string @@ -588,6 +596,7 @@ func NewAgoricApp( app.EvidenceKeeper = *evidenceKeeper skipGenesisInvariants := cast.ToBool(appOpts.Get(crisis.FlagSkipGenesisInvariants)) + swingStoreExportDir := cast.ToString(appOpts.Get(FlagSwingStoreExportDir)) // NOTE: Any module instantiated in the module manager that is later modified // must be passed by reference here. @@ -617,7 +626,7 @@ func NewAgoricApp( transferModule, icaModule, vstorage.NewAppModule(app.VstorageKeeper), - swingset.NewAppModule(app.SwingSetKeeper, setBootstrapNeeded, app.ensureControllerInited), + swingset.NewAppModule(app.SwingSetKeeper, &app.SwingStoreExportsHandler, setBootstrapNeeded, app.ensureControllerInited, swingStoreExportDir), vibcModule, vbankModule, lienModule, diff --git a/golang/cosmos/daemon/cmd/root.go b/golang/cosmos/daemon/cmd/root.go index 9fe90a0b018..95c6c1f2433 100644 --- a/golang/cosmos/daemon/cmd/root.go +++ b/golang/cosmos/daemon/cmd/root.go @@ -276,10 +276,15 @@ func (ac appCreator) newApp( const ( // FlagExportDir is the command-line flag for the "export" command specifying - // where the output of the export should be placed. + // where the output of the export should be placed. It contains both the + // items names below: the genesis file, and a directory containing the + // exported swing-store artifacts FlagExportDir = "export-dir" // ExportedGenesisFileName is the file name used to save the genesis in the export-dir ExportedGenesisFileName = "genesis.json" + // ExportedSwingStoreDirectoryName is the directory name used to save the swing-store + // export (artifacts only) in the export-dir + ExportedSwingStoreDirectoryName = "swing-store" ) // extendCosmosExportCommand monkey-patches the "export" command added by @@ -295,6 +300,8 @@ func extendCosmosExportCommand(cmd *cobra.Command, hasVMController bool) { originalRunE := cmd.RunE extendedRunE := func(cmd *cobra.Command, args []string) error { + serverCtx := server.GetServerContextFromCmd(cmd) + exportDir, _ := cmd.Flags().GetString(FlagExportDir) err := os.MkdirAll(exportDir, os.ModePerm) if err != nil { @@ -302,6 +309,17 @@ func extendCosmosExportCommand(cmd *cobra.Command, hasVMController bool) { } genesisPath := filepath.Join(exportDir, ExportedGenesisFileName) + swingStoreExportPath := filepath.Join(exportDir, ExportedSwingStoreDirectoryName) + + err = os.MkdirAll(swingStoreExportPath, os.ModePerm) + if err != nil { + return err + } + // We unconditionally set FlagSwingStoreExportDir as for export, it makes + // little sense for users to control this location separately, and we don't + // want to override any swing-store artifacts that may be associated to the + // current genesis. + serverCtx.Viper.Set(gaia.FlagSwingStoreExportDir, swingStoreExportPath) // This will fail is a genesis.json already exists in the export-dir genesisFile, err := os.OpenFile(genesisPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, os.ModePerm) diff --git a/golang/cosmos/x/swingset/genesis.go b/golang/cosmos/x/swingset/genesis.go index 5b27c603b70..1334da697bd 100644 --- a/golang/cosmos/x/swingset/genesis.go +++ b/golang/cosmos/x/swingset/genesis.go @@ -4,6 +4,8 @@ import ( // "os" "fmt" + agoric "github.com/Agoric/agoric-sdk/golang/cosmos/types" + "github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset/keeper" "github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset/types" sdk "github.com/cosmos/cosmos-sdk/types" ) @@ -42,19 +44,13 @@ func InitGenesis(ctx sdk.Context, keeper Keeper, data *types.GenesisState) bool return true } -func ExportGenesis(ctx sdk.Context, k Keeper) *types.GenesisState { +func ExportGenesis(ctx sdk.Context, k Keeper, swingStoreExportsHandler *SwingStoreExportsHandler, swingStoreExportDir string) *types.GenesisState { gs := &types.GenesisState{ Params: k.GetParams(ctx), State: k.GetState(ctx), SwingStoreExportData: []*types.SwingStoreExportDataEntry{}, } - // Only export the swing-store shadow copy for now - // TODO: - // - perform state-sync export with check blockHeight (figure out how to - // handle export of historical height), - // - include swing-store artifacts in genesis state - // See https://github.com/Agoric/agoric-sdk/issues/6527 exportDataIterator := k.GetSwingStore(ctx).Iterator(nil, nil) defer exportDataIterator.Close() for ; exportDataIterator.Valid(); exportDataIterator.Next() { @@ -64,5 +60,51 @@ func ExportGenesis(ctx sdk.Context, k Keeper) *types.GenesisState { } gs.SwingStoreExportData = append(gs.SwingStoreExportData, &entry) } + + snapshotHeight := uint64(ctx.BlockHeight()) + + err := swingStoreExportsHandler.InitiateExport( + // The export will fail if the export of a historical height was requested + snapshotHeight, + swingStoreGenesisEventHandler{exportDir: swingStoreExportDir, snapshotHeight: snapshotHeight}, + // The export will fail if the swing-store does not contain all replay artifacts + keeper.SwingStoreExportOptions{ + ArtifactMode: keeper.SwingStoreArtifactModeReplay, + ExportDataMode: keeper.SwingStoreExportDataModeSkip, + }, + ) + if err != nil { + panic(err) + } + + err = keeper.WaitUntilSwingStoreExportDone() + if err != nil { + panic(err) + } + return gs } + +type swingStoreGenesisEventHandler struct { + exportDir string + snapshotHeight uint64 +} + +func (eventHandler swingStoreGenesisEventHandler) OnExportStarted(height uint64, retrieveSwingStoreExport func() error) error { + return retrieveSwingStoreExport() +} + +func (eventHandler swingStoreGenesisEventHandler) OnExportRetrieved(provider keeper.SwingStoreExportProvider) error { + if eventHandler.snapshotHeight != provider.BlockHeight { + return fmt.Errorf("snapshot block height (%d) doesn't match requested height (%d)", provider.BlockHeight, eventHandler.snapshotHeight) + } + + artifactsProvider := keeper.SwingStoreExportProvider{ + GetExportDataReader: func() (agoric.KVEntryReader, error) { + return nil, nil + }, + ReadNextArtifact: provider.ReadNextArtifact, + } + + return keeper.WriteSwingStoreExportToDirectory(artifactsProvider, eventHandler.exportDir) +} diff --git a/golang/cosmos/x/swingset/module.go b/golang/cosmos/x/swingset/module.go index a5f180beba4..42b207dab38 100644 --- a/golang/cosmos/x/swingset/module.go +++ b/golang/cosmos/x/swingset/module.go @@ -80,18 +80,22 @@ func (AppModuleBasic) GetTxCmd() *cobra.Command { type AppModule struct { AppModuleBasic - keeper Keeper - setBootstrapNeeded func() - ensureControllerInited func(sdk.Context) + keeper Keeper + swingStoreExportsHandler *SwingStoreExportsHandler + setBootstrapNeeded func() + ensureControllerInited func(sdk.Context) + swingStoreExportDir string } // NewAppModule creates a new AppModule Object -func NewAppModule(k Keeper, setBootstrapNeeded func(), ensureControllerInited func(sdk.Context)) AppModule { +func NewAppModule(k Keeper, swingStoreExportsHandler *SwingStoreExportsHandler, setBootstrapNeeded func(), ensureControllerInited func(sdk.Context), swingStoreExportDir string) AppModule { am := AppModule{ - AppModuleBasic: AppModuleBasic{}, - keeper: k, - setBootstrapNeeded: setBootstrapNeeded, - ensureControllerInited: ensureControllerInited, + AppModuleBasic: AppModuleBasic{}, + keeper: k, + swingStoreExportsHandler: swingStoreExportsHandler, + setBootstrapNeeded: setBootstrapNeeded, + ensureControllerInited: ensureControllerInited, + swingStoreExportDir: swingStoreExportDir, } return am } @@ -150,6 +154,12 @@ func (am AppModule) EndBlock(ctx sdk.Context, req abci.RequestEndBlock) []abci.V return []abci.ValidatorUpdate{} } +func (am AppModule) checkSwingStoreExportSetup() { + if am.swingStoreExportDir == "" { + panic(fmt.Errorf("SwingStore export dir not set")) + } +} + func (am AppModule) InitGenesis(ctx sdk.Context, cdc codec.JSONCodec, data json.RawMessage) []abci.ValidatorUpdate { var genesisState types.GenesisState cdc.MustUnmarshalJSON(data, &genesisState) @@ -161,6 +171,7 @@ func (am AppModule) InitGenesis(ctx sdk.Context, cdc codec.JSONCodec, data json. } func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONCodec) json.RawMessage { - gs := ExportGenesis(ctx, am.keeper) + am.checkSwingStoreExportSetup() + gs := ExportGenesis(ctx, am.keeper, am.swingStoreExportsHandler, am.swingStoreExportDir) return cdc.MustMarshalJSON(gs) } diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh index 7a76e3ec011..24e968fa674 100644 --- a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh +++ b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh @@ -11,8 +11,7 @@ upgrade11=./upgrade-test-scripts/agoric-upgrade-11 # hacky restore of pruned artifacts killAgd EXPORT_DIR=$(mktemp -t -d swing-store-export-upgrade-11-XXX) -make_swing_store_snapshot $EXPORT_DIR --artifact-mode debug || fail "Couldn't make swing-store snapshot" -test_val "$(compare_swing_store_export_data $EXPORT_DIR)" "match" "swing-store export data" +WITHOUT_GENESIS_EXPORT=1 make_swing_store_snapshot $EXPORT_DIR --artifact-mode debug || fail "Couldn't make swing-store snapshot" HISTORICAL_ARTIFACTS="$(cd $HOME/.agoric/data/agoric/swing-store-historical-artifacts/; for i in *; do echo -n "[\"$i\",\"$i\"],"; done)" mv -n $HOME/.agoric/data/agoric/swing-store-historical-artifacts/* $EXPORT_DIR || fail "some historical artifacts not pruned" mv $EXPORT_DIR/export-manifest.json $EXPORT_DIR/export-manifest-original.json diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh index 4ab804dcd9e..3a9537719f1 100644 --- a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh +++ b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh @@ -104,16 +104,19 @@ make_swing_store_snapshot() {( set -euo pipefail /usr/src/agoric-sdk/packages/cosmic-swingset/src/export-kernel-db.js --home "$HOME/.agoric" --export-dir "$EXPORT_DIR" --verbose --artifact-mode replay --export-data-mode all "$@" EXPORT_MANIFEST_FILE="$EXPORT_DIR/export-manifest.json" - EXPORT_DATA_FILE="$EXPORT_DIR/$(cat "$EXPORT_MANIFEST_FILE" | jq -r .data)" - EXPORT_DATA_UNTRUSTED_FILE="${EXPORT_DATA_FILE%.*}-untrusted.jsonl" EXPORT_HEIGHT=$(cat "$EXPORT_MANIFEST_FILE" | jq -r .blockHeight) - EXPORT_MANIFEST="$(cat $EXPORT_MANIFEST_FILE)" + + [ "x${WITHOUT_GENESIS_EXPORT:-0}" = "x1" ] || { + EXPORT_DATA_FILE="$EXPORT_DIR/$(cat "$EXPORT_MANIFEST_FILE" | jq -r .data)" + EXPORT_DATA_UNTRUSTED_FILE="${EXPORT_DATA_FILE%.*}-untrusted.jsonl" + EXPORT_MANIFEST="$(cat $EXPORT_MANIFEST_FILE)" - mv "$EXPORT_DATA_FILE" "$EXPORT_DATA_UNTRUSTED_FILE" - export_genesis "$EXPORT_DIR/genesis-export" $EXPORT_HEIGHT - cat $EXPORT_DIR/genesis-export/genesis.json | jq -cr '.app_state.swingset.swing_store_export_data[] | [.key,.value]' > "$EXPORT_DATA_FILE" + mv "$EXPORT_DATA_FILE" "$EXPORT_DATA_UNTRUSTED_FILE" + export_genesis "$EXPORT_DIR/genesis-export" $EXPORT_HEIGHT + cat $EXPORT_DIR/genesis-export/genesis.json | jq -cr '.app_state.swingset.swing_store_export_data[] | [.key,.value]' > "$EXPORT_DATA_FILE" - jq -n "$EXPORT_MANIFEST | .untrustedData=\"$(basename -- "$EXPORT_DATA_UNTRUSTED_FILE")\"" > "$EXPORT_MANIFEST_FILE" + jq -n "$EXPORT_MANIFEST | .untrustedData=\"$(basename -- "$EXPORT_DATA_UNTRUSTED_FILE")\"" > "$EXPORT_MANIFEST_FILE" + } echo "Successful swing-store export for block $EXPORT_HEIGHT" )} From 55ba7f6c3d6a958d3eb5c6c8b191b649da05809f Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Sat, 22 Jul 2023 00:16:08 +0000 Subject: [PATCH 6/7] feat(x/swingset): import swing store from genesis state --- golang/cosmos/daemon/cmd/root.go | 8 +++++ golang/cosmos/x/swingset/genesis.go | 47 ++++++++++++++++++++++++----- golang/cosmos/x/swingset/module.go | 3 +- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/golang/cosmos/daemon/cmd/root.go b/golang/cosmos/daemon/cmd/root.go index 95c6c1f2433..35a59ceb101 100644 --- a/golang/cosmos/daemon/cmd/root.go +++ b/golang/cosmos/daemon/cmd/root.go @@ -28,6 +28,7 @@ import ( genutilcli "github.com/cosmos/cosmos-sdk/x/genutil/client/cli" "github.com/spf13/cast" "github.com/spf13/cobra" + "github.com/spf13/viper" tmcli "github.com/tendermint/tendermint/libs/cli" "github.com/tendermint/tendermint/libs/log" dbm "github.com/tendermint/tm-db" @@ -243,6 +244,13 @@ func (ac appCreator) newApp( homePath := cast.ToString(appOpts.Get(flags.FlagHome)) + // Set a default value for FlagSwingStoreExportDir based on the homePath + // in case we need to InitGenesis with swing-store data + viper, ok := appOpts.(*viper.Viper) + if ok && cast.ToString(appOpts.Get(gaia.FlagSwingStoreExportDir)) == "" { + viper.Set(gaia.FlagSwingStoreExportDir, filepath.Join(homePath, "config", ExportedSwingStoreDirectoryName)) + } + snapshotDir := filepath.Join(homePath, "data", "snapshots") snapshotDB, err := sdk.NewLevelDB("metadata", snapshotDir) if err != nil { diff --git a/golang/cosmos/x/swingset/genesis.go b/golang/cosmos/x/swingset/genesis.go index 1334da697bd..a738e1497c7 100644 --- a/golang/cosmos/x/swingset/genesis.go +++ b/golang/cosmos/x/swingset/genesis.go @@ -30,18 +30,49 @@ func DefaultGenesisState() *types.GenesisState { // InitGenesis initializes the (Cosmos-side) SwingSet state from the GenesisState. // Returns whether the app should send a bootstrap action to the controller. -func InitGenesis(ctx sdk.Context, keeper Keeper, data *types.GenesisState) bool { - keeper.SetParams(ctx, data.GetParams()) - keeper.SetState(ctx, data.GetState()) +func InitGenesis(ctx sdk.Context, k Keeper, swingStoreExportsHandler *SwingStoreExportsHandler, swingStoreExportDir string, data *types.GenesisState) bool { + k.SetParams(ctx, data.GetParams()) + k.SetState(ctx, data.GetState()) swingStoreExportData := data.GetSwingStoreExportData() - if len(swingStoreExportData) > 0 { - // See https://github.com/Agoric/agoric-sdk/issues/6527 - panic("genesis with swing-store state not implemented") + if len(swingStoreExportData) == 0 { + return true } - // TODO: bootstrap only if not restoring swing-store from genesis state - return true + artifactProvider, err := keeper.OpenSwingStoreExportDirectory(swingStoreExportDir) + if err != nil { + panic(err) + } + + swingStore := k.GetSwingStore(ctx) + + for _, entry := range swingStoreExportData { + swingStore.Set([]byte(entry.Key), []byte(entry.Value)) + } + + snapshotHeight := uint64(ctx.BlockHeight()) + + getExportDataReader := func() (agoric.KVEntryReader, error) { + exportDataIterator := swingStore.Iterator(nil, nil) + return agoric.NewKVIteratorReader(exportDataIterator), nil + } + + err = swingStoreExportsHandler.RestoreExport( + keeper.SwingStoreExportProvider{ + BlockHeight: snapshotHeight, + GetExportDataReader: getExportDataReader, + ReadNextArtifact: artifactProvider.ReadNextArtifact, + }, + keeper.SwingStoreRestoreOptions{ + ArtifactMode: keeper.SwingStoreArtifactModeReplay, + ExportDataMode: keeper.SwingStoreExportDataModeAll, + }, + ) + if err != nil { + panic(err) + } + + return false } func ExportGenesis(ctx sdk.Context, k Keeper, swingStoreExportsHandler *SwingStoreExportsHandler, swingStoreExportDir string) *types.GenesisState { diff --git a/golang/cosmos/x/swingset/module.go b/golang/cosmos/x/swingset/module.go index 42b207dab38..ec6f3b4fd45 100644 --- a/golang/cosmos/x/swingset/module.go +++ b/golang/cosmos/x/swingset/module.go @@ -163,7 +163,8 @@ func (am AppModule) checkSwingStoreExportSetup() { func (am AppModule) InitGenesis(ctx sdk.Context, cdc codec.JSONCodec, data json.RawMessage) []abci.ValidatorUpdate { var genesisState types.GenesisState cdc.MustUnmarshalJSON(data, &genesisState) - bootstrapNeeded := InitGenesis(ctx, am.keeper, &genesisState) + am.checkSwingStoreExportSetup() + bootstrapNeeded := InitGenesis(ctx, am.keeper, am.swingStoreExportsHandler, am.swingStoreExportDir, &genesisState) if bootstrapNeeded { am.setBootstrapNeeded() } From 46d34667c4dd66714046c5682b05a5c891bfa8fc Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Thu, 27 Jul 2023 18:43:32 +0000 Subject: [PATCH 7/7] feat(deployment): add genesis export test --- .../upgrade-test-scripts/agoric-upgrade-11/test.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/test.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/test.sh index cd771b3b10e..d2e82ccefff 100755 --- a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/test.sh +++ b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/test.sh @@ -8,13 +8,21 @@ waitForBlock 2 # CWD is agoric-sdk upgrade11=./upgrade-test-scripts/agoric-upgrade-11 -# verify swing-store export-data is consistent +# verify swing-store export-data is consistent and perform genesis style "upgrade" killAgd EXPORT_DIR=$(mktemp -t -d swing-store-export-upgrade-11-XXX) make_swing_store_snapshot $EXPORT_DIR --artifact-mode none || fail "Couldn't make swing-store snapshot" -test_val "$(compare_swing_store_export_data $EXPORT_DIR)" "match" "swing-store consistent state-sync" -rm -rf $EXPORT_DIR +test_val "$(compare_swing_store_export_data $EXPORT_DIR)" "match" "swing-store consistent cosmos kvstore" + +TMP_GENESIS_DIR=$EXPORT_DIR/genesis-export +cp $HOME/.agoric/config/genesis.json $TMP_GENESIS_DIR/old_genesis.json +cp $HOME/.agoric/data/priv_validator_state.json $TMP_GENESIS_DIR/priv_validator_state.json +rm -rf $HOME/.agoric/data +mkdir $HOME/.agoric/data +mv $TMP_GENESIS_DIR/priv_validator_state.json $HOME/.agoric/data +mv $TMP_GENESIS_DIR/* $HOME/.agoric/config/ startAgd +rm -rf $EXPORT_DIR # zoe vat is at incarnation 1 echo "FIXME: bypassed zoe-full-upgrade validation"; return 0