FM-191: Execute BottomUpResolve (#201)

* FM-191: Pass the relayed checkpoint to the inner during execution

* FM-191: Add fendermint_vm_resolver crate

* FM-191: Add ResolvePool

* FM-191: Add ResolvePool to App and pass it to execution

* FM-191: Make ResolvePool generic and remember the items for which it resolved CIDs

* FM-191: Submit the checkpoint for resolution

* FM-191: Comment about multi-CID items

* FM-191: Comments about future methods

* FM-191: Move the actor bundle path into AppConfig

* Update fendermint/vm/interpreter/src/chain.rs

Co-authored-by: adlrocha <6717133+adlrocha@users.noreply.github.com>

---------

Co-authored-by: adlrocha <6717133+adlrocha@users.noreply.github.com>

Cargo.toml

@@ -21,44 +21,52 @@ license-file = "LICENSE-APACHE"
 
 [workspace.dependencies]
 anyhow = "1"
-arbitrary = {version = "1", features = ["derive"]}
+arbitrary = { version = "1", features = ["derive"] }
 arbtest = "0.2"
 async-stm = "0.2"
 async-trait = "0.1"
-axum = {version = "0.6", features = ["ws"]}
+axum = { version = "0.6", features = ["ws"] }
 base64 = "0.21"
 blake2b_simd = "1.0"
 bytes = "1.4"
-clap = {version = "4.1", features = ["derive", "env"]}
+clap = { version = "4.1", features = ["derive", "env"] }
 config = "0.13"
 dirs = "5.0"
-ethers = {version = "2.0", features = ["abigen", "ws"]}
-ethers-core = {version = "2.0"}
+ethers = { version = "2.0", features = ["abigen", "ws"] }
+ethers-core = { version = "2.0" }
 fnv = "1.0"
 futures = "0.3"
 hex = "0.4"
-jsonrpc-v2 = {version = "0.11", default-features = false, features = ["bytes-v10"]}
+im = "15.1.0"
+jsonrpc-v2 = { version = "0.11", default-features = false, features = ["bytes-v10"] }
 k256 = "0.11" # Same as tendermint-rs
 lazy_static = "1.4"
 libsecp256k1 = "0.7"
-multihash = {version = "0.16.1", default-features = false}
+multihash = { version = "0.16.1", default-features = false }
 num-traits = "0.2"
 paste = "1"
-prost = {version = "0.11"}
+prost = { version = "0.11" }
 quickcheck = "1"
 quickcheck_macros = "1"
 rand = "0.8"
 rand_chacha = "0.3"
 regex = "1"
-serde = {version = "1", features = ["derive"]}
-serde_json = {version = "1"}
+serde = { version = "1", features = ["derive"] }
+serde_json = { version = "1" }
 serde_tuple = "0.5"
 serde_with = "2.3"
 tempfile = "3.3"
 thiserror = "1"
-tokio = { version = "1", features = ["rt-multi-thread", "macros", "fs", "io-util", "io-std", "sync"] }
+tokio = { version = "1", features = [
+  "rt-multi-thread",
+  "macros",
+  "fs",
+  "io-util",
+  "io-std",
+  "sync",
+] }
 tokio-stream = "0.1.14"
-tokio-util = {version = "0.7.8", features = ["compat"]}
+tokio-util = { version = "0.7.8", features = ["compat"] }
 pin-project = "1.1.2"
 tracing = "0.1"
 tracing-subscriber = "0.3"
@@ -93,7 +101,7 @@ fil_actors_evm_shared = { git = "https://github.com/filecoin-project/builtin-act
 # Using 0.8 because of ref-fvm.
 # 0.9 would be better because of its updated quickcheck dependency.
 # 0.10 breaks some API.
-cid = {version = "0.8", features = ["serde-codec", "std"]}
+cid = { version = "0.8", features = ["serde-codec", "std"] }
 
 # Depending on the release cycle, this dependency might want an earlier version of the FVM.
 # We can work around it by hardcoding the method hashes; currently there is only one.
@@ -106,4 +114,9 @@ tendermint-rpc = { version = "0.31", features = ["secp256k1", "http-client", "we
 tendermint-proto = { version = "0.31" }
 
 # Using the IPC SDK without the `fil-actor` feature so as not to depend on the actor `Runtime`.
-ipc-sdk = { git = "https://github.com/consensus-shipyard/ipc-actors.git", default-features = false, branch = "main" }
+ipc-sdk = { git = "https://github.com/consensus-shipyard/ipc-actors.git", default-features = false, branch = "main" }
+ipc_ipld_resolver = { git = "https://github.com/consensus-shipyard/ipc-ipld-resolver.git", branch = "main" }
+
+[patch.crates-io]
+# Use stable-only features.
+gcra = { git = "https://github.com/consensus-shipyard/gcra-rs.git", branch = "main" }

fendermint/abci/Cargo.toml

@@ -17,8 +17,8 @@ tendermint = { workspace = true }
 
 [dev-dependencies]
 async-stm = { workspace = true }
-im = "15.1.0"
+im = { workspace = true }
 structopt = "0.3"
-tokio = { version = "1" }
-tracing = "0.1"
-tracing-subscriber = "0.3"
+tokio = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }

fendermint/app/Cargo.toml

@@ -41,6 +41,7 @@ fendermint_vm_core = { path = "../vm/core" }
 fendermint_vm_interpreter = { path = "../vm/interpreter", features = ["bundle"] }
 fendermint_vm_message = { path = "../vm/message" }
 fendermint_vm_genesis = { path = "../vm/genesis" }
+fendermint_vm_resolver = { path = "../vm/resolver" }
 
 cid = { workspace = true }
 fvm = { workspace = true }

fendermint/app/src/app.rs

@@ -16,7 +16,7 @@ use fendermint_vm_core::Timestamp;
 use fendermint_vm_interpreter::bytes::{
     BytesMessageApplyRet, BytesMessageCheckRet, BytesMessageQuery, BytesMessageQueryRet,
 };
-use fendermint_vm_interpreter::chain::{ChainMessageApplyRet, IllegalMessage};
+use fendermint_vm_interpreter::chain::{ChainMessageApplyRet, CheckpointPool, IllegalMessage};
 use fendermint_vm_interpreter::fvm::state::{
     empty_state_tree, FvmCheckState, FvmExecState, FvmGenesisState, FvmQueryState, FvmStateParams,
 };
@@ -59,6 +59,7 @@ pub enum AppError {
     NotInitialized = 54,
 }
 
+/// The application state record we keep a history of in the database.
 #[derive(Serialize, Deserialize)]
 pub struct AppState {
     /// Last committed block height.
@@ -82,6 +83,19 @@ impl AppState {
     }
 }
 
+pub struct AppConfig<S: KVStore> {
+    /// Namespace to store the current app state.
+    pub app_namespace: S::Namespace,
+    /// Namespace to store the app state history.
+    pub state_hist_namespace: S::Namespace,
+    /// Size of state history to keep; 0 means unlimited.
+    pub state_hist_size: u64,
+    /// Path to the Wasm bundle.
+    ///
+    /// Only loaded once during genesis; later comes from the [`StateTree`].
+    pub builtin_actors_bundle: PathBuf,
+}
+
 /// Handle ABCI requests.
 #[derive(Clone)]
 pub struct App<DB, SS, S, I>
@@ -118,6 +132,8 @@ where
     state_hist: KVCollection<S, BlockHeight, FvmStateParams>,
     /// Interpreter for block lifecycle events.
     interpreter: Arc<I>,
+    /// CID resolution pool.
+    resolve_pool: CheckpointPool,
     /// State accumulating changes during block execution.
     exec_state: Arc<Mutex<Option<FvmExecState<SS>>>>,
     /// Projected partial state accumulating during transaction checks.
@@ -139,23 +155,22 @@ where
     SS: Blockstore + Clone + 'static,
 {
     pub fn new(
+        config: AppConfig<S>,
         db: DB,
         state_store: SS,
-        builtin_actors_bundle: PathBuf,
-        app_namespace: S::Namespace,
-        state_hist_namespace: S::Namespace,
-        state_hist_size: u64,
         interpreter: I,
+        resolve_pool: CheckpointPool,
     ) -> Result<Self> {
         let app = Self {
             db: Arc::new(db),
             state_store: Arc::new(state_store),
             multi_engine: Arc::new(MultiEngine::new(1)),
-            builtin_actors_bundle,
-            namespace: app_namespace,
-            state_hist: KVCollection::new(state_hist_namespace),
-            state_hist_size,
+            builtin_actors_bundle: config.builtin_actors_bundle,
+            namespace: config.app_namespace,
+            state_hist: KVCollection::new(config.state_hist_namespace),
+            state_hist_size: config.state_hist_size,
             interpreter: Arc::new(interpreter),
+            resolve_pool,
             exec_state: Arc::new(Mutex::new(None)),
             check_state: Arc::new(tokio::sync::Mutex::new(None)),
         };
@@ -259,11 +274,11 @@ where
     /// Take the execution state, update it, put it back, return the output.
     async fn modify_exec_state<T, F, R>(&self, f: F) -> Result<T>
     where
-        F: FnOnce(FvmExecState<SS>) -> R,
-        R: Future<Output = Result<(FvmExecState<SS>, T)>>,
+        F: FnOnce((CheckpointPool, FvmExecState<SS>)) -> R,
+        R: Future<Output = Result<((CheckpointPool, FvmExecState<SS>), T)>>,
     {
         let state = self.take_exec_state();
-        let (state, ret) = f(state).await?;
+        let ((_pool, state), ret) = f((self.resolve_pool.clone(), state)).await?;
         self.put_exec_state(state);
         Ok(ret)
     }
@@ -320,7 +335,7 @@ where
         Message = Vec<u8>,
     >,
     I: ExecInterpreter<
-        State = FvmExecState<SS>,
+        State = (CheckpointPool, FvmExecState<SS>),
         Message = Vec<u8>,
         BeginOutput = FvmApplyRet,
         DeliverOutput = BytesMessageApplyRet,
@@ -611,6 +626,7 @@ where
     async fn commit(&self) -> AbciResult<response::Commit> {
         let exec_state = self.take_exec_state();
 
+        // Commit the execution state to the datastore.
         let mut state = self.committed_state()?;
         state.block_height = exec_state.block_height().try_into()?;
         state.state_params.timestamp = exec_state.timestamp();
@@ -624,6 +640,22 @@ where
             "commit state"
         );
 
+        // TODO: We can defer committing changes the resolution pool to this point.
+        // For example if a checkpoint is successfully executed, that's when we want to remove
+        // that checkpoint from the pool, and not propose it to other validators again.
+        // However, once Tendermint starts delivering the transactions, the commit will surely
+        // follow at the end, so we can also remove these checkpoints from memory at the time
+        // the transaction is delivered, rather than when the whole thing is committed.
+        // It is only important to the persistent database changes as an atomic step in the
+        // commit in case the block execution fails somewhere in the middle for uknown reasons.
+        // But if that happened, we will have to restart the application again anyway, and
+        // repopulate the in-memory checkpoints based on the last committed ledger.
+        // So, while the pool is theoretically part of the evolving state and we can pass
+        // it in and out, unless we want to defer commit to here (which the interpreters aren't
+        // notified about), we could add it to the `ChainMessageInterpreter` as a constructor argument,
+        // a sort of "ambient state", and not worry about in in the `App` at all.
+
+        // Commit app state to the datastore.
         self.set_committed_state(state)?;
 
         // Reset check state.

fendermint/app/src/cmd/run.rs

@@ -3,11 +3,11 @@
 
 use anyhow::{anyhow, Context};
 use fendermint_abci::ApplicationService;
-use fendermint_app::{App, AppStore};
+use fendermint_app::{App, AppConfig, AppStore};
 use fendermint_rocksdb::{blockstore::NamespaceBlockstore, namespaces, RocksDb, RocksDbConfig};
 use fendermint_vm_interpreter::{
     bytes::{BytesMessageInterpreter, ProposalPrepareMode},
-    chain::ChainMessageInterpreter,
+    chain::{ChainMessageInterpreter, CheckpointPool},
     fvm::FvmMessageInterpreter,
     signed::SignedMessageInterpreter,
 };
@@ -38,14 +38,19 @@ async fn run(settings: Settings) -> anyhow::Result<()> {
     let state_store =
         NamespaceBlockstore::new(db.clone(), ns.state_store).context("error creating state DB")?;
 
+    let resolve_pool = CheckpointPool::new();
+
     let app: App<_, _, AppStore, _> = App::new(
+        AppConfig {
+            app_namespace: ns.app,
+            state_hist_namespace: ns.state_hist,
+            state_hist_size: settings.db.state_hist_size,
+            builtin_actors_bundle: settings.builtin_actors_bundle(),
+        },
         db,
         state_store,
-        settings.builtin_actors_bundle(),
-        ns.app,
-        ns.state_hist,
-        settings.db.state_hist_size,
         interpreter,
+        resolve_pool,
     )?;
 
     let service = ApplicationService(app);

fendermint/app/src/lib.rs

@@ -4,7 +4,7 @@ mod app;
 mod store;
 mod tmconv;
 
-pub use app::App;
+pub use app::{App, AppConfig};
 pub use store::AppStore;
 
 // Different type from `ChainEpoch` just because we might use epoch in a more traditional sense for checkpointing.

fendermint/vm/interpreter/Cargo.toml

@@ -14,9 +14,11 @@ fendermint_vm_ipc_actors = { path = "../ipc_actors" }
 fendermint_vm_core = { path = "../core" }
 fendermint_vm_genesis = { path = "../genesis" }
 fendermint_vm_message = { path = "../message" }
+fendermint_vm_resolver = { path = "../resolver" }
 fendermint_eth_hardhat = { path = "../../eth/hardhat" }
 
 async-trait = { workspace = true }
+async-stm = { workspace = true }
 anyhow = { workspace = true }
 ethers = { workspace = true }
 hex = { workspace = true }