[VM] Automatically handle when VM Panic occurs. (#2491)

* set sequencer status to maintence and pause the relayer if vm panic occurs

Revert tx when l2 tx execution failed

* Reset the ABORT status code to Error

* remove println

* use custom error to handle the panic

send a ServiceStatusMessage to notify the sequencer and the relayer

* refactor the function execute_action

crates/rooch-benchmarks/src/tx.rs

@@ -36,6 +36,7 @@ pub fn gen_sequencer(
         rooch_store.clone(),
         ServiceStatus::Active,
         registry,
+        None,
     )
 }
 

crates/rooch-db/Cargo.toml

@@ -23,4 +23,5 @@ moveos-store = { workspace = true }
 
 rooch-config = { workspace = true }
 rooch-store = { workspace = true }
-rooch-indexer = { workspace = true }
+rooch-indexer = { workspace = true }
+rooch-types = { workspace = true }

crates/rooch-db/src/lib.rs

@@ -3,15 +3,19 @@
 
 use std::collections::HashSet;
 
-use anyhow::Result;
+use anyhow::{Error, Result};
+use moveos_store::transaction_store::TransactionStore as TxExecutionInfoStore;
 use moveos_store::MoveOSStore;
+use moveos_types::h256::H256;
 use moveos_types::moveos_std::object::ObjectMeta;
 use prometheus::Registry;
 use raw_store::metrics::DBMetrics;
 use raw_store::{rocks::RocksDB, StoreInstance};
 use rooch_config::store_config::StoreConfig;
 use rooch_indexer::{indexer_reader::IndexerReader, IndexerStore};
+use rooch_store::transaction_store::TransactionStore;
 use rooch_store::RoochStore;
+use rooch_types::sequencer::SequencerInfo;
 
 #[derive(Clone)]
 pub struct RoochDB {
@@ -82,4 +86,91 @@ impl RoochDB {
 
         Ok(startup_info.map(|s| s.into_root_metadata()))
     }
+
+    pub fn revert_tx(&self, tx_hash: H256) -> Result<()> {
+        let last_sequencer_info = self
+            .rooch_store
+            .get_meta_store()
+            .get_sequencer_info()?
+            .ok_or_else(|| anyhow::anyhow!("Load sequencer info failed"))?;
+        let (last_order, last_accumulator_info) = (
+            last_sequencer_info.last_order,
+            last_sequencer_info.last_accumulator_info.clone(),
+        );
+        println!("Load latest sequencer order {:?}", last_order);
+        println!(
+            "Load latest sequencer accumulator info {:?}",
+            last_accumulator_info
+        );
+
+        let ledger_tx_opt = self
+            .rooch_store
+            .transaction_store
+            .get_transaction_by_hash(tx_hash)?;
+        if ledger_tx_opt.is_none() {
+            println!("the ledger tx not exist via tx_hash {}", tx_hash);
+            return Ok(());
+        }
+        let sequencer_info = ledger_tx_opt.unwrap().sequence_info;
+        let tx_order = sequencer_info.tx_order;
+        // check last order equals to sequencer tx order via tx_hash
+        if sequencer_info.tx_order != last_sequencer_info.last_order {
+            return Err(Error::msg(format!(
+                "the last order {} not match current sequencer info tx order {}, tx_hash {}",
+                last_sequencer_info.last_order, sequencer_info.tx_order, tx_hash
+            )));
+        }
+
+        // check only write tx sequence info succ, but not write tx execution info
+        let execution_info = self
+            .moveos_store
+            .transaction_store
+            .get_tx_execution_info(tx_hash)?;
+        if execution_info.is_some() {
+            return Err(Error::msg(format!(
+                "the tx execution info already exist via tx_hash {}, revert tx failed",
+                tx_hash
+            )));
+        }
+
+        let previous_tx_order = last_order - 1;
+        let previous_tx_hash_opt = self
+            .rooch_store
+            .transaction_store
+            .get_tx_hashs(vec![previous_tx_order])?;
+        if previous_tx_hash_opt.is_empty() || previous_tx_hash_opt[0].is_none() {
+            return Err(Error::msg(format!(
+                "the previous tx hash not exist via previous_tx_order  {}",
+                previous_tx_order
+            )));
+        }
+        let previous_tx_hash = previous_tx_hash_opt[0].unwrap();
+        let previous_ledger_tx_opt = self
+            .rooch_store
+            .transaction_store
+            .get_transaction_by_hash(previous_tx_hash)?;
+        if previous_ledger_tx_opt.is_none() {
+            return Err(Error::msg(format!(
+                "the previous ledger tx not exist via tx_hash {}, revert tx failed",
+                previous_tx_hash
+            )));
+        }
+        let previous_sequencer_info = previous_ledger_tx_opt.unwrap().sequence_info;
+
+        let revert_sequencer_info = SequencerInfo::new(
+            previous_tx_order,
+            previous_sequencer_info.tx_accumulator_info(),
+        );
+        self.rooch_store
+            .meta_store
+            .save_sequencer_info_ignore_check(revert_sequencer_info)?;
+        self.rooch_store.remove_transaction(tx_hash, tx_order)?;
+
+        println!(
+            "revert tx succ, tx_hash: {:?}, tx_order {}",
+            tx_hash, tx_order
+        );
+
+        Ok(())
+    }
 }

crates/rooch-event/Cargo.toml

@@ -14,4 +14,6 @@ anyhow = { workspace = true }
 coerce = { workspace = true }
 async-trait = { workspace = true }
 moveos-eventbus = { workspace = true }
-log = { workspace = true }
+log = { workspace = true }
+
+rooch-types = { workspace = true }

crates/rooch-event/src/actor.rs

@@ -1,13 +1,14 @@
 // Copyright (c) RoochNetwork
 // SPDX-License-Identifier: Apache-2.0
 
+use crate::event::{GasUpgradeEvent, ServiceStatusEvent};
 use async_trait::async_trait;
 use coerce::actor::context::ActorContext;
 use coerce::actor::message::{Handler, Message};
 use coerce::actor::Actor;
 use log;
 use moveos_eventbus::bus::{EventBus, EventNotifier};
-use moveos_eventbus::event::GasUpgradeEvent;
+use rooch_types::service_status::ServiceStatus;
 
 #[derive(Default)]
 pub struct EventActor {
@@ -43,6 +44,31 @@ impl Handler<GasUpgradeMessage> for EventActor {
     }
 }
 
+#[derive(Default, Clone, Debug)]
+pub struct ServiceStatusMessage {
+    pub status: ServiceStatus,
+}
+
+impl Message for ServiceStatusMessage {
+    type Result = anyhow::Result<()>;
+}
+
+#[async_trait]
+impl Handler<ServiceStatusMessage> for EventActor {
+    async fn handle(
+        &mut self,
+        message: ServiceStatusMessage,
+        _ctx: &mut ActorContext,
+    ) -> anyhow::Result<()> {
+        log::debug!("EventActor receive message {:?}", message);
+        self.event_bus
+            .notify::<ServiceStatusEvent>(ServiceStatusEvent {
+                status: message.status,
+            })?;
+        Ok(())
+    }
+}
+
 pub struct EventActorSubscribeMessage<T: Send + Sync + 'static> {
     event_type: T,
     subscriber: String,

crates/rooch-event/src/event.rs

@@ -0,0 +1,12 @@
+// Copyright (c) RoochNetwork
+// SPDX-License-Identifier: Apache-2.0
+
+use rooch_types::service_status::ServiceStatus;
+
+#[derive(Default, Clone, Debug)]
+pub struct GasUpgradeEvent {}
+
+#[derive(Default, Clone, Debug)]
+pub struct ServiceStatusEvent {
+    pub status: ServiceStatus,
+}

crates/rooch-event/src/lib.rs

@@ -2,3 +2,4 @@
 // SPDX-License-Identifier: Apache-2.0
 
 pub mod actor;
+pub mod event;

crates/rooch-executor/src/actor/executor.rs

@@ -15,7 +15,6 @@ use move_core_types::vm_status::VMStatus;
 use moveos::moveos::{MoveOS, MoveOSConfig};
 use moveos::vm::vm_status_explainer::explain_vm_status;
 use moveos_eventbus::bus::EventData;
-use moveos_eventbus::event::GasUpgradeEvent;
 use moveos_store::MoveOSStore;
 use moveos_types::function_return_value::FunctionResult;
 use moveos_types::module_binding::MoveFunctionCaller;
@@ -28,6 +27,7 @@ use moveos_types::transaction::{FunctionCall, MoveOSTransaction, VerifiedMoveAct
 use moveos_types::transaction::{MoveAction, VerifiedMoveOSTransaction};
 use prometheus::Registry;
 use rooch_event::actor::{EventActor, EventActorSubscribeMessage, GasUpgradeMessage};
+use rooch_event::event::GasUpgradeEvent;
 use rooch_genesis::FrameworksGasParameters;
 use rooch_store::RoochStore;
 use rooch_types::address::{BitcoinAddress, MultiChainAddress};

crates/rooch-executor/src/actor/reader_executor.rs

@@ -16,7 +16,6 @@ use move_resource_viewer::MoveValueAnnotator;
 use moveos::moveos::MoveOS;
 use moveos::moveos::MoveOSConfig;
 use moveos_eventbus::bus::EventData;
-use moveos_eventbus::event::GasUpgradeEvent;
 use moveos_store::transaction_store::TransactionStore;
 use moveos_store::MoveOSStore;
 use moveos_types::function_return_value::AnnotatedFunctionResult;
@@ -29,6 +28,7 @@ use moveos_types::state_resolver::RootObjectResolver;
 use moveos_types::state_resolver::{AnnotatedStateKV, AnnotatedStateReader, StateKV, StateReader};
 use moveos_types::transaction::TransactionExecutionInfo;
 use rooch_event::actor::{EventActor, EventActorSubscribeMessage};
+use rooch_event::event::GasUpgradeEvent;
 use rooch_genesis::FrameworksGasParameters;
 use rooch_store::RoochStore;
 use rooch_types::framework::{system_post_execute_functions, system_pre_execute_functions};

crates/rooch-pipeline-processor/Cargo.toml

@@ -20,12 +20,18 @@ tracing = { workspace = true }
 prometheus = { workspace = true }
 function_name = { workspace = true }
 
+move-binary-format = { workspace = true }
+move-core-types = { workspace = true }
 
 moveos-types = { workspace = true }
+moveos-stdlib = { workspace = true }
+moveos = { workspace = true }
 metrics = { workspace = true }
 
 rooch-types = { workspace = true }
 rooch-sequencer = { workspace = true }
 rooch-executor = { workspace = true }
 rooch-proposer = { workspace = true }
-rooch-indexer = { workspace = true }
+rooch-indexer = { workspace = true }
+rooch-event = { workspace = true }
+rooch-db = { workspace = true }