add guard track to database

libraries/chain/controller.cpp

@@ -1256,10 +1256,12 @@ fork_database& controller::fork_db()const { return my->fork_db; }
 
 
 void controller::start_block( block_timestamp_type when, uint16_t confirm_block_count) {
+   validate_db_available_size();
    my->start_block(when, confirm_block_count, block_status::incomplete );
 }
 
 void controller::finalize_block() {
+   validate_db_available_size();
    my->finalize_block();
 }
 
@@ -1268,6 +1270,8 @@ void controller::sign_block( const std::function<signature_type( const digest_ty
 }
 
 void controller::commit_block() {
+   validate_db_available_size();
+   validate_reversible_available_size();
    my->commit_block(true);
 }
 
@@ -1276,19 +1280,24 @@ void controller::abort_block() {
 }
 
 void controller::push_block( const signed_block_ptr& b, block_status s ) {
+   validate_db_available_size();
+   validate_reversible_available_size();
    my->push_block( b, s );
 }
 
 void controller::push_confirmation( const header_confirmation& c ) {
+   validate_db_available_size();
    my->push_confirmation( c );
 }
 
 transaction_trace_ptr controller::push_transaction( const transaction_metadata_ptr& trx, fc::time_point deadline, uint32_t billed_cpu_time_us ) {
+   validate_db_available_size();
    return my->push_transaction(trx, deadline, false, billed_cpu_time_us);
 }
 
 transaction_trace_ptr controller::push_scheduled_transaction( const transaction_id_type& trxid, fc::time_point deadline, uint32_t billed_cpu_time_us )
 {
+   validate_db_available_size();
    return my->push_scheduled_transaction( trxid, deadline, billed_cpu_time_us );
 }
 
@@ -1604,6 +1613,18 @@ void controller::validate_tapos( const transaction& trx )const { try {
               ("tapos_summary", tapos_block_summary));
 } FC_CAPTURE_AND_RETHROW() }
 
+void controller::validate_db_available_size() const {
+   const auto free = db().get_segment_manager()->get_free_memory();
+   const auto guard = my->conf.state_guard_size;
+   EOS_ASSERT(free >= guard, database_guard_exception, "database free: ${f}, guard size: ${g}", ("f", free)("g",guard));
+}
+
+void controller::validate_reversible_available_size() const {
+   const auto free = my->reversible_blocks.get_segment_manager()->get_free_memory();
+   const auto guard = my->conf.reversible_guard_size;
+   EOS_ASSERT(free >= guard, reversible_guard_exception, "reversible free: ${f}, guard size: ${g}", ("f", free)("g",guard));
+}
+
 bool controller::is_known_unexpired_transaction( const transaction_id_type& id) const {
    return db().find<transaction_object, by_trx_id>(id);
 }

libraries/chain/include/eosio/chain/config.hpp

@@ -15,10 +15,12 @@ typedef __uint128_t uint128_t;
 const static auto default_blocks_dir_name    = "blocks";
 const static auto reversible_blocks_dir_name = "reversible";
 const static auto default_reversible_cache_size = 340*1024*1024ll;/// 1MB * 340 blocks based on 21 producer BFT delay
+const static auto default_reversible_guard_size = 2*1024*1024ll;/// 1MB * 340 blocks based on 21 producer BFT delay
 
 const static auto default_state_dir_name     = "state";
 const static auto forkdb_filename            = "forkdb.dat";
 const static auto default_state_size            = 1*1024*1024*1024ll;
+const static auto default_state_guard_size      = 128*1024*1024ll;
 
 
 const static uint64_t system_account_name    = N(eosio);

libraries/chain/include/eosio/chain/controller.hpp

@@ -52,7 +52,9 @@ namespace eosio { namespace chain {
             path                     blocks_dir             =  chain::config::default_blocks_dir_name;
             path                     state_dir              =  chain::config::default_state_dir_name;
             uint64_t                 state_size             =  chain::config::default_state_size;
+            uint64_t                 state_guard_size       =  chain::config::default_state_guard_size;
             uint64_t                 reversible_cache_size  =  chain::config::default_reversible_cache_size;
+            uint64_t                 reversible_guard_size  =  chain::config::default_reversible_guard_size;
             bool                     read_only              =  false;
             bool                     force_all_checks       =  false;
             bool                     contracts_console      =  false;
@@ -183,6 +185,8 @@ namespace eosio { namespace chain {
          void validate_referenced_accounts( const transaction& t )const;
          void validate_expiration( const transaction& t )const;
          void validate_tapos( const transaction& t )const;
+         void validate_db_available_size() const;
+         void validate_reversible_available_size() const;
 
          bool is_known_unexpired_transaction( const transaction_id_type& id) const;
 

libraries/chain/include/eosio/chain/exceptions.hpp

@@ -228,7 +228,15 @@ namespace eosio { namespace chain {
                                     3060003, "Contract Table Query Exception" )
       FC_DECLARE_DERIVED_EXCEPTION( contract_query_exception,       database_exception,
                                     3060004, "Contract Query Exception" )
-
+
+   FC_DECLARE_DERIVED_EXCEPTION( guard_exception, database_exception,
+                                 3060100, "Database exception" )
+
+      FC_DECLARE_DERIVED_EXCEPTION( database_guard_exception, guard_exception,
+                                    3060101, "Database usage is at unsafe levels" )
+      FC_DECLARE_DERIVED_EXCEPTION( reversible_guard_exception, guard_exception,
+                                    3060102, "Reversible block log usage is at unsafe levels" )
+
    FC_DECLARE_DERIVED_EXCEPTION( wasm_exception, chain_exception,
                                  3070000, "WASM Exception" )
       FC_DECLARE_DERIVED_EXCEPTION( page_memory_error,        wasm_exception,

libraries/testing/include/eosio/testing/tester.hpp

@@ -319,7 +319,9 @@ namespace eosio { namespace testing {
          vcfg.blocks_dir      = tempdir.path() / std::string("v_").append(config::default_blocks_dir_name);
          vcfg.state_dir  = tempdir.path() /  std::string("v_").append(config::default_state_dir_name);
          vcfg.state_size = 1024*1024*8;
+         vcfg.state_guard_size = 0;
          vcfg.reversible_cache_size = 1024*1024*8;
+         vcfg.reversible_guard_size = 0;
          vcfg.contracts_console = false;
 
          vcfg.genesis.initial_timestamp = fc::time_point::from_iso_string("2020-01-01T00:00:00.000");

libraries/testing/tester.cpp

@@ -86,7 +86,9 @@ namespace eosio { namespace testing {
       cfg.blocks_dir      = tempdir.path() / config::default_blocks_dir_name;
       cfg.state_dir  = tempdir.path() / config::default_state_dir_name;
       cfg.state_size = 1024*1024*8;
+      cfg.state_guard_size = 0;
       cfg.reversible_cache_size = 1024*1024*8;
+      cfg.reversible_guard_size = 0;
       cfg.contracts_console = true;
       cfg.read_mode = read_mode;
 

plugins/chain_plugin/chain_plugin.cpp

@@ -179,8 +179,10 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
          ("wasm-runtime", bpo::value<eosio::chain::wasm_interface::vm_type>()->value_name("wavm/binaryen"), "Override default WASM runtime")
          ("abi-serializer-max-time-ms", bpo::value<uint32_t>()->default_value(config::default_abi_serializer_max_time_ms),
           "Override default maximum ABI serialization time allowed in ms")
-         ("chain-state-db-size-mb", bpo::value<uint64_t>()->default_value(config::default_state_size / (1024  * 1024)), "Maximum size (in MB) of the chain state database")
-         ("reversible-blocks-db-size-mb", bpo::value<uint64_t>()->default_value(config::default_reversible_cache_size / (1024  * 1024)), "Maximum size (in MB) of the reversible blocks database")
+         ("chain-state-db-size-mb", bpo::value<uint64_t>()->default_value(config::default_state_size / (1024  * 1024)), "Maximum size (in MiB) of the chain state database")
+         ("chain-state-db-guard-size-mb", bpo::value<uint64_t>()->default_value(config::default_state_guard_size / (1024  * 1024)), "Safely shut down node when free space remaining in the chain state database drops below this size (in MiB).")
+         ("reversible-blocks-db-size-mb", bpo::value<uint64_t>()->default_value(config::default_reversible_cache_size / (1024  * 1024)), "Maximum size (in MiB) of the reversible blocks database")
+         ("reversible-blocks-db-guard-size-mb", bpo::value<uint64_t>()->default_value(config::default_reversible_guard_size / (1024  * 1024)), "Safely shut down node when free space remaining in the reverseible blocks database drops below this size (in MiB).")
          ("contracts-console", bpo::bool_switch()->default_value(false),
           "print contract's output to console")
          ("actor-whitelist", boost::program_options::value<vector<string>>()->composing()->multitoken(),
@@ -341,10 +343,16 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
       if( options.count( "chain-state-db-size-mb" ))
          my->chain_config->state_size = options.at( "chain-state-db-size-mb" ).as<uint64_t>() * 1024 * 1024;
 
+      if( options.count( "chain-state-db-guard-size-mb" ))
+         my->chain_config->state_guard_size = options.at( "chain-state-db-guard-size-mb" ).as<uint64_t>() * 1024 * 1024;
+
       if( options.count( "reversible-blocks-db-size-mb" ))
          my->chain_config->reversible_cache_size =
                options.at( "reversible-blocks-db-size-mb" ).as<uint64_t>() * 1024 * 1024;
 
+      if( options.count( "reversible-blocks-db-guard-size-mb" ))
+         my->chain_config->reversible_guard_size = options.at( "reversible-blocks-db-guard-size-mb" ).as<uint64_t>() * 1024 * 1024;
+
       if( my->wasm_runtime )
          my->chain_config->wasm_runtime = *my->wasm_runtime;
 
@@ -557,7 +565,14 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
 
 void chain_plugin::plugin_startup()
 { try {
-   my->chain->startup();
+   try {
+      my->chain->startup();
+   } catch (const database_guard_exception& e) {
+      log_guard_exception(e);
+      // make sure to properly close the db
+      my->chain.reset();
+      throw;
+   }
 
    if(!my->readonly) {
       ilog("starting chain in read/write mode");
@@ -757,6 +772,24 @@ fc::microseconds chain_plugin::get_abi_serializer_max_time() const {
    return my->abi_serializer_max_time_ms;
 }
 
+void chain_plugin::log_guard_exception(const chain::guard_exception&e ) const {
+   if (e.code() == chain::database_guard_exception::code_value) {
+      elog("Database has reached an unsafe level of usage, shutting down to avoid corrupting the database.  "
+           "Please increase the value set for \"chain-state-db-size-mb\" and restart the process!");
+   } else if (e.code() == chain::reversible_guard_exception::code_value) {
+      elog("Reversible block database has reached an unsafe level of usage, shutting down to avoid corrupting the database.  "
+           "Please increase the value set for \"reversible-blocks-db-size-mb\" and restart the process!");
+   }
+
+   dlog("Details: ${details}", ("details", e.to_detail_string()));
+}
+
+void chain_plugin::handle_guard_exception(const chain::guard_exception& e) const {
+   log_guard_exception(e);
+
+   // quit the app
+   app().quit();
+}
 
 namespace chain_apis {
 

plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp

@@ -525,7 +525,10 @@ class chain_plugin : public plugin<chain_plugin> {
    chain::chain_id_type get_chain_id() const;
    fc::microseconds get_abi_serializer_max_time() const;
 
+   void handle_guard_exception(const chain::guard_exception& e) const;
 private:
+   void log_guard_exception(const chain::guard_exception& e) const;
+
    unique_ptr<class chain_plugin_impl> my;
 };
 

plugins/producer_plugin/producer_plugin.cpp

@@ -299,6 +299,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          bool except = false;
          try {
             chain.push_block(block);
+         } catch ( const guard_exception& e ) {
+            app().get_plugin<chain_plugin>().handle_guard_exception(e);
+            return;
          } catch( const fc::exception& e ) {
             elog((e.to_detail_string()));
             except = true;
@@ -381,6 +384,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                send_response(trace);
             }
 
+         } catch ( const guard_exception& e ) {
+            app().get_plugin<chain_plugin>().handle_guard_exception(e);
          } catch ( boost::interprocess::bad_alloc& ) {
             raise(SIGUSR1);
          } CATCH_AND_CALL(send_response);
@@ -886,6 +891,9 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block(bool
                // the state of the chain including this transaction
                try {
                   chain.push_transaction(trx, fc::time_point::maximum());
+               } catch ( const guard_exception& e ) {
+                  app().get_plugin<chain_plugin>().handle_guard_exception(e);
+                  return start_block_result::failed;
                } FC_LOG_AND_DROP();
 
                // remove it from further consideration as it is applied
@@ -927,6 +935,9 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block(bool
                         chain.drop_unapplied_transaction(trx);
                      }
                   }
+               } catch ( const guard_exception& e ) {
+                  app().get_plugin<chain_plugin>().handle_guard_exception(e);
+                  return start_block_result::failed;
                } FC_LOG_AND_DROP();
             }
 
@@ -965,6 +976,9 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block(bool
                         _blacklisted_transactions.insert(transaction_id_with_expiry{trx, expiration});
                      }
                   }
+               } catch ( const guard_exception& e ) {
+                  app().get_plugin<chain_plugin>().handle_guard_exception(e);
+                  return start_block_result::failed;
                } FC_LOG_AND_DROP();
             }
          }
@@ -1079,6 +1093,9 @@ bool producer_plugin_impl::maybe_produce_block() {
    try {
       produce_block();
       return true;
+   } catch ( const guard_exception& e ) {
+      app().get_plugin<chain_plugin>().handle_guard_exception(e);
+      return false;
    } catch ( boost::interprocess::bad_alloc& ) {
       raise(SIGUSR1);
       return false;