add guard track to database

In addition to setting the size you can now set a minimum guard on the chain state db.  If there is a db-growing operation called on controller while this guard is violated it will throw and the node should dhutdown gracefully.

related to EOSIO#4555

libraries/chain/controller.cpp

@@ -1256,10 +1256,12 @@ fork_database& controller::fork_db()const { return my->fork_db; }
 
 
 void controller::start_block( block_timestamp_type when, uint16_t confirm_block_count) {
+   validate_db_available_size();
    my->start_block(when, confirm_block_count, block_status::incomplete );
 }
 
 void controller::finalize_block() {
+   validate_db_available_size();
    my->finalize_block();
 }
 
@@ -1268,6 +1270,7 @@ void controller::sign_block( const std::function<signature_type( const digest_ty
 }
 
 void controller::commit_block() {
+   validate_db_available_size();
    my->commit_block(true);
 }
 
@@ -1276,19 +1279,23 @@ void controller::abort_block() {
 }
 
 void controller::push_block( const signed_block_ptr& b, block_status s ) {
+   validate_db_available_size();
    my->push_block( b, s );
 }
 
 void controller::push_confirmation( const header_confirmation& c ) {
+   validate_db_available_size();
    my->push_confirmation( c );
 }
 
 transaction_trace_ptr controller::push_transaction( const transaction_metadata_ptr& trx, fc::time_point deadline, uint32_t billed_cpu_time_us ) {
+   validate_db_available_size();
    return my->push_transaction(trx, deadline, false, billed_cpu_time_us);
 }
 
 transaction_trace_ptr controller::push_scheduled_transaction( const transaction_id_type& trxid, fc::time_point deadline, uint32_t billed_cpu_time_us )
 {
+   validate_db_available_size();
    return my->push_scheduled_transaction( trxid, deadline, billed_cpu_time_us );
 }
 
@@ -1604,6 +1611,12 @@ void controller::validate_tapos( const transaction& trx )const { try {
               ("tapos_summary", tapos_block_summary));
 } FC_CAPTURE_AND_RETHROW() }
 
+void controller::validate_db_available_size() const {
+   const auto free = db().get_segment_manager()->get_free_memory();
+   const auto guard = my->conf.state_guard_size;
+   EOS_ASSERT(free >= guard, database_guard_exception, "database free: ${f}, guard size: ${g}", ("f", free)("g",guard));
+}
+
 bool controller::is_known_unexpired_transaction( const transaction_id_type& id) const {
    return db().find<transaction_object, by_trx_id>(id);
 }

libraries/chain/include/eosio/chain/config.hpp

@@ -19,6 +19,7 @@ const static auto default_reversible_cache_size = 340*1024*1024ll;/// 1MB * 340
 const static auto default_state_dir_name     = "state";
 const static auto forkdb_filename            = "forkdb.dat";
 const static auto default_state_size            = 1*1024*1024*1024ll;
+const static auto default_state_guard_size      = 512*1024ll;
 
 
 const static uint64_t system_account_name    = N(eosio);

libraries/chain/include/eosio/chain/controller.hpp

@@ -52,6 +52,7 @@ namespace eosio { namespace chain {
             path                     blocks_dir             =  chain::config::default_blocks_dir_name;
             path                     state_dir              =  chain::config::default_state_dir_name;
             uint64_t                 state_size             =  chain::config::default_state_size;
+            uint64_t                 state_guard_size       =  chain::config::default_state_guard_size;
             uint64_t                 reversible_cache_size  =  chain::config::default_reversible_cache_size;
             bool                     read_only              =  false;
             bool                     force_all_checks       =  false;
@@ -183,6 +184,7 @@ namespace eosio { namespace chain {
          void validate_referenced_accounts( const transaction& t )const;
          void validate_expiration( const transaction& t )const;
          void validate_tapos( const transaction& t )const;
+         void validate_db_available_size() const;
 
          bool is_known_unexpired_transaction( const transaction_id_type& id) const;
 

libraries/chain/include/eosio/chain/exceptions.hpp

@@ -228,7 +228,9 @@ namespace eosio { namespace chain {
                                     3060003, "Contract Table Query Exception" )
       FC_DECLARE_DERIVED_EXCEPTION( contract_query_exception,       database_exception,
                                     3060004, "Contract Query Exception" )
-
+      FC_DECLARE_DERIVED_EXCEPTION( database_guard_exception, database_exception,
+                                    3060004, "Database usage is at unsafe levels" )
+
    FC_DECLARE_DERIVED_EXCEPTION( wasm_exception, chain_exception,
                                  3070000, "WASM Exception" )
       FC_DECLARE_DERIVED_EXCEPTION( page_memory_error,        wasm_exception,

plugins/chain_plugin/chain_plugin.cpp

@@ -180,6 +180,7 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
          ("abi-serializer-max-time-ms", bpo::value<uint32_t>()->default_value(config::default_abi_serializer_max_time_ms),
           "Override default maximum ABI serialization time allowed in ms")
          ("chain-state-db-size-mb", bpo::value<uint64_t>()->default_value(config::default_state_size / (1024  * 1024)), "Maximum size (in MB) of the chain state database")
+         ("chain-state-db-guard-size-mb", bpo::value<uint64_t>()->default_value(config::default_state_guard_size / (1024  * 1024)), "Minimum available size (in MB) in the chain state database before the node shutsdown to prevent corruption")
          ("reversible-blocks-db-size-mb", bpo::value<uint64_t>()->default_value(config::default_reversible_cache_size / (1024  * 1024)), "Maximum size (in MB) of the reversible blocks database")
          ("contracts-console", bpo::bool_switch()->default_value(false),
           "print contract's output to console")
@@ -341,6 +342,9 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
       if( options.count( "chain-state-db-size-mb" ))
          my->chain_config->state_size = options.at( "chain-state-db-size-mb" ).as<uint64_t>() * 1024 * 1024;
 
+      if( options.count( "chain-state-db-guard-size-mb" ))
+         my->chain_config->state_guard_size = options.at( "chain-state-db-guard-size-mb" ).as<uint64_t>() * 1024 * 1024;
+
       if( options.count( "reversible-blocks-db-size-mb" ))
          my->chain_config->reversible_cache_size =
                options.at( "reversible-blocks-db-size-mb" ).as<uint64_t>() * 1024 * 1024;
@@ -757,6 +761,14 @@ fc::microseconds chain_plugin::get_abi_serializer_max_time() const {
    return my->abi_serializer_max_time_ms;
 }
 
+void chain_plugin::handle_database_guard_exception(const chain::database_guard_exception& e) const {
+   elog("Database has reached an unsafe level of usage, shutting down to avoid corrupting the database.  "
+        "Please increase the value set for \"chain-state-db-size-mb\" and restart the process!");
+   dlog("Details: ${details}", ("details", e.to_detail_string()));
+
+   // quit the app
+   app().quit();
+}
 
 namespace chain_apis {
 

plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp

@@ -525,6 +525,7 @@ class chain_plugin : public plugin<chain_plugin> {
    chain::chain_id_type get_chain_id() const;
    fc::microseconds get_abi_serializer_max_time() const;
 
+   void handle_database_guard_exception(const chain::database_guard_exception& e) const;
 private:
    unique_ptr<class chain_plugin_impl> my;
 };

plugins/producer_plugin/producer_plugin.cpp

@@ -299,6 +299,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          bool except = false;
          try {
             chain.push_block(block);
+         } catch ( const database_guard_exception& e ) {
+            app().get_plugin<chain_plugin>().handle_database_guard_exception(e);
+            return;
          } catch( const fc::exception& e ) {
             elog((e.to_detail_string()));
             except = true;
@@ -381,6 +384,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                send_response(trace);
             }
 
+         } catch ( const database_guard_exception& e ) {
+            app().get_plugin<chain_plugin>().handle_database_guard_exception(e);
          } catch ( boost::interprocess::bad_alloc& ) {
             raise(SIGUSR1);
          } CATCH_AND_CALL(send_response);
@@ -886,6 +891,9 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block(bool
                // the state of the chain including this transaction
                try {
                   chain.push_transaction(trx, fc::time_point::maximum());
+               } catch ( const database_guard_exception& e ) {
+                  app().get_plugin<chain_plugin>().handle_database_guard_exception(e);
+                  return start_block_result::failed;
                } FC_LOG_AND_DROP();
 
                // remove it from further consideration as it is applied
@@ -927,6 +935,9 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block(bool
                         chain.drop_unapplied_transaction(trx);
                      }
                   }
+               } catch ( const database_guard_exception& e ) {
+                  app().get_plugin<chain_plugin>().handle_database_guard_exception(e);
+                  return start_block_result::failed;
                } FC_LOG_AND_DROP();
             }
 
@@ -965,6 +976,9 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block(bool
                         _blacklisted_transactions.insert(transaction_id_with_expiry{trx, expiration});
                      }
                   }
+               } catch ( const database_guard_exception& e ) {
+                  app().get_plugin<chain_plugin>().handle_database_guard_exception(e);
+                  return start_block_result::failed;
                } FC_LOG_AND_DROP();
             }
          }
@@ -1079,6 +1093,9 @@ bool producer_plugin_impl::maybe_produce_block() {
    try {
       produce_block();
       return true;
+   } catch ( const database_guard_exception& e ) {
+      app().get_plugin<chain_plugin>().handle_database_guard_exception(e);
+      return false;
    } catch ( boost::interprocess::bad_alloc& ) {
       raise(SIGUSR1);
       return false;