Add support for block log splitting

libraries/chain/controller.cpp

@@ -310,7 +310,7 @@ struct controller_impl {
     reversible_blocks( cfg.blocks_dir/config::reversible_blocks_dir_name,
         cfg.read_only ? database::read_only : database::read_write,
         cfg.reversible_cache_size, false, cfg.db_map_mode, cfg.db_hugepage_paths ),
-    blog( cfg.blocks_dir ),
+    blog( cfg.blocks_dir, cfg.blocks_archive_dir, cfg.blocks_log_stride, cfg.max_retained_block_files, cfg.allow_block_log_auto_fix),
     fork_db( cfg.state_dir ),
     wasmif( cfg.wasm_runtime, cfg.eosvmoc_tierup, db, cfg.state_dir, cfg.eosvmoc_config ),
     resource_limits( db, [&s]() { return s.get_deep_mind_logger(); }),

libraries/chain/include/eosio/chain/block_log.hpp

@@ -8,7 +8,7 @@ namespace eosio { namespace chain {
    namespace detail { class block_log_impl; }
 
    /* The block log is an external append only log of the blocks with a header. Blocks should only
-    * be written to the log after they irreverisble as the log is append only. The log is a doubly
+    * be written to the log after they irreversible as the log is append only. The log is a doubly
     * linked list of blocks. There is a secondary index file of only block positions that enables
     * O(1) random access lookup by block number.
     *
@@ -34,7 +34,8 @@ namespace eosio { namespace chain {
 
    class block_log {
       public:
-         block_log(const fc::path& data_dir);
+         block_log(const fc::path& data_dir, fc::path backup_dir = fc::path(), uint64_t stride=1000, 
+                   uint16_t max_retained_files=10, bool allow_block_log_auto_fix=false);
          block_log(block_log&& other) = default;
          ~block_log();
 
@@ -64,7 +65,7 @@ namespace eosio { namespace chain {
          static const uint32_t min_supported_version;
          static const uint32_t max_supported_version;
 
-         static fc::path repair_log( const fc::path& data_dir, uint32_t truncate_at_block = UINT32_MAX );
+         static fc::path repair_log( const fc::path& data_dir, uint32_t truncate_at_block = UINT32_MAX, const char* reversible_block_dir_name="" );
 
          static fc::optional<genesis_state> extract_genesis_state( const fc::path& data_dir );
 
@@ -83,6 +84,7 @@ namespace eosio { namespace chain {
 
          // used for unit test to generate older version blocklog
          static void set_version(uint32_t);
+         uint32_t    version() const;
 
          /**
           * @param n Only test 1 block out of every n blocks. If n is 0, it is maximum between 1 and the ceiling of the total number blocks divided by 8.

libraries/chain/include/eosio/chain/chain_id_type.hpp

@@ -38,6 +38,8 @@ namespace chain {
 
       void reflector_init()const;
 
+      bool empty() const { return *this == chain_id_type{};}
+
       private:
          chain_id_type() = default;
 
@@ -51,6 +53,7 @@ namespace chain {
          friend struct eosio::handshake_message;
          friend class block_log;
          friend struct block_log_preamble;
+         friend struct block_log_catalog;
          friend class controller;
          friend struct controller_impl;
          friend class global_property_object;

libraries/chain/include/eosio/chain/config.hpp

@@ -9,7 +9,10 @@ namespace eosio { namespace chain { namespace config {
 typedef __uint128_t uint128_t;
 
 const static auto default_blocks_dir_name    = "blocks";
-const static auto reversible_blocks_dir_name = "reversible";
+const static auto default_blocks_archive_dir_name  = "archive";
+const static auto default_blocks_log_stride        = UINT32_MAX;
+const static auto default_max_retained_block_files = 10;
+const static auto reversible_blocks_dir_name       = "reversible";
 const static auto default_reversible_cache_size = 340*1024*1024ll;/// 1MB * 340 blocks based on 21 producer BFT delay
 const static auto default_reversible_guard_size = 2*1024*1024ll;/// 1MB * 340 blocks based on 21 producer BFT delay
 

libraries/chain/include/eosio/chain/controller.hpp

@@ -66,19 +66,23 @@ namespace eosio { namespace chain {
             flat_set< pair<account_name, action_name> > action_blacklist;
             flat_set<public_key_type> key_blacklist;
             path                     blocks_dir                 = chain::config::default_blocks_dir_name;
+            path                     blocks_archive_dir         = chain::config::default_blocks_archive_dir_name;
             path                     state_dir                  = chain::config::default_state_dir_name;
             uint64_t                 state_size                 = chain::config::default_state_size;
             uint64_t                 state_guard_size           = chain::config::default_state_guard_size;
             uint64_t                 reversible_cache_size      = chain::config::default_reversible_cache_size;
             uint64_t                 reversible_guard_size      = chain::config::default_reversible_guard_size;
             uint32_t                 sig_cpu_bill_pct           = chain::config::default_sig_cpu_bill_pct;
             uint16_t                 thread_pool_size           = chain::config::default_controller_thread_pool_size;
+            uint16_t                 max_retained_block_files   = chain::config::default_max_retained_block_files;
+            uint64_t                 blocks_log_stride          = chain::config::default_blocks_log_stride;
             fc::microseconds         abi_serializer_max_time_us = fc::microseconds(chain::config::default_abi_serializer_max_time_us);
             bool                     read_only                  = false;
             bool                     force_all_checks           = false;
             bool                     disable_replay_opts        = false;
             bool                     contracts_console          = false;
             bool                     allow_ram_billing_in_notify = false;
+            bool                     allow_block_log_auto_fix   = false;
             uint32_t                 maximum_variable_signature_length = chain::config::default_max_variable_signature_length;
             bool                     disable_all_subjective_mitigations = false; //< for developer & testing purposes, can be configured using `disable-all-subjective-mitigations` when `EOSIO_DEVELOPER` build option is provided
             uint32_t                 terminate_at_block     = 0; //< primarily for testing purposes

libraries/fc/include/fc/filesystem.hpp

@@ -81,6 +81,7 @@ namespace fc {
 
       bool       is_relative()const;
       bool       is_absolute()const;
+      bool       empty() const;
 
       static char    separator_char;
 

libraries/fc/src/filesystem.cpp

@@ -170,6 +170,8 @@ namespace fc {
   bool path::is_relative()const { return _p->is_relative(); }
   bool path::is_absolute()const { return _p->is_absolute(); }
 
+  bool path::empty() const { return _p->empty(); }
+
       directory_iterator::directory_iterator( const fc::path& p )
       :_p(p){}
 

plugins/chain_plugin/chain_plugin.cpp

@@ -150,7 +150,6 @@ class chain_plugin_impl {
 
 
    fc::optional<fork_database>      fork_db;
-   fc::optional<block_log>          block_logger;
    fc::optional<controller::config> chain_config;
    fc::optional<controller>         chain;
    fc::optional<genesis_state>      genesis;
@@ -228,6 +227,21 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
    cfg.add_options()
          ("blocks-dir", bpo::value<bfs::path>()->default_value("blocks"),
           "the location of the blocks directory (absolute path or relative to application data dir)")
+         ("blocks-log-stride", bpo::value<uint32_t>()->default_value(config::default_blocks_log_stride),
+         "split the block log file when the head block number is the multiple of the split factor\n"
+         "When the stride is reached, the current block log and index will be renamed 'blocks-num_begin-num_end.log/index'\n"
+         "and a new current block log and index will be created with the most recent block. All files following\n"
+         "this format will be used to construct an extended block log.")
+         ("max-retained-block-files", bpo::value<uint16_t>()->default_value(config::default_max_retained_block_files),
+          "the maximum number of blocks files to retain so that the blocks in those files can be queried.\n" 
+          "When the number is reached, the oldest block file would be move to archive dir or deleted if the archive dir is empty.\n"
+          "The retained block log files should not be manipulated by users." )
+         ("blocks-archive-dir", bpo::value<bfs::path>()->default_value(config::default_blocks_archive_dir_name),
+          "the location of the blocks archive directory (absolute path or relative to blocks dir).\n"
+          "If the value is empty, blocks files beyond the retained limit will be deleted.\n"
+          "All files in the archive directory are completely under user's control, i.e. they won't be accessed by nodeos anymore.")
+         ("allow-block-log-auto-fix", bpo::value<bool>()->default_value("false"),
+          "When the existing block log is inconsistent with the index, allows fixing the block log and index files")
          ("protocol-features-dir", bpo::value<bfs::path>()->default_value("protocol_features"),
           "the location of the protocol_features directory (absolute path or relative to application config dir)")
          ("checkpoint", bpo::value<vector<string>>()->composing(), "Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints.")
@@ -612,7 +626,7 @@ void
 chain_plugin::do_hard_replay(const variables_map& options) {
          ilog( "Hard replay requested: deleting state database" );
          clear_directory_contents( my->chain_config->state_dir );
-         auto backup_dir = block_log::repair_log( my->blocks_dir, options.at( "truncate-at-block" ).as<uint32_t>());
+         auto backup_dir = block_log::repair_log( my->blocks_dir, options.at( "truncate-at-block" ).as<uint32_t>(),config::reversible_blocks_dir_name);
          if( fc::exists( backup_dir / config::reversible_blocks_dir_name ) ||
              options.at( "fix-reversible-blocks" ).as<bool>()) {
             // Do not try to recover reversible blocks if the directory does not exist, unless the option was explicitly provided.
@@ -742,6 +756,10 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
       my->chain_config->blocks_dir = my->blocks_dir;
       my->chain_config->state_dir = app().data_dir() / config::default_state_dir_name;
       my->chain_config->read_only = my->readonly;
+      my->chain_config->blocks_archive_dir = options.at("blocks-archive-dir").as<bfs::path>();
+      my->chain_config->blocks_log_stride  = options.at("blocks-log-stride").as<uint32_t>();
+      my->chain_config->max_retained_block_files = options.at("max-retained-block-files").as<uint16_t>();
+      my->chain_config->allow_block_log_auto_fix = options.at("allow-block-log-auto-fix").as<bool>();
 
       if (auto resmon_plugin = app().find_plugin<resource_monitor_plugin>()) {
         resmon_plugin->monitor_directory(my->chain_config->blocks_dir);

unittests/auth_tests.cpp

@@ -7,7 +7,6 @@
 #include <eosio/chain/resource_limits.hpp>
 #include <eosio/chain/resource_limits_private.hpp>
 
-#include <eosio/testing/tester_network.hpp>
 
 #ifdef NON_VALIDATING_TEST
 #define TESTER tester

unittests/delay_tests.cpp

@@ -1,6 +1,5 @@
 #include <eosio/chain/generated_transaction_object.hpp>
 #include <eosio/chain/global_property_object.hpp>
-#include <eosio/testing/tester_network.hpp>
 
 #include <boost/test/unit_test.hpp>