Merge branch 'huangminghuang/block_log' into GH-592-ship-crash-split-…

…ship-log

# Conflicts:
#	libraries/chain/include/eosio/chain/log_catalog.hpp
#	libraries/chain/include/eosio/chain/log_data_base.hpp
#	libraries/chain/include/eosio/chain/log_index.hpp

libraries/chain/controller.cpp

@@ -236,7 +236,8 @@ struct controller_impl {
    std::optional<fc::microseconds> subjective_cpu_leeway;
    bool                            trusted_producer_light_validation = false;
    uint32_t                        snapshot_head_block = 0;
-   named_thread_pool               thread_pool;
+   struct chain; // chain is a namespace so use an embedded type for the named_thread_pool tag
+   named_thread_pool<chain>        thread_pool;
    platform_timer                  timer;
    deep_mind_handler*              deep_mind_logger = nullptr;
    bool                            okay_to_print_integrity_hash_on_stop = false;
@@ -292,7 +293,7 @@ struct controller_impl {
     db( cfg.state_dir,
         cfg.read_only ? database::read_only : database::read_write,
         cfg.state_size, false, cfg.db_map_mode ),
-    blog( cfg.blocks_dir, cfg.prune_config ),
+    blog( cfg.blocks_dir, cfg.blog ),
     fork_db( cfg.blocks_dir / config::reversible_blocks_dir_name ),
     wasmif( cfg.wasm_runtime, cfg.eosvmoc_tierup, db, cfg.state_dir, cfg.eosvmoc_config, !cfg.profile_accounts.empty() ),
     resource_limits( db, [&s]() { return s.get_deep_mind_logger(); }),
@@ -301,7 +302,7 @@ struct controller_impl {
     conf( cfg ),
     chain_id( chain_id ),
     read_mode( cfg.read_mode ),
-    thread_pool( "chain" )
+    thread_pool()
    {
       fork_db.open( [this]( block_timestamp_type timestamp,
                             const flat_set<digest_type>& cur_features,
@@ -586,10 +587,6 @@ struct controller_impl {
          elog( "Failed initialization from snapshot - db storage not configured to have enough storage for the provided snapshot, please increase and retry snapshot" );
          shutdown();
       }
-
-      if (conf.prune_config && conf.prune_config->prune_blocks == 0) {
-         blog.remove();
-      }
    }
 
    void startup(std::function<void()> shutdown, std::function<bool()> check_shutdown, const genesis_state& genesis) {
@@ -623,10 +620,6 @@ struct controller_impl {
          blog.reset( genesis, head->block );
       }
       init(check_shutdown);
-
-      if (conf.prune_config && conf.prune_config->prune_blocks == 0) {
-         blog.remove();
-      }
    }
 
    void startup(std::function<void()> shutdown, std::function<bool()> check_shutdown) {
@@ -657,10 +650,6 @@ struct controller_impl {
       head = fork_db.head();
 
       init(check_shutdown);
-
-      if (conf.prune_config && conf.prune_config->prune_blocks == 0) {
-         blog.remove();
-      }
    }
 
 
@@ -3571,7 +3560,7 @@ std::optional<chain_id_type> controller::extract_chain_id_from_db( const path& s
       if (gpo==nullptr) return {};
 
       return gpo->chain_id;
-   } catch (std::system_error &) {} //  do not propagate db_error_code::not_found" for absent db, so it will be created 
+   } catch (std::system_error &) {} //  do not propagate db_error_code::not_found" for absent db, so it will be created
 
    return {};
 }

libraries/chain/include/eosio/chain/block_log.hpp

@@ -2,10 +2,11 @@
 #include <fc/filesystem.hpp>
 #include <eosio/chain/block.hpp>
 #include <eosio/chain/genesis_state.hpp>
+#include <eosio/chain/block_log_config.hpp>
 
 namespace eosio { namespace chain {
 
-   namespace detail { class block_log_impl; }
+   namespace detail { struct block_log_impl; }
 
    /* The block log is an external append only log of the blocks with a header. Blocks should only
     * be written to the log after they irreverisble as the log is append only. The log is a doubly
@@ -36,15 +37,10 @@ namespace eosio { namespace chain {
     * and unreadable due to reclamation for purposes of saving space.
     */
 
-   struct block_log_prune_config {
-      uint32_t                prune_blocks;                  //number of blocks to prune to when doing a prune
-      size_t                  prune_threshold = 4*1024*1024; //(approximately) how many bytes need to be added before a prune is performed
-      std::optional<size_t>   vacuum_on_close;               //when set, a vacuum is performed on dtor if log contains less than this many live bytes
-   };
 
    class block_log {
       public:
-         block_log(const fc::path& data_dir, std::optional<block_log_prune_config> prune_config);
+         block_log(const fc::path& data_dir, const block_log_config& config = block_log_config{});
          block_log(block_log&& other);
          ~block_log();
 
@@ -54,23 +50,22 @@ namespace eosio { namespace chain {
          void flush();
          void reset( const genesis_state& gs, const signed_block_ptr& genesis_block );
          void reset( const chain_id_type& chain_id, uint32_t first_block_num );
-         void remove(); // remove blocks.log and blocks.index
 
-         signed_block_ptr read_block(uint64_t file_pos)const;
-         void             read_block_header(block_header& bh, uint64_t file_pos)const;
          signed_block_ptr read_block_by_num(uint32_t block_num)const;
          block_id_type    read_block_id_by_num(uint32_t block_num)const;
+
          signed_block_ptr read_block_by_id(const block_id_type& id)const {
             return read_block_by_num(block_header::num_from_id(id));
          }
 
          /**
           * Return offset of block in file, or block_log::npos if it does not exist.
           */
-         uint64_t get_block_pos(uint32_t block_num) const;
-         signed_block_ptr        read_head()const;
+
+         signed_block_ptr        read_head()const; //use blocklog
          const signed_block_ptr& head()const;
          const block_id_type&    head_id()const;
+
          uint32_t                first_block_num() const;
 
          static const uint64_t npos = std::numeric_limits<uint64_t>::max();
@@ -96,35 +91,26 @@ namespace eosio { namespace chain {
 
          static bool extract_block_range(const fc::path& block_dir, const fc::path&output_dir, block_num_type& start, block_num_type& end, bool rename_input=false);
 
+         static bool trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block);
+         static int  trim_blocklog_end(fc::path block_dir, uint32_t n);
+
+         // used for unit test to generate older version blocklog
+         static void set_initial_version(uint32_t);
+         uint32_t    version() const;
+         uint64_t get_block_pos(uint32_t block_num) const;
+
+
+         /**
+          * @param n Only test 1 block out of every n blocks. If n is 0, the interval is adjusted so that at most 8 blocks are tested.
+          */
+         static void smoke_test(fc::path block_dir, uint32_t n);
+
+         static void extract_blocklog(const fc::path& log_filename, const fc::path& index_filename,
+                                      const fc::path& dest_dir, uint32_t start_block, uint32_t num_blocks);
+         static void split_blocklog(const fc::path& block_dir, const fc::path& dest_dir, uint32_t stride);
+         static void merge_blocklogs(const fc::path& block_dir, const fc::path& dest_dir);
    private:
-         void open(const fc::path& data_dir);
-         void construct_index();
 
          std::unique_ptr<detail::block_log_impl> my;
    };
-
-//to derive blknum_offset==14 see block_header.hpp and note on disk struct is packed
-//   block_timestamp_type timestamp;                  //bytes 0:3
-//   account_name         producer;                   //bytes 4:11
-//   uint16_t             confirmed;                  //bytes 12:13
-//   block_id_type        previous;                   //bytes 14:45, low 4 bytes is big endian block number of previous block
-
-   struct trim_data {            //used by trim_blocklog_front(), trim_blocklog_end(), and smoke_test()
-      trim_data(fc::path block_dir);
-      ~trim_data();
-      uint64_t block_index(uint32_t n) const;
-      uint64_t block_pos(uint32_t n);
-      fc::path block_file_name, index_file_name;        //full pathname for blocks.log and blocks.index
-      uint32_t version = 0;                              //blocklog version
-      uint32_t first_block = 0;                          //first block in blocks.log
-      uint32_t last_block = 0;                          //last block in blocks.log
-      FILE* blk_in = nullptr;                            //C style files for reading blocks.log and blocks.index
-      FILE* ind_in = nullptr;                            //C style files for reading blocks.log and blocks.index
-      //we use low level file IO because it is distinctly faster than C++ filebuf or iostream
-      uint64_t first_block_pos = 0;                      //file position in blocks.log for the first block in the log
-      genesis_state gs;
-      chain_id_type chain_id;
-
-      static constexpr int blknum_offset{14};            //offset from start of block to 4 byte block number, valid for the only allowed versions
-   };
 } }

libraries/chain/include/eosio/chain/block_log_config.hpp

@@ -0,0 +1,34 @@
+#pragma once
+#include <boost/filesystem/path.hpp>
+#include <variant>
+
+namespace eosio { namespace chain {
+
+   namespace bfs = boost::filesystem;
+
+   struct basic_blocklog_config {
+      bool      fix_irreversible_blocks = false;
+   };
+
+   struct empty_blocklog_config {};
+
+   struct partitioned_blocklog_config {
+      bfs::path retained_dir;
+      bfs::path archive_dir;
+      uint32_t  stride                  = UINT32_MAX;
+      uint32_t  max_retained_files      = UINT32_MAX;
+      bool      fix_irreversible_blocks = false;
+   };
+
+   struct prune_blocklog_config {
+      uint32_t prune_blocks; // number of blocks to prune to when doing a prune
+      size_t   prune_threshold =
+            4 * 1024 * 1024; //(approximately) how many bytes need to be added before a prune is performed
+      std::optional<size_t>
+            vacuum_on_close; // when set, a vacuum is performed on dtor if log contains less than this many live bytes
+   };
+
+   using block_log_config =
+         std::variant<basic_blocklog_config, empty_blocklog_config, partitioned_blocklog_config, prune_blocklog_config>;
+
+}} // namespace eosio::chain

libraries/chain/include/eosio/chain/chain_id_type.hpp

@@ -37,6 +37,12 @@ namespace chain {
 
       void reflector_init()const;
 
+      static chain_id_type empty_chain_id() {
+         return {};
+      }
+
+      bool empty() const { return *this == chain_id_type{};}
+
       private:
          chain_id_type() = default;
 

libraries/chain/include/eosio/chain/controller.hpp

@@ -68,7 +68,7 @@ namespace eosio { namespace chain {
             flat_set< pair<account_name, action_name> > action_blacklist;
             flat_set<public_key_type> key_blacklist;
             path                     blocks_dir             =  chain::config::default_blocks_dir_name;
-            std::optional<block_log_prune_config>  prune_config;
+            block_log_config         blog;
             path                     state_dir              =  chain::config::default_state_dir_name;
             uint64_t                 state_size             =  chain::config::default_state_size;
             uint64_t                 state_guard_size       =  chain::config::default_state_guard_size;

libraries/chain/include/eosio/chain/log_catalog.hpp

@@ -44,8 +44,6 @@ struct log_catalog {
    using size_type                 = typename collection_t::size_type;
    static constexpr size_type npos = std::numeric_limits<size_type>::max();
 
-   using mapmode = boost::iostreams::mapped_file::mapmode;
-
    bfs::path    retained_dir;
    bfs::path    archive_dir;
    size_type    max_retained_files = std::numeric_limits<size_type>::max();
@@ -119,7 +117,7 @@ struct log_catalog {
       });
    }
 
-   bool index_matches_data(const bfs::path& index_path, const LogData& log) const {
+   bool index_matches_data(const bfs::path& index_path, LogData& log) const {
       if (!bfs::exists(index_path))
          return false;
 
@@ -137,12 +135,11 @@ struct log_catalog {
       return pos == log.last_block_position();
    }
 
-   std::optional<uint64_t> get_block_position(uint32_t block_num, mapmode mode = mapmode::readonly) {
+   std::optional<uint64_t> get_block_position(uint32_t block_num) {
       try {
          if (active_index != npos) {
             auto active_item = collection.nth(active_index);
-            if (active_item->first <= block_num && block_num <= active_item->second.last_block_num &&
-                log_data.flags() == mode) {
+            if (active_item->first <= block_num && block_num <= active_item->second.last_block_num) {
                return log_index.nth_block_position(block_num - log_data.first_block_num());
             }
          }
@@ -153,7 +150,7 @@ struct log_catalog {
 
          if (block_num <= it->second.last_block_num) {
             auto name = it->second.filename_base;
-            log_data.open(name.replace_extension("log"), mode);
+            log_data.open(name.replace_extension("log"));
             log_index.open(name.replace_extension("index"));
             active_index = collection.index_of(it);
             return log_index.nth_block_position(block_num - log_data.first_block_num());
@@ -165,12 +162,12 @@ struct log_catalog {
       }
    }
 
-   std::pair<fc::datastream<const char*>, uint32_t> ro_stream_for_block(uint32_t block_num) {
-      auto pos = get_block_position(block_num, mapmode::readonly);
+   fc::datastream<fc::cfile>* ro_stream_for_block(uint32_t block_num) {
+      auto pos = get_block_position(block_num);
       if (pos) {
-         return log_data.ro_stream_at(*pos);
+         return &log_data.ro_stream_at(*pos);
       }
-      return {fc::datastream<const char*>(nullptr, 0), static_cast<uint32_t>(0)};
+      return nullptr;
    }
 
    template <typename ...Rest>
@@ -182,16 +179,8 @@ struct log_catalog {
       return {};
    }
 
-   std::pair<fc::datastream<char*>, uint32_t> rw_stream_for_block(uint32_t block_num) {
-      auto pos = get_block_position(block_num, mapmode::readwrite);
-      if (pos) {
-         return log_data.rw_stream_at(*pos);
-      }
-      return {fc::datastream<char*>(nullptr, 0), static_cast<uint32_t>(0)};
-   }
-
    std::optional<block_id_type> id_for_block(uint32_t block_num) {
-      auto pos = get_block_position(block_num, mapmode::readonly);
+      auto pos = get_block_position(block_num);
       if (pos) {
          return log_data.block_id_at(*pos);
       }

libraries/chain/include/eosio/chain/log_data_base.hpp

@@ -1,5 +1,5 @@
 #pragma once
-#include <boost/iostreams/device/mapped_file.hpp>
+#include <fc/io/cfile.hpp>
 
 namespace eosio {
 namespace chain {
@@ -11,37 +11,44 @@ T read_buffer(const char* buf) {
    return result;
 }
 
+template <typename T>
+T read_data_at(fc::datastream<fc::cfile>& file, std::size_t offset) {
+   file.seek(offset);
+   T value;
+   fc::raw::unpack(file, value);
+   return value;
+}
+
 template <typename Derived>
 class log_data_base {
  protected:
-   boost::iostreams::mapped_file file;
+   fc::datastream<fc::cfile> file;
 
+   Derived* self() { return static_cast<Derived*>(this); }
    const Derived* self() const { return static_cast<const Derived*>(this); }
 
  public:
-   using mapmode = boost::iostreams::mapped_file::mapmode;
 
    log_data_base() = default;
 
    void    close() { file.close(); }
    bool    is_open() const { return file.is_open(); }
-   mapmode flags() const { return file.flags(); }
 
-   const char* data() const { return file.const_data(); }
-   uint64_t    size() const { return file.size(); }
-   uint32_t    last_block_num() const { return self()->block_num_at(last_block_position()); }
-   uint64_t    last_block_position() const {
+   uint64_t    size() const { return self()->size(); }
+
+   uint32_t    last_block_num() { return self()->block_num_at(last_block_position()); }
+   uint64_t    last_block_position() {
       uint32_t offset = sizeof(uint64_t);
       if (self()->is_currently_pruned())
          offset += sizeof(uint32_t);
-      return read_buffer<uint64_t>(data() + size() - offset);
+      return read_data_at<uint64_t>(file, size() - offset);
    }
 
-   uint32_t num_blocks() const {
-      if (self()->first_block_position() == file.size())
+   uint32_t num_blocks() {
+      if (self()->first_block_position() == size())
          return 0;
       else if (self()->is_currently_pruned())
-         return read_buffer<uint32_t>(data() + size() - sizeof(uint32_t));
+         return read_data_at<uint32_t>(file, size() - sizeof(uint32_t));
       return last_block_num() - self()->first_block_num() + 1;
    }
 };