From 27ad70703a71b5b4160b43efeb8dccc7d9d851e5 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 2 Jul 2020 15:46:31 -0500 Subject: [PATCH 1/6] Support state history log splitting --- libraries/chain/block_log.cpp | 514 ++++++------------ libraries/chain/controller.cpp | 4 +- .../chain/include/eosio/chain/block_log.hpp | 9 +- .../include/eosio/chain/block_log_config.hpp | 18 + .../include/eosio/chain/chain_id_type.hpp | 2 +- .../chain/include/eosio/chain/controller.hpp | 9 +- .../chain/include/eosio/chain/log_catalog.hpp | 232 ++++++++ .../include/eosio/chain/log_data_base.hpp | 41 ++ .../chain/include/eosio/chain/log_index.hpp | 38 ++ .../include/eosio/state_history/log.hpp | 118 +++- libraries/state_history/log.cpp | 183 ++++--- .../testing/include/eosio/testing/tester.hpp | 11 +- libraries/testing/tester.cpp | 18 +- plugins/chain_plugin/chain_plugin.cpp | 30 +- .../state_history_plugin.cpp | 61 ++- programs/eosio-blocklog/main.cpp | 14 +- programs/eosio-tester/main.cpp | 2 +- tests/rodeos_test.py | 2 +- unittests/restart_chain_tests.cpp | 233 ++++---- unittests/snapshot_tests.cpp | 22 +- unittests/state_history_tests.cpp | 115 +++- 21 files changed, 1054 insertions(+), 622 deletions(-) create mode 100644 libraries/chain/include/eosio/chain/block_log_config.hpp create mode 100644 libraries/chain/include/eosio/chain/log_catalog.hpp create mode 100644 libraries/chain/include/eosio/chain/log_data_base.hpp create mode 100644 libraries/chain/include/eosio/chain/log_index.hpp diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 4829cfaaef9..1b20aa55a84 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -1,12 +1,11 @@ #include #include +#include +#include +#include +#include #include -#include #include -#include -#include -#include -#include #include namespace eosio { namespace chain { @@ -271,36 +270,30 @@ namespace eosio { namespace chain { return bh.calculate_id(); } - /// Provide the read only view of the blocks.log file - class block_log_data { - boost::iostreams::mapped_file_source file; + /// Provide the memory mapped view of the blocks.log file + class block_log_data : public chain::log_data_base { block_log_preamble preamble; uint64_t first_block_pos = block_log::npos; public: - block_log_data() = default; - block_log_data(const fc::path& path) { open(path); } - - const block_log_preamble& get_preamble() const { return preamble; } - - fc::datastream open(const fc::path& path) { - if (file.is_open()) - file.close(); - file.open(path.generic_string()); - fc::datastream ds(file.data(), file.size()); - preamble.read_from(ds); - first_block_pos = ds.tellp(); - return ds; + + block_log_data() = default; + block_log_data(const fc::path& path, mapmode mode = mapmode::readonly) { open(path, mode); } + + const block_log_preamble& get_preamble() const { return preamble; } + + fc::datastream open(const fc::path& path, mapmode mode = mapmode::readonly) { + if (file.is_open()) + file.close(); + file.open(path.string(), mode); + fc::datastream ds(this->data(), this->size()); + preamble.read_from(ds); + first_block_pos = ds.tellp(); + return ds; } - bool is_open() const { return file.is_open(); } - - const char* data() const { return file.data(); } - uint64_t size() const { return file.size(); } uint32_t version() const { return preamble.version; } uint32_t first_block_num() const { return preamble.first_block_num; } - uint32_t last_block_num() const { return block_num_at(last_block_position()); } uint64_t first_block_position() const { return first_block_pos; } - uint64_t last_block_position() const { return read_buffer(data() + size() - sizeof(uint64_t)); } chain_id_type chain_id() const { return preamble.chain_id(); } fc::optional get_genesis_state() const { @@ -325,14 +318,12 @@ namespace eosio { namespace chain { return fc::endian_reverse_u32(prev_block_num) + 1; } - uint32_t num_blocks() const { - if (first_block_pos == file.size()) - return 0; - return last_block_num() - first_block_num() + 1; + fc::datastream ro_stream_at(uint64_t pos) { + return fc::datastream(file.const_data() + pos, file.size() - pos); } - fc::datastream datastream_at(uint64_t pos) { - return fc::datastream(data() + pos, file.size() - pos); + fc::datastream rw_stream_at(uint64_t pos) { + return fc::datastream(file.data() + pos, file.size() - pos); } /** @@ -393,43 +384,19 @@ namespace eosio { namespace chain { EOS_ASSERT(pos == tmp_pos, block_log_exception, "the block position for block ${num} at the end of a block entry is incorrect", ("num", block_num)); return std::make_tuple(block_num, id); } - }; - - /// Provide the read only view of the blocks.index file - class block_log_index { - boost::iostreams::mapped_file_source file; - - public: - block_log_index() = default; - block_log_index(const fc::path& path) { open(path); } - - void open(const fc::path& path) { - if (file.is_open()) - file.close(); - file.open(path.generic_string()); - EOS_ASSERT(file.size() % sizeof(uint64_t) == 0, block_log_exception, "The size of ${file} is not the multiple of sizeof(uint64_t)", - ("file", path.generic_string())); - } - - bool is_open() const { return file.is_open(); } - - using iterator = const uint64_t*; - iterator begin() const { return reinterpret_cast(file.data()); } - iterator end() const { return reinterpret_cast(file.data() + file.size()); } - /// @pre file.size() > 0 - uint64_t back() const { return *(this->end() - 1); } - int num_blocks() const { return file.size() / sizeof(uint64_t); } - uint64_t nth_block_position(uint32_t n) const { return *(begin() + n); } + void construct_index(const fc::path& index_file_name); }; + using block_log_index = eosio::chain::log_index; + /// Provide the read only view for both blocks.log and blocks.index files - struct block_log_archive { + struct block_log_bundle { fc::path block_file_name, index_file_name; // full pathname for blocks.log and blocks.index block_log_data log_data; block_log_index log_index; - block_log_archive(fc::path block_dir) { + block_log_bundle(fc::path block_dir) { block_file_name = block_dir / "blocks.log"; index_file_name = block_dir / "blocks.index"; @@ -489,164 +456,54 @@ namespace eosio { namespace chain { return reverse_block_position_iterator(t, first_block_position); } - template - void for_each_file_in_dir_matches(const fc::path& dir, const char* pattern, Lambda&& lambda) { - const std::regex my_filter(pattern); - std::smatch what; - boost::filesystem::directory_iterator end_itr; // Default ctor yields past-the-end - for (boost::filesystem::directory_iterator p(dir); p != end_itr; ++p) { - // Skip if not a file - if (!boost::filesystem::is_regular_file(p->status())) - continue; - // skip if it's not match blocks-*-*.log - if (!std::regex_match(p->path().filename().string(), what, my_filter)) - continue; - lambda(p->path()); - } - } - } // namespace + void block_log_data::construct_index(const fc::path& index_file_path) { + std::string index_file_name = index_file_path.generic_string(); + ilog("Will write new blocks.index file ${file}", ("file", index_file_name)); - struct block_log_catalog { - using block_num_t = uint32_t; + const uint32_t num_blocks = this->num_blocks(); - struct mapped_type { - block_num_t last_block_num; - boost::filesystem::path filename_base; - }; - using collection_t = boost::container::flat_map; - using size_type = collection_t::size_type; - static constexpr size_type npos = std::numeric_limits::max(); - - boost::filesystem::path archive_dir; - size_type max_retained_files = 10; - collection_t collection; - size_type active_index = npos; - block_log_data log_data; - block_log_index log_index; - chain_id_type chain_id; - - bool empty() const { return collection.empty(); } - - void open(fc::path block_dir) { - for_each_file_in_dir_matches(block_dir, R"(blocks-\d+-\d+\.log)", [this](boost::filesystem::path path) { - auto log_path = path; - auto index_path = path.replace_extension("index"); - auto path_without_extension = log_path.parent_path() / log_path.stem().string(); - - block_log_data log(log_path); - - if (chain_id.empty()) { - chain_id = log.chain_id(); - } else { - EOS_ASSERT(chain_id == log.chain_id(), block_log_exception, "block log file ${path} has a different chain id", - ("path", log_path.generic_string())); - } + ilog("block log version= ${version}", ("version", this->version())); - // check if index file matches the log file - if (!index_matches_data(index_path, log)) - block_log::construct_index(log_path, index_path); - - auto existing_itr = collection.find(log.first_block_num()); - if (existing_itr != collection.end()) { - if (log.last_block_num() <= existing_itr->second.last_block_num) { - wlog("${log_path} contains the overlapping range with ${existing_path}.log, droping ${log_path} " - "from catelog", - ("log_path", log_path.string())("existing_path", existing_itr->second.filename_base.string())); - return; - } - else { - wlog("${log_path} contains the overlapping range with ${existing_path}.log, droping ${existing_path}.log " - "from catelog", - ("log_path", log_path.string())("existing_path", existing_itr->second.filename_base.string())); - } - } - - collection.insert_or_assign(log.first_block_num(), mapped_type{ log.last_block_num(), path_without_extension }); - }); - } - - bool index_matches_data(const boost::filesystem::path& index_path, const block_log_data& log) const { - if (boost::filesystem::exists(index_path) && - boost::filesystem::file_size(index_path) / sizeof(uint64_t) != log.num_blocks()) { - // make sure the last 8 bytes of index and log matches - - fc::cfile index_file; - index_file.set_file_path(index_path); - index_file.open("r"); - index_file.seek_end(-sizeof(uint64_t)); - uint64_t pos; - index_file.read(reinterpret_cast(&pos), sizeof(pos)); - return pos == log.last_block_position(); - } - return false; + if (num_blocks == 0) { + return; } - bool set_active_item(uint32_t block_num) { - try { - if (active_index != npos) { - auto active_item = collection.nth(active_index); - if (active_item->first <= block_num && block_num <= active_item->second.last_block_num) { - if (!log_index.is_open()) { - log_index.open(active_item->second.filename_base.replace_extension("index")); - } - return true; - } - } - if (collection.empty() || block_num < collection.begin()->first ) - return false; + ilog("first block= ${first} last block= ${last}", + ("first", this->first_block_num())("last", (this->last_block_num()))); - auto it = --collection.upper_bound(block_num); + index_writer index(index_file_path, num_blocks); + uint32_t blocks_found = 0; - if (block_num <= it->second.last_block_num ) { - auto name = it->second.filename_base; - log_data.open(name.replace_extension("log")); - log_index.open(name.replace_extension("index")); - this->active_index = collection.index_of(it); - return true; - } - return false; - } catch (...) { - this->active_index = npos; - return false; - } + for (auto iter = make_reverse_block_position_iterator(*this); + iter.get_value() != block_log::npos && blocks_found < num_blocks; ++iter, ++blocks_found) { + index.write(iter.get_value()); } - std::pair, uint32_t> datastream_for_block(uint32_t block_num) { - auto pos = log_index.nth_block_position(block_num - log_data.first_block_num()); - return std::make_pair(log_data.datastream_at(pos), log_data.get_preamble().version); - } + EOS_ASSERT(blocks_found == num_blocks, block_log_exception, + "Block log file at '${blocks_log}' formatting indicated last block: ${last_block_num}, first " + "block: ${first_block_num}, but found ${num} blocks", + ("blocks_log", index_file_name.replace(index_file_name.size() - 5, 5, "log"))( + "last_block_num", this->last_block_num())("first_block_num", + this->first_block_num())("num", blocks_found)); + } - /// Add a new entry into the catalog. - /// - /// Notice that \c start_block_num must be monotonically increasing between the invocations of this function - /// so that the new entry would be inserted at the end of the flat_map; otherwise, \c active_index would be invalidated - /// and the mapping between the log data their block range would be wrong. This function is only used during - /// the splitting of block log. Using this function for other purpose should make sure if the monotonically - /// increasing block num guarantee can be met. - void add(uint32_t start_block_num, uint32_t end_block_num, fc::path filename_base) { - if (this->collection.size() >= max_retained_files) { - auto items_to_erase = max_retained_files > 0 ? this->collection.size() - max_retained_files + 1 : this->collection.size(); - for (auto it = this->collection.begin(); it < this->collection.begin() + items_to_erase; ++it) { - auto name = it->second.filename_base; - if (archive_dir.empty()) { - // delete the old files when no backup dir is specified - boost::filesystem::remove(name.replace_extension("log")); - boost::filesystem::remove(name.replace_extension("index")); - } else { - // move the the backup dir - auto new_name = archive_dir / boost::filesystem::path(name).filename(); - boost::filesystem::rename(name.replace_extension("log"), new_name.replace_extension("log")); - boost::filesystem::rename(name.replace_extension("index"), new_name.replace_extension("index")); - } - } - this->collection.erase(this->collection.begin(), this->collection.begin() + items_to_erase); - this->active_index = this->active_index == npos || this->active_index < items_to_erase ? npos : this->active_index - items_to_erase; + } // namespace + + struct block_log_verifier { + chain_id_type chain_id; + + void verify(const block_log_data& log, const boost::filesystem::path& log_path) { + if (chain_id.empty()) { + chain_id = log.chain_id(); + } else { + EOS_ASSERT(chain_id == log.chain_id(), block_log_exception, + "block log file ${path} has a different chain id", ("path", log_path.generic_string())); } - if (max_retained_files > 0) - this->collection.emplace(start_block_num, mapped_type{end_block_num, filename_base}); } }; + using block_log_catalog = eosio::chain::log_catalog; + namespace detail { /** @@ -667,7 +524,7 @@ namespace eosio { namespace chain { size_t stride = std::numeric_limits::max(); static uint32_t default_version; - block_log_impl(const fc::path& data_dir, fc::path archive_dir, uint64_t stride, uint16_t max_retained_files, bool fix_irreversible_blocks); + block_log_impl(const block_log::config_type& config); static void ensure_file_exists(fc::cfile& f) { if (fc::exists(f.get_file_path())) @@ -696,9 +553,8 @@ namespace eosio { namespace chain { uint32_t block_log_impl::default_version = block_log::max_supported_version; } // namespace detail - block_log::block_log(const fc::path& data_dir, fc::path archive_dir, uint64_t stride, - uint16_t max_retained_files, bool fix_irreversible_blocks) - : my(new detail::block_log_impl(data_dir, archive_dir, stride, max_retained_files, fix_irreversible_blocks)) {} + block_log::block_log(const block_log::config_type& config) + : my(new detail::block_log_impl(config)) {} block_log::~block_log() {} @@ -706,29 +562,18 @@ namespace eosio { namespace chain { void block_log::set_version(uint32_t ver) { detail::block_log_impl::default_version = ver; } uint32_t block_log::version() const { return my->preamble.version; } - detail::block_log_impl::block_log_impl(const fc::path& data_dir, fc::path archive_dir, uint64_t stride, - uint16_t max_retained_files, bool fix_irreversible_blocks) { - - if (!fc::is_directory(data_dir)) - fc::create_directories(data_dir); - else - catalog.open(data_dir); + detail::block_log_impl::block_log_impl(const block_log::config_type& config) { - if (! archive_dir.empty()) { - if (archive_dir.is_relative()) - archive_dir = data_dir / archive_dir; - - if (!fc::is_directory(archive_dir)) - fc::create_directories(archive_dir); - } + if (!fc::is_directory(config.log_dir)) + fc::create_directories(config.log_dir); + catalog.open(config.log_dir, config.archive_dir, "blocks"); + + catalog.max_retained_files = config.max_retained_files; + this->stride = config.stride; - catalog.archive_dir = archive_dir; - catalog.max_retained_files = max_retained_files; - this->stride = stride; - - block_file.set_file_path(data_dir / "blocks.log"); - index_file.set_file_path(data_dir / "blocks.index"); + block_file.set_file_path(config.log_dir / "blocks.log"); + index_file.set_file_path(config.log_dir / "blocks.index"); /* On startup of the block log, there are several states the log file and the index file can be * in relation to each other. * @@ -757,7 +602,7 @@ namespace eosio { namespace chain { block_log_data log_data(block_file.get_file_path()); preamble = log_data.get_preamble(); - EOS_ASSERT(catalog.chain_id.empty() || catalog.chain_id == preamble.chain_id(), block_log_exception, + EOS_ASSERT(catalog.verifier.chain_id.empty() || catalog.verifier.chain_id == preamble.chain_id(), block_log_exception, "block log file ${path} has a different chain id", ("path", block_file.get_file_path())); genesis_written_to_block_log = true; // Assume it was constructed properly. @@ -768,26 +613,30 @@ namespace eosio { namespace chain { block_log_index index(index_file.get_file_path()); if (log_data.last_block_position() != index.back()) { - if (!fix_irreversible_blocks) { + if (!config.fix_irreversible_blocks) { ilog("The last block positions from blocks.log and blocks.index are different, Reconstructing index..."); - block_log::construct_index(block_file.get_file_path(), index_file.get_file_path()); + log_data.construct_index(index_file.get_file_path()); } else if (!recover_from_incomplete_block_head(log_data, index)) { block_log::repair_log(block_file.get_file_path().parent_path(), UINT32_MAX); block_log::construct_index(block_file.get_file_path(), index_file.get_file_path()); } - } else if (fix_irreversible_blocks) { + } else if (config.fix_irreversible_blocks) { ilog("Irreversible blocks was not corrupted."); } } else { - if (fix_irreversible_blocks) + if (config.fix_irreversible_blocks) { block_log::repair_log(block_file.get_file_path().parent_path(), UINT32_MAX); - block_log::construct_index(block_file.get_file_path(), index_file.get_file_path()); + block_log::construct_index(block_file.get_file_path(), index_file.get_file_path()); + } + else { + log_data.construct_index(index_file.get_file_path()); + } } } else { ilog("Index is empty. Reconstructing index..."); - block_log::construct_index(block_file.get_file_path(), index_file.get_file_path()); + log_data.construct_index(index_file.get_file_path()); } } else if (index_size) { ilog("Log file is empty while the index file is nonempty, discard the index file"); @@ -849,16 +698,8 @@ namespace eosio { namespace chain { void detail::block_log_impl::split_log() { block_file.close(); index_file.close(); - - auto data_dir = block_file.get_file_path().parent_path(); - const int bufsize = 64; - char filename[bufsize]; - int n = snprintf(filename, bufsize, "blocks-%d-%d", preamble.first_block_num, this->head->block_num()); - catalog.add(preamble.first_block_num, this->head->block_num(), data_dir / filename); - strncpy(filename + n, ".log", bufsize-n); - boost::filesystem::rename(block_file.get_file_path(), data_dir / filename); - strncpy(filename + n, ".index", bufsize-n); - boost::filesystem::rename(index_file.get_file_path(), data_dir / filename); + + catalog.add(preamble.first_block_num, this->head->block_num(), block_file.get_file_path().parent_path(), "blocks"); block_file.open(fc::cfile::truncate_rw_mode); index_file.open(fc::cfile::truncate_rw_mode); @@ -898,7 +739,7 @@ namespace eosio { namespace chain { EOS_ASSERT(first_block_num > 1, block_log_exception, "Block log version ${ver} needs to be created with a genesis state if starting from block number 1."); - EOS_ASSERT(my->catalog.chain_id.empty() || chain_id == my->catalog.chain_id, block_log_exception, + EOS_ASSERT(my->catalog.verifier.chain_id.empty() || chain_id == my->catalog.verifier.chain_id, block_log_exception, "Trying to reset to the chain to a different chain id"); my->reset(first_block_num, chain_id); @@ -910,9 +751,10 @@ namespace eosio { namespace chain { if (pos != block_log::npos) { block_file.seek(pos); return read_block(block_file, preamble.version, block_num); - } else if (catalog.set_active_item(block_num)) { - auto [ds, version] = catalog.datastream_for_block(block_num); - return read_block(ds, version, block_num); + } else { + auto [ds, version] = catalog.ro_stream_for_block(block_num); + if (ds.remaining()) + return read_block(ds, version, block_num); } return {}; } @@ -922,9 +764,10 @@ namespace eosio { namespace chain { if (pos != block_log::npos) { block_file.seek(pos); return read_block_id(block_file, preamble.version, block_num); - } else if (catalog.set_active_item(block_num)) { - auto [ds, version] = catalog.datastream_for_block(block_num); - return read_block_id(ds, version, block_num); + } else { + auto [ds, version] = catalog.ro_stream_for_block(block_num); + if (ds.remaining()) + return read_block_id(ds, version, block_num); } return {}; } @@ -973,30 +816,7 @@ namespace eosio { namespace chain { ilog("Will write new blocks.index file ${file}", ("file", index_file_name.generic_string())); block_log_data log_data(block_file_name); - const uint32_t num_blocks = log_data.num_blocks(); - - ilog("block log version= ${version}", ("version", log_data.version())); - - if (num_blocks == 0) { - return; - } - - ilog("first block= ${first} last block= ${last}", - ("first", log_data.first_block_num())("last", (log_data.last_block_num()))); - - index_writer index(index_file_name, num_blocks); - uint32_t blocks_found = 0; - - for (auto iter = make_reverse_block_position_iterator(log_data); - iter.get_value() != npos && blocks_found < num_blocks; ++iter, ++blocks_found) { - index.write(iter.get_value()); - } - - EOS_ASSERT( blocks_found == num_blocks, - block_log_exception, - "Block log file at '${blocks_log}' formatting indicated last block: ${last_block_num}, first block: ${first_block_num}, but found ${num} blocks", - ("blocks_log", block_file_name.generic_string())("last_block_num", log_data.last_block_num())("first_block_num", log_data.first_block_num())("num", blocks_found)); - + log_data.construct_index(index_file_name); } static void write_incomplete_block_data(const fc::path& blocks_dir, fc::time_point now, uint32_t block_num, const char* start, int size) { @@ -1141,55 +961,55 @@ namespace eosio { namespace chain { chain_id_type block_log::extract_chain_id( const fc::path& data_dir ) { return block_log_data(data_dir / "blocks.log").chain_id(); } - - size_t block_log::prune_transactions(uint32_t block_num, std::vector& ids) { - try { - EOS_ASSERT(my->preamble.version >= pruned_transaction_version, block_log_exception, - "The block log version ${version} does not support transaction pruning.", ("version", my->preamble.version)); - const uint64_t pos = my->get_block_pos(block_num); - EOS_ASSERT( pos != npos, block_log_exception, - "Specified block_num ${block_num} does not exist in block log.", ("block_num", block_num) ); + size_t prune_trxs(fc::datastream strm, uint32_t block_num, std::vector& ids, uint32_t version) { - log_entry_v4 entry; - my->block_file.seek(pos); - auto ds = my->block_file.create_datastream(); - unpack(ds, entry); + EOS_ASSERT(version >= pruned_transaction_version, block_log_exception, + "The block log version ${version} does not support transaction pruning.", ("version", version)); - EOS_ASSERT(entry.block.block_num() == block_num, block_log_exception, + auto read_strm = strm; + log_entry_v4 entry; + unpack(read_strm, entry); + + EOS_ASSERT(entry.block.block_num() == block_num, block_log_exception, "Wrong block was read from block log."); - auto pruner = overloaded{[](transaction_id_type&) { return false; }, - [&ids](packed_transaction& ptx) { - auto it = std::find(ids.begin(), ids.end(), ptx.id()); - if (it != ids.end()) { - ptx.prune_all(); - // remove the found entry from ids - ids.erase(it); - return true; - } - return false; - }}; - - size_t num_trx_pruned = 0; - for (auto& trx : entry.block.transactions) { - num_trx_pruned += trx.trx.visit(pruner); - } + auto pruner = overloaded{[](transaction_id_type&) { return false; }, + [&ids](packed_transaction& ptx) { + auto it = std::find(ids.begin(), ids.end(), ptx.id()); + if (it != ids.end()) { + ptx.prune_all(); + // remove the found entry from ids + ids.erase(it); + return true; + } + return false; + }}; + + size_t num_trx_pruned = 0; + for (auto& trx : entry.block.transactions) { + num_trx_pruned += trx.trx.visit(pruner); + } + strm.skip(offset_to_block_start(version)); + entry.block.pack(strm, entry.meta.compression); + return num_trx_pruned; + } - if (num_trx_pruned) { - // we don't want to rewrite entire entry, just the block data itself. - const auto block_offset = offset_to_block_start(my->preamble.version); - my->block_file.seek(pos + block_offset); - const uint32_t max_block_size = entry.meta.size - block_offset - sizeof(uint64_t); - std::vector buffer(max_block_size); - fc::datastream stream(buffer.data(), buffer.size()); - entry.block.pack(stream, entry.meta.compression); - my->block_file.write(buffer.data(), buffer.size()); - my->block_file.flush(); - } - return num_trx_pruned; + size_t block_log::prune_transactions(uint32_t block_num, std::vector& ids) { + + auto [strm, version] = my->catalog.rw_stream_for_block(block_num); + if (strm.remaining()) { + return prune_trxs(strm, block_num, ids, version); } - FC_LOG_AND_RETHROW() + + const uint64_t pos = my->get_block_pos(block_num); + EOS_ASSERT(pos != npos, block_log_exception, "Specified block_num ${block_num} does not exist in block log.", + ("block_num", block_num)); + + using boost::iostreams::mapped_file_sink; + mapped_file_sink sink(my->block_file.get_file_path().string(), mapped_file_sink::max_length, 0); + fc::datastream ds(sink.data() + pos , sink.size() - pos); + return prune_trxs(ds, block_num, ids, my->preamble.version); } bool block_log::contains_genesis_state(uint32_t version, uint32_t first_block_num) { @@ -1210,13 +1030,13 @@ namespace eosio { namespace chain { ilog("In directory ${dir} will trim all blocks before block ${n} from blocks.log and blocks.index.", ("dir", block_dir.generic_string())("n", truncate_at_block)); - block_log_archive archive(block_dir); + block_log_bundle log_bundle(block_dir); - if (truncate_at_block <= archive.log_data.first_block_num()) { + if (truncate_at_block <= log_bundle.log_data.first_block_num()) { dlog("There are no blocks before block ${n} so do nothing.", ("n", truncate_at_block)); return false; } - if (truncate_at_block > archive.log_data.last_block_num()) { + if (truncate_at_block > log_bundle.log_data.last_block_num()) { dlog("All blocks are before block ${n} so do nothing (trim front would delete entire blocks.log).", ("n", truncate_at_block)); return false; } @@ -1229,10 +1049,10 @@ namespace eosio { namespace chain { "Code was written to support format of version 4 or lower, need to update this code for latest format." ); const auto preamble_size = block_log_preamble::nbytes_with_chain_id; - const auto num_blocks_to_truncate = truncate_at_block - archive.log_data.first_block_num(); - const uint64_t first_kept_block_pos = archive.log_index.nth_block_position(num_blocks_to_truncate); + const auto num_blocks_to_truncate = truncate_at_block - log_bundle.log_data.first_block_num(); + const uint64_t first_kept_block_pos = log_bundle.log_index.nth_block_position(num_blocks_to_truncate); const uint64_t nbytes_to_trim = first_kept_block_pos - preamble_size; - const auto new_block_file_size = archive.log_data.size() - nbytes_to_trim; + const auto new_block_file_size = log_bundle.log_data.size() - nbytes_to_trim; boost::iostreams::mapped_file_sink new_block_file; create_mapped_file(new_block_file, new_block_filename.generic_string(), new_block_file_size); @@ -1240,15 +1060,15 @@ namespace eosio { namespace chain { block_log_preamble preamble; // version 4 or above have different log entry format; therefore version 1 to 3 can only be upgrade up to version 3 format. - preamble.version = archive.log_data.version() < pruned_transaction_version ? genesis_state_or_chain_id_version : block_log::max_supported_version; + preamble.version = log_bundle.log_data.version() < pruned_transaction_version ? genesis_state_or_chain_id_version : block_log::max_supported_version; preamble.first_block_num = truncate_at_block; - preamble.chain_context = archive.log_data.chain_id(); + preamble.chain_context = log_bundle.log_data.chain_id(); preamble.write_to(ds); - memcpy(new_block_file.data() + preamble_size, archive.log_data.data() + first_kept_block_pos, new_block_file_size - preamble_size); + memcpy(new_block_file.data() + preamble_size, log_bundle.log_data.data() + first_kept_block_pos, new_block_file_size - preamble_size); fc::path new_index_filename = temp_dir / "blocks.index"; - index_writer index(new_index_filename, archive.log_index.num_blocks() - num_blocks_to_truncate); + index_writer index(new_index_filename, log_bundle.log_index.num_blocks() - num_blocks_to_truncate); // walk along the block position of each block entry and decrement its value by nbytes_to_trim for (auto itr = make_reverse_block_position_iterator(new_block_file, preamble_size); @@ -1262,55 +1082,55 @@ namespace eosio { namespace chain { new_block_file.close(); fc::path old_log = temp_dir / "old.log"; - rename(archive.block_file_name, old_log); - rename(new_block_filename, archive.block_file_name); + rename(log_bundle.block_file_name, old_log); + rename(new_block_filename, log_bundle.block_file_name); fc::path old_ind = temp_dir / "old.index"; - rename(archive.index_file_name, old_ind); - rename(new_index_filename, archive.index_file_name); + rename(log_bundle.index_file_name, old_ind); + rename(new_index_filename, log_bundle.index_file_name); return true; } int block_log::trim_blocklog_end(fc::path block_dir, uint32_t n) { //n is last block to keep (remove later blocks) - block_log_archive archive(block_dir); + block_log_bundle log_bundle(block_dir); ilog("In directory ${block_dir} will trim all blocks after block ${n} from ${block_file} and ${index_file}", - ("block_dir", block_dir.generic_string())("n", n)("block_file",archive.block_file_name.generic_string())("index_file", archive.index_file_name.generic_string())); + ("block_dir", block_dir.generic_string())("n", n)("block_file",log_bundle.block_file_name.generic_string())("index_file", log_bundle.index_file_name.generic_string())); - if (n < archive.log_data.first_block_num()) { + if (n < log_bundle.log_data.first_block_num()) { dlog("All blocks are after block ${n} so do nothing (trim_end would delete entire blocks.log)",("n", n)); return 1; } - if (n > archive.log_data.last_block_num()) { + if (n > log_bundle.log_data.last_block_num()) { dlog("There are no blocks after block ${n} so do nothing",("n", n)); return 2; } - const auto to_trim_block_index = n + 1 - archive.log_data.first_block_num(); - const auto to_trim_block_position = archive.log_index.nth_block_position(to_trim_block_index); + const auto to_trim_block_index = n + 1 - log_bundle.log_data.first_block_num(); + const auto to_trim_block_position = log_bundle.log_index.nth_block_position(to_trim_block_index); const auto index_file_size = to_trim_block_index * sizeof(uint64_t); - boost::filesystem::resize_file(archive.block_file_name, to_trim_block_position); - boost::filesystem::resize_file(archive.index_file_name, index_file_size); + boost::filesystem::resize_file(log_bundle.block_file_name, to_trim_block_position); + boost::filesystem::resize_file(log_bundle.index_file_name, index_file_size); ilog("blocks.index has been trimmed to ${index_file_size} bytes", ("index_file_size", index_file_size)); return 0; } void block_log::smoke_test(fc::path block_dir, uint32_t interval) { - block_log_archive archive(block_dir); + block_log_bundle log_bundle(block_dir); ilog("blocks.log and blocks.index agree on number of blocks"); if (interval == 0) { - interval = std::max((archive.log_index.num_blocks() + 7) >> 3, 1); + interval = std::max((log_bundle.log_index.num_blocks() + 7) >> 3, 1); } - uint32_t expected_block_num = archive.log_data.first_block_num(); + uint32_t expected_block_num = log_bundle.log_data.first_block_num(); - for (auto pos_itr = archive.log_index.begin(); pos_itr < archive.log_index.end(); + for (auto pos_itr = log_bundle.log_index.begin(); pos_itr < log_bundle.log_index.end(); pos_itr += interval, expected_block_num += interval) { - archive.log_data.light_validate_block_entry_at(*pos_itr, expected_block_num); + log_bundle.log_data.light_validate_block_entry_at(*pos_itr, expected_block_num); } } diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 2b615ad5d3b..10e5642cbf4 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -307,10 +307,10 @@ struct controller_impl { db( cfg.state_dir, cfg.read_only ? database::read_only : database::read_write, cfg.state_size, false, cfg.db_map_mode, cfg.db_hugepage_paths ), - reversible_blocks( cfg.blocks_dir/config::reversible_blocks_dir_name, + reversible_blocks( cfg.blog.log_dir/config::reversible_blocks_dir_name, cfg.read_only ? database::read_only : database::read_write, cfg.reversible_cache_size, false, cfg.db_map_mode, cfg.db_hugepage_paths ), - blog( cfg.blocks_dir, cfg.blocks_archive_dir, cfg.blocks_log_stride, cfg.max_retained_block_files, cfg.fix_irreversible_blocks), + blog( cfg.blog ), fork_db( cfg.state_dir ), wasmif( cfg.wasm_runtime, cfg.eosvmoc_tierup, db, cfg.state_dir, cfg.eosvmoc_config ), resource_limits( db, [&s]() { return s.get_deep_mind_logger(); }), diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index cd5d3ed34c7..fe6e6e9b10f 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace eosio { namespace chain { @@ -32,10 +33,14 @@ namespace eosio { namespace chain { * linear scan of the main file. */ + namespace bfs = boost::filesystem; + class block_log { public: - block_log(const fc::path& data_dir, fc::path backup_dir = fc::path(), uint64_t stride=1000, - uint16_t max_retained_files=10, bool fix_irreversible_blocks=false); + + using config_type = block_log_config; + + block_log(const config_type& config); block_log(block_log&& other) = default; ~block_log(); diff --git a/libraries/chain/include/eosio/chain/block_log_config.hpp b/libraries/chain/include/eosio/chain/block_log_config.hpp new file mode 100644 index 00000000000..86f78464c58 --- /dev/null +++ b/libraries/chain/include/eosio/chain/block_log_config.hpp @@ -0,0 +1,18 @@ +#pragma once +#include + +namespace eosio { +namespace chain { + +namespace bfs = boost::filesystem; + +struct block_log_config { + bfs::path log_dir; + bfs::path archive_dir; + uint32_t stride = UINT32_MAX; + uint16_t max_retained_files = 10; + bool fix_irreversible_blocks = false; +}; + +} // namespace chain +} // namespace eosio diff --git a/libraries/chain/include/eosio/chain/chain_id_type.hpp b/libraries/chain/include/eosio/chain/chain_id_type.hpp index fb165a38039..36d85b0657f 100644 --- a/libraries/chain/include/eosio/chain/chain_id_type.hpp +++ b/libraries/chain/include/eosio/chain/chain_id_type.hpp @@ -53,7 +53,7 @@ namespace chain { friend struct eosio::handshake_message; friend class block_log; friend struct block_log_preamble; - friend struct block_log_catalog; + friend struct block_log_verifier; friend class controller; friend struct controller_impl; friend class global_property_object; diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index b3daa922076..17914dd617a 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace chainbase { class database; @@ -65,8 +66,7 @@ namespace eosio { namespace chain { flat_set contract_blacklist; flat_set< pair > action_blacklist; flat_set key_blacklist; - path blocks_dir = chain::config::default_blocks_dir_name; - path blocks_archive_dir = chain::config::default_blocks_archive_dir_name; + block_log_config blog; path state_dir = chain::config::default_state_dir_name; uint64_t state_size = chain::config::default_state_size; uint64_t state_guard_size = chain::config::default_state_guard_size; @@ -74,15 +74,14 @@ namespace eosio { namespace chain { uint64_t reversible_guard_size = chain::config::default_reversible_guard_size; uint32_t sig_cpu_bill_pct = chain::config::default_sig_cpu_bill_pct; uint16_t thread_pool_size = chain::config::default_controller_thread_pool_size; - uint16_t max_retained_block_files = chain::config::default_max_retained_block_files; - uint64_t blocks_log_stride = chain::config::default_blocks_log_stride; + fc::microseconds abi_serializer_max_time_us = fc::microseconds(chain::config::default_abi_serializer_max_time_us); bool read_only = false; bool force_all_checks = false; bool disable_replay_opts = false; bool contracts_console = false; bool allow_ram_billing_in_notify = false; - bool fix_irreversible_blocks = false; + uint32_t maximum_variable_signature_length = chain::config::default_max_variable_signature_length; bool disable_all_subjective_mitigations = false; //< for developer & testing purposes, can be configured using `disable-all-subjective-mitigations` when `EOSIO_DEVELOPER` build option is provided uint32_t terminate_at_block = 0; //< primarily for testing purposes diff --git a/libraries/chain/include/eosio/chain/log_catalog.hpp b/libraries/chain/include/eosio/chain/log_catalog.hpp new file mode 100644 index 00000000000..38b9f00baf8 --- /dev/null +++ b/libraries/chain/include/eosio/chain/log_catalog.hpp @@ -0,0 +1,232 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace eosio { +namespace chain { + +namespace bfs = bfs; + +template +void for_each_file_in_dir_matches(const bfs::path& dir, std::string pattern, Lambda&& lambda) { + const std::regex my_filter(pattern); + std::smatch what; + bfs::directory_iterator end_itr; // Default ctor yields past-the-end + for (bfs::directory_iterator p(dir); p != end_itr; ++p) { + // Skip if not a file + if (!bfs::is_regular_file(p->status())) + continue; + // skip if it's not match blocks-*-*.log + if (!std::regex_match(p->path().filename().string(), what, my_filter)) + continue; + lambda(p->path()); + } +} + +struct null_verifier { + template + void verify(const LogData&, const bfs::path&) {} +}; + +template +struct log_catalog { + using block_num_t = uint32_t; + + struct mapped_type { + block_num_t last_block_num; + bfs::path filename_base; + }; + using collection_t = boost::container::flat_map; + using size_type = typename collection_t::size_type; + static constexpr size_type npos = std::numeric_limits::max(); + + using mapmode = boost::iostreams::mapped_file::mapmode; + + bfs::path archive_dir; + size_type max_retained_files = 10; + collection_t collection; + size_type active_index = npos; + LogData log_data; + LogIndex log_index; + LogVerifier verifier; + + bool empty() const { return collection.empty(); } + + uint32_t first_block_num() const { + if (empty()) + return 0; + return collection.begin()->first; + } + + void open(const bfs::path& log_dir, const bfs::path& archive_dir, const char* name, + const char* suffix_pattern = R"(-\d+-\d+\.log)") { + + if (!archive_dir.empty()) { + if (archive_dir.is_relative()) + this->archive_dir = log_dir / archive_dir; + else + this->archive_dir = archive_dir; + + if (!bfs::is_directory(this->archive_dir)) + bfs::create_directories(this->archive_dir); + } + + for_each_file_in_dir_matches(log_dir, std::string(name) + suffix_pattern, [this](bfs::path path) { + auto log_path = path; + auto index_path = path.replace_extension("index"); + auto path_without_extension = log_path.parent_path() / log_path.stem().string(); + + LogData log(log_path); + + verifier.verify(log, log_path); + + // check if index file matches the log file + if (!index_matches_data(index_path, log)) + log.construct_index(index_path); + + auto existing_itr = collection.find(log.first_block_num()); + if (existing_itr != collection.end()) { + if (log.last_block_num() <= existing_itr->second.last_block_num) { + wlog("${log_path} contains the overlapping range with ${existing_path}.log, droping ${log_path} " + "from catelog", + ("log_path", log_path.string())("existing_path", existing_itr->second.filename_base.string())); + return; + } else { + wlog( + "${log_path} contains the overlapping range with ${existing_path}.log, droping ${existing_path}.log " + "from catelog", + ("log_path", log_path.string())("existing_path", existing_itr->second.filename_base.string())); + } + } + + collection.insert_or_assign(log.first_block_num(), mapped_type{log.last_block_num(), path_without_extension}); + }); + } + + bool index_matches_data(const bfs::path& index_path, const LogData& log) const { + if (bfs::exists(index_path) && bfs::file_size(index_path) / sizeof(uint64_t) != log.num_blocks()) { + // make sure the last 8 bytes of index and log matches + + fc::cfile index_file; + index_file.set_file_path(index_path); + index_file.open("r"); + index_file.seek_end(-sizeof(uint64_t)); + uint64_t pos; + index_file.read(reinterpret_cast(&pos), sizeof(pos)); + return pos == log.last_block_position(); + } + return false; + } + + std::string filebase_for_block(uint32_t block_num) { + if (collection.empty() || block_num < collection.begin()->first) + return ""; + auto it = --collection.upper_bound(block_num); + + if (block_num <= it->second.last_block_num) + return it->second.filename_base; + return ""; + } + + bool set_active_item(uint32_t block_num, mapmode mode = mapmode::readonly) { + try { + if (active_index != npos) { + auto active_item = collection.nth(active_index); + if (active_item->first <= block_num && block_num <= active_item->second.last_block_num && + log_data.flags() == mode) { + return true; + } + } + if (collection.empty() || block_num < collection.begin()->first) + return false; + + auto it = --collection.upper_bound(block_num); + + if (block_num <= it->second.last_block_num) { + auto name = it->second.filename_base; + log_data.open(name.replace_extension("log"), mode); + log_index.open(name.replace_extension("index")); + this->active_index = collection.index_of(it); + return true; + } + return false; + } catch (...) { + this->active_index = npos; + return false; + } + } + + std::pair, uint32_t> ro_stream_for_block(uint32_t block_num) { + if (set_active_item(block_num, mapmode::readonly)) { + auto pos = log_index.nth_block_position(block_num - log_data.first_block_num()); + return std::make_pair(log_data.ro_stream_at(pos), log_data.version()); + } + return {fc::datastream(nullptr, 0), static_cast(0)}; + } + + std::pair, uint32_t> rw_stream_for_block(uint32_t block_num) { + if (set_active_item(block_num, mapmode::readwrite)) { + auto pos = log_index.nth_block_position(block_num - log_data.first_block_num()); + return std::make_pair(log_data.rw_stream_at(pos), log_data.version()); + } + return {fc::datastream(nullptr, 0), static_cast(0)}; + } + + std::optional id_for_block(uint32_t block_num) { + if (set_active_item(block_num, mapmode::readonly)) { + auto pos = log_index.nth_block_position(block_num - log_data.first_block_num()); + return log_data.block_id_at(pos); + } + return {}; + } + + static void rename_bundle(bfs::path orig_path, bfs::path new_path) { + bfs::rename(orig_path.replace_extension(".log"), new_path.replace_extension(".log")); + bfs::rename(orig_path.replace_extension(".index"), new_path.replace_extension(".index")); + } + + /// Add a new entry into the catalog. + /// + /// Notice that \c start_block_num must be monotonically increasing between the invocations of this function + /// so that the new entry would be inserted at the end of the flat_map; otherwise, \c active_index would be + /// invalidated and the mapping between the log data their block range would be wrong. This function is only used + /// during the splitting of block log. Using this function for other purpose should make sure if the monotonically + /// increasing block num guarantee can be met. + void add(uint32_t start_block_num, uint32_t end_block_num, const bfs::path& dir, const char* name) { + + const int bufsize = 64; + char buf[bufsize]; + snprintf(buf, bufsize, "%s-%d-%d", name, start_block_num, end_block_num); + bfs::path new_path = dir / buf; + rename_bundle(dir / name, new_path); + + if (this->collection.size() >= max_retained_files) { + auto items_to_erase = + max_retained_files > 0 ? this->collection.size() - max_retained_files + 1 : this->collection.size(); + for (auto it = this->collection.begin(); it < this->collection.begin() + items_to_erase; ++it) { + auto orig_name = it->second.filename_base; + if (archive_dir.empty()) { + // delete the old files when no backup dir is specified + bfs::remove(orig_name.replace_extension("log")); + bfs::remove(orig_name.replace_extension("index")); + } else { + // move the the archive dir + rename_bundle(orig_name, archive_dir/orig_name.filename() ); + } + } + this->collection.erase(this->collection.begin(), this->collection.begin() + items_to_erase); + this->active_index = this->active_index == npos || this->active_index < items_to_erase + ? npos + : this->active_index - items_to_erase; + } + if (max_retained_files > 0) + this->collection.emplace(start_block_num, mapped_type{end_block_num, new_path}); + } +}; + +} // namespace chain +} // namespace eosio \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/log_data_base.hpp b/libraries/chain/include/eosio/chain/log_data_base.hpp new file mode 100644 index 00000000000..c03d2b6f4ce --- /dev/null +++ b/libraries/chain/include/eosio/chain/log_data_base.hpp @@ -0,0 +1,41 @@ +#pragma once +#include + +namespace eosio { +namespace chain { + +template +T read_buffer(const char* buf) { + T result; + memcpy(&result, buf, sizeof(T)); + return result; +} + +template +class log_data_base { + protected: + boost::iostreams::mapped_file file; + + const Derived* self() const { return static_cast(this); } + + public: + using mapmode = boost::iostreams::mapped_file::mapmode; + + log_data_base() = default; + + bool is_open() const { return file.is_open(); } + mapmode flags() const { return file.flags(); } + + const char* data() const { return file.const_data(); } + uint64_t size() const { return file.size(); } + uint32_t last_block_num() const { return self()->block_num_at(last_block_position()); } + uint64_t last_block_position() const { return read_buffer(data() + size() - sizeof(uint64_t)); } + + uint32_t num_blocks() const { + if (self()->first_block_position() == file.size()) + return 0; + return last_block_num() - self()->first_block_num() + 1; + } +}; +} // namespace chain +} // namespace eosio \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/log_index.hpp b/libraries/chain/include/eosio/chain/log_index.hpp new file mode 100644 index 00000000000..fed17c23164 --- /dev/null +++ b/libraries/chain/include/eosio/chain/log_index.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +namespace eosio { +namespace chain { + +template +class log_index { + boost::iostreams::mapped_file_source file; + + public: + log_index() = default; + log_index(const boost::filesystem::path& path) { open(path); } + + void open(const boost::filesystem::path& path) { + if (file.is_open()) + file.close(); + file.open(path.generic_string()); + EOS_ASSERT(file.size() % sizeof(uint64_t) == 0, Exception, + "The size of ${file} is not the multiple of sizeof(uint64_t)", ("file", path.generic_string())); + } + + bool is_open() const { return file.is_open(); } + + using iterator = const uint64_t*; + iterator begin() const { return reinterpret_cast(file.data()); } + iterator end() const { return reinterpret_cast(file.data() + file.size()); } + + /// @pre file.size() > 0 + uint64_t back() const { return *(this->end() - 1); } + int num_blocks() const { return file.size() / sizeof(uint64_t); } + uint64_t nth_block_position(uint32_t n) const { return *(begin() + n); } +}; + +} // namespace chain +} // namespace eosio diff --git a/libraries/state_history/include/eosio/state_history/log.hpp b/libraries/state_history/include/eosio/state_history/log.hpp index 6139de3065f..936e8083deb 100644 --- a/libraries/state_history/include/eosio/state_history/log.hpp +++ b/libraries/state_history/include/eosio/state_history/log.hpp @@ -4,16 +4,23 @@ #include #include +#include #include #include +#include +#include +#include #include #include +#include +#include #include #include -#include namespace eosio { +namespace bfs = boost::filesystem; + /* * *.log: * +---------+----------------+-----------+------------------+-----+---------+----------------+ @@ -45,66 +52,125 @@ static const int state_history_log_header_serial_size = sizeof(state_history_log sizeof(state_history_log_header::block_id) + sizeof(state_history_log_header::payload_size); +class state_history_log_data : public chain::log_data_base { + std::string filename; + + public: + state_history_log_data() = default; + state_history_log_data(const fc::path& path, mapmode mode = mapmode::readonly) + : filename(path.string()) { + open(path, mode); + } + + void open(const fc::path& path, mapmode mode = mapmode::readonly) { + if (file.is_open()) + file.close(); + file.open(path.string(), mode); + return; + } + + uint32_t version() const { return get_ship_version(chain::read_buffer(file.const_data())); } + uint32_t first_block_num() const { return block_num_at(0); } + uint32_t first_block_position() const { return 0; } + + fc::datastream ro_stream_at(uint64_t pos) const { + return fc::datastream(file.const_data() + pos + sizeof(state_history_log_header), + payload_size_at(pos)); + } + + fc::datastream rw_stream_at(uint64_t pos) const { + return fc::datastream(file.data() + pos + sizeof(state_history_log_header), payload_size_at(pos)); + } + + uint32_t block_num_at(uint64_t position) const { + return fc::endian_reverse_u32( + chain::read_buffer(file.const_data() + position + offsetof(state_history_log_header, block_id))); + } + + chain::block_id_type block_id_at(uint64_t position) const { + return chain::read_buffer(file.const_data() + position + + offsetof(state_history_log_header, block_id)); + } + + uint64_t payload_size_at(uint64_t pos) const; + void construct_index(const fc::path& index_file_name) const; +}; + +struct state_history_config { + bfs::path log_dir; + bfs::path archive_dir; + uint32_t stride = UINT32_MAX; + uint32_t max_retained_files = 10; +}; + class state_history_log { private: - using cfile_stream = fc::datastream; + using cfile_stream = fc::datastream; const char* const name = ""; - std::string log_filename; - std::string index_filename; cfile_stream index; uint32_t _begin_block = 0; uint32_t _end_block = 0; chain::block_id_type last_block_id; uint32_t version = ship_current_version; + uint32_t stride; + protected: - cfile_stream write_log; - cfile_stream read_log; + cfile_stream write_log; + cfile_stream read_log; + + using catalog_t = chain::log_catalog>; + catalog_t catalog; public: // The type aliases below help to make it obvious about the meanings of member function return values. using block_num_type = uint32_t; using version_type = uint32_t; using file_position_type = uint64_t; + using config_type = state_history_config; - state_history_log(const char* const name, std::string log_filename, std::string index_filename); + state_history_log(const char* const name, const state_history_config& conf); - block_num_type begin_block() const { return _begin_block; } + block_num_type begin_block() const { + block_num_type result = catalog.first_block_num(); + return result != 0 ? result : _begin_block; + } block_num_type end_block() const { return _end_block; } - void read_header(state_history_log_header& header, bool assert_version = true); - void write_header(const state_history_log_header& header); - template void write_entry(state_history_log_header& header, const chain::block_id_type& prev_id, F write_payload) { auto start_pos = write_log.tellp(); try { - auto block_num = write_entry_header(header, prev_id); + auto block_num = write_entry_header(header, prev_id); write_payload(write_log); write_entry_position(header, start_pos, block_num); } catch (...) { write_log.close(); - boost::filesystem::resize_file(log_filename, start_pos); + boost::filesystem::resize_file(write_log.get_file_path(), start_pos); write_log.open("rb+"); throw; } } - void get_entry_header(block_num_type block_num, state_history_log_header& header); - chain::block_id_type get_block_id(block_num_type block_num); + std::optional get_block_id(block_num_type block_num); + + protected: + void get_entry_header(block_num_type block_num, state_history_log_header& header); private: + void read_header(state_history_log_header& header, bool assert_version = true); + void write_header(const state_history_log_header& header); bool get_last_block(uint64_t size); void recover_blocks(uint64_t size); - void open_log(); - void open_index(); + void open_log(bfs::path filename); + void open_index(bfs::path filename); file_position_type get_pos(block_num_type block_num); void truncate(block_num_type block_num); + void split_log(); /** * @returns the block num **/ - block_num_type write_entry_header(const state_history_log_header& header, - const chain::block_id_type& prev_id); + block_num_type write_entry_header(const state_history_log_header& header, const chain::block_id_type& prev_id); void write_entry_position(const state_history_log_header& header, file_position_type pos, block_num_type block_num); }; // state_history_log @@ -112,12 +178,12 @@ class state_history_traces_log : public state_history_log { state_history::transaction_trace_cache cache; public: - bool trace_debug_mode = false; - state_history::compression_type compression = state_history::compression_type::zlib; + bool trace_debug_mode = false; + state_history::compression_type compression = state_history::compression_type::zlib; - state_history_traces_log(fc::path state_history_dir); + state_history_traces_log(const state_history_config& conf); - static bool exists(fc::path state_history_dir); + static bool exists(bfs::path state_history_dir); void add_transaction(const chain::transaction_trace_ptr& trace, const chain::packed_transaction_ptr& transaction) { cache.add_transaction(trace, transaction); @@ -125,9 +191,7 @@ class state_history_traces_log : public state_history_log { std::vector get_traces(block_num_type block_num); - void block_start(uint32_t block_num) { - cache.clear(); - } + void block_start(uint32_t block_num) { cache.clear(); } void store(const chainbase::database& db, const chain::block_state_ptr& block_state); @@ -139,7 +203,7 @@ class state_history_traces_log : public state_history_log { class state_history_chain_state_log : public state_history_log { public: - state_history_chain_state_log(fc::path state_history_dir); + state_history_chain_state_log(const state_history_config& conf); chain::bytes get_log_entry(block_num_type block_num); diff --git a/libraries/state_history/log.cpp b/libraries/state_history/log.cpp index 3f6e5ed6ea8..dff6d005c40 100644 --- a/libraries/state_history/log.cpp +++ b/libraries/state_history/log.cpp @@ -1,17 +1,48 @@ #include #include #include -#include #include +#include namespace eosio { -state_history_log::state_history_log(const char* const name, std::string log_filename, std::string index_filename) - : name(name) - , log_filename(std::move(log_filename)) - , index_filename(std::move(index_filename)) { - open_log(); - open_index(); +uint64_t state_history_log_data::payload_size_at(uint64_t pos) const { + EOS_ASSERT(file.size() >= pos + sizeof(state_history_log_header), chain::state_history_exception, + "corrupt ${name}: invalid entry size at at position ${pos}", ("name", filename)("pos", pos)); + + fc::datastream ds(file.const_data() + pos, sizeof(state_history_log_header)); + state_history_log_header header; + fc::raw::unpack(ds, header); + + EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic), chain::state_history_exception, + "corrupt ${name}: invalid header for entry at position ${pos}", ("name", filename)("pos", pos)); + + EOS_ASSERT(file.size() >= pos + sizeof(state_history_log_header) + header.payload_size, + chain::state_history_exception, "corrupt ${name}: invalid payload size for entry at position ${pos}", + ("name", filename)("pos", pos)); + return header.payload_size; +} + +void state_history_log_data::construct_index(const fc::path& index_file_name) const { + fc::cfile index; + index.set_file_path(index_file_name); + index.open("w+b"); + + uint64_t pos = 0; + while (pos < file.size()) { + uint64_t payload_size = payload_size_at(pos); + index.write(reinterpret_cast(&pos), sizeof(pos)); + pos += (sizeof(state_history_log_header) + payload_size + sizeof(uint64_t)); + } +} + +state_history_log::state_history_log(const char* const name, const state_history_config& config) + : name(name) { + catalog.open(config.log_dir, config.archive_dir, name); + catalog.max_retained_files = config.max_retained_files; + this->stride = config.stride; + open_log(config.log_dir / (std::string(name) + ".log")); + open_index(config.log_dir / (std::string(name) + ".index")); } void state_history_log::read_header(state_history_log_header& header, bool assert_version) { @@ -21,14 +52,12 @@ void state_history_log::read_header(state_history_log_header& header, bool asser fc::raw::unpack(ds, header); EOS_ASSERT(!ds.remaining(), chain::state_history_exception, "state_history_log_header_serial_size mismatch"); version = get_ship_version(header.magic); - if (assert_version) + if (assert_version) EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic), chain::state_history_exception, "corrupt ${name}.log (0)", ("name", name)); } -void state_history_log::write_header(const state_history_log_header& header) { - fc::raw::pack(write_log, header); -} +void state_history_log::write_header(const state_history_log_header& header) { fc::raw::pack(write_log, header); } // returns cfile positioned at payload void state_history_log::get_entry_header(state_history_log::block_num_type block_num, @@ -39,10 +68,14 @@ void state_history_log::get_entry_header(state_history_log::block_num_type block read_header(header); } -chain::block_id_type state_history_log::get_block_id(state_history_log::block_num_type block_num) { - state_history_log_header header; - get_entry_header(block_num, header); - return header.block_id; +std::optional state_history_log::get_block_id(state_history_log::block_num_type block_num) { + auto result = catalog.id_for_block(block_num); + if (!result && block_num >= _begin_block && block_num < _end_block){ + state_history_log_header header; + get_entry_header(block_num, header); + return header.block_id; + } + return result; } bool state_history_log::get_last_block(uint64_t size) { @@ -98,12 +131,12 @@ void state_history_log::recover_blocks(uint64_t size) { } } read_log.flush(); - boost::filesystem::resize_file(log_filename, pos); + boost::filesystem::resize_file(read_log.get_file_path(), pos); read_log.flush(); EOS_ASSERT(get_last_block(pos), chain::state_history_exception, "recover ${name}.log failed", ("name", name)); } -void state_history_log::open_log() { +void state_history_log::open_log(bfs::path log_filename) { write_log.set_file_path(log_filename); read_log.set_file_path(log_filename); write_log.open("a+b"); // create file if not exists @@ -130,7 +163,7 @@ void state_history_log::open_log() { } } -void state_history_log::open_index() { +void state_history_log::open_index(bfs::path index_filename) { index.set_file_path(index_filename); index.open("a+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app index.seek_end(0); @@ -138,36 +171,10 @@ void state_history_log::open_index() { return; ilog("Regenerate ${name}.index", ("name", name)); index.close(); - index.open("w+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc - - write_log.seek_end(0); - uint64_t size = write_log.tellp(); - uint64_t pos = 0; - uint32_t num_found = 0; - while (pos < size) { - state_history_log_header header; - EOS_ASSERT(pos + state_history_log_header_serial_size <= size, chain::state_history_exception, - "corrupt ${name}.log (6)", ("name", name)); - read_log.seek(pos); - read_header(header, false); - uint64_t suffix_pos = pos + state_history_log_header_serial_size + header.payload_size; - uint64_t suffix; - EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic) && - suffix_pos + sizeof(suffix) <= size, - chain::state_history_exception, "corrupt ${name}.log (7)", ("name", name)); - read_log.seek(suffix_pos); - read_log.read((char*)&suffix, sizeof(suffix)); - // ilog("block ${b} at ${pos}-${end} suffix=${suffix} file_size=${fs}", - // ("b", header.block_num)("pos", pos)("end", suffix_pos + sizeof(suffix))("suffix", suffix)("fs", size)); - EOS_ASSERT(suffix == pos, chain::state_history_exception, "corrupt ${name}.log (8)", ("name", name)); - index.write((char*)&pos, sizeof(pos)); - pos = suffix_pos + sizeof(suffix); - if (!(++num_found % 10000)) { - printf("%10u blocks found, log pos=%12llu\r", (unsigned)num_found, (unsigned long long)pos); - fflush(stdout); - } - } + state_history_log_data(read_log.get_file_path()).construct_index(index_filename); + index.open("a+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app + index.seek_end(0); } state_history_log::file_position_type state_history_log::get_pos(state_history_log::block_num_type block_num) { @@ -185,16 +192,16 @@ void state_history_log::truncate(state_history_log::block_num_type block_num) { num_removed = _end_block - _begin_block; write_log.seek(0); index.seek(0); - boost::filesystem::resize_file(log_filename, 0); - boost::filesystem::resize_file(index_filename, 0); + boost::filesystem::resize_file(read_log.get_file_path(), 0); + boost::filesystem::resize_file(index.get_file_path(), 0); _begin_block = _end_block = 0; } else { num_removed = _end_block - block_num; uint64_t pos = get_pos(block_num); write_log.seek(0); index.seek(0); - boost::filesystem::resize_file(log_filename, pos); - boost::filesystem::resize_file(index_filename, (block_num - _begin_block) * sizeof(uint64_t)); + boost::filesystem::resize_file(index.get_file_path(), pos); + boost::filesystem::resize_file(read_log.get_file_path(), (block_num - _begin_block) * sizeof(uint64_t)); _end_block = block_num; } write_log.flush(); @@ -202,8 +209,8 @@ void state_history_log::truncate(state_history_log::block_num_type block_num) { ilog("fork or replay: removed ${n} blocks from ${name}.log", ("n", num_removed)("name", name)); } -state_history_log::block_num_type -state_history_log::write_entry_header(const state_history_log_header& header, const chain::block_id_type& prev_id) { +state_history_log::block_num_type state_history_log::write_entry_header(const state_history_log_header& header, + const chain::block_id_type& prev_id) { auto block_num = chain::block_header::num_from_id(header.block_id); EOS_ASSERT(_begin_block == _end_block || block_num <= _end_block, chain::state_history_exception, "missed a block in ${name}.log", ("name", name)); @@ -214,7 +221,7 @@ state_history_log::write_entry_header(const state_history_log_header& header, co ("name", name)); } else { state_history_log_header prev; - get_entry_header( block_num - 1, prev); + get_entry_header(block_num - 1, prev); EOS_ASSERT(prev_id == prev.block_id, chain::state_history_exception, "missed a fork change in ${name}.log", ("name", name)); } @@ -231,7 +238,7 @@ state_history_log::write_entry_header(const state_history_log_header& header, co void state_history_log::write_entry_position(const state_history_log_header& header, state_history_log::file_position_type pos, state_history_log::block_num_type block_num) { - uint64_t end = write_log.tellp(); + uint64_t end = write_log.tellp(); uint64_t payload_start_pos = pos + state_history_log_header_serial_size; uint64_t payload_size = end - payload_start_pos; write_log.write((char*)&pos, sizeof(pos)); @@ -248,13 +255,39 @@ void state_history_log::write_entry_position(const state_history_log_header& write_log.flush(); index.flush(); + + if (block_num % stride == 0) { + split_log(); + } } -state_history_traces_log::state_history_traces_log(fc::path state_history_dir) - : state_history_log("trace_history", (state_history_dir / "trace_history.log").string(), - (state_history_dir / "trace_history.index").string()) {} +void state_history_log::split_log() { + index.close(); + read_log.close(); + write_log.close(); + + catalog.add(_begin_block, _end_block - 1, read_log.get_file_path().parent_path(), name); + + _begin_block = 0; + _end_block = 0; + + write_log.open("w+b"); + read_log.open("rb"); + index.open("w+b"); +} + +state_history_traces_log::state_history_traces_log(const state_history_config& config) + : state_history_log("trace_history", config) {} std::vector state_history_traces_log::get_traces(block_num_type block_num) { + + auto [ds, _] = catalog.ro_stream_for_block(block_num); + if (ds.remaining()) { + std::vector traces; + state_history::trace_converter::unpack(ds, traces); + return traces; + } + if (block_num < begin_block() || block_num >= end_block()) return {}; state_history_log_header header; @@ -266,6 +299,12 @@ std::vector state_history_traces_log::get_trac void state_history_traces_log::prune_transactions(state_history_log::block_num_type block_num, std::vector& ids) { + auto [ds, _] = catalog.rw_stream_for_block(block_num); + if (ds.remaining()) { + state_history::trace_converter::prune_traces(ds, ds.remaining(), ids); + return; + } + if (block_num < begin_block() || block_num >= end_block()) return; state_history_log_header header; @@ -277,8 +316,7 @@ void state_history_traces_log::prune_transactions(state_history_log::block_num_t void state_history_traces_log::store(const chainbase::database& db, const chain::block_state_ptr& block_state) { - state_history_log_header header{.magic = ship_magic(ship_current_version), - .block_id = block_state->id}; + state_history_log_header header{.magic = ship_magic(ship_current_version), .block_id = block_state->id}; auto trace = cache.prepare_traces(block_state); this->write_entry(header, block_state->block->previous, [&](auto& stream) { @@ -286,15 +324,21 @@ void state_history_traces_log::store(const chainbase::database& db, const chain: }); } -bool state_history_traces_log::exists(fc::path state_history_dir) { - return fc::exists(state_history_dir / "trace_history.log") && fc::exists(state_history_dir / "trace_history.index"); +bool state_history_traces_log::exists(bfs::path state_history_dir) { + return bfs::exists(state_history_dir / "trace_history.log") && + bfs::exists(state_history_dir / "trace_history.index"); } -state_history_chain_state_log::state_history_chain_state_log(fc::path state_history_dir) - : state_history_log("chain_state_history", (state_history_dir / "chain_state_history.log").string(), - (state_history_dir / "chain_state_history.index").string()) {} +state_history_chain_state_log::state_history_chain_state_log(const state_history_config& config) + : state_history_log("chain_state_history", config) {} chain::bytes state_history_chain_state_log::get_log_entry(block_num_type block_num) { + + auto [ds, _] = catalog.ro_stream_for_block(block_num); + if (ds.remaining()) { + return state_history::zlib_decompress(ds); + } + if (block_num < begin_block() || block_num >= end_block()) return {}; state_history_log_header header; @@ -308,13 +352,10 @@ void state_history_chain_state_log::store(const chainbase::database& db, const c ilog("Placing initial state in block ${n}", ("n", block_state->block->block_num())); using namespace state_history; - std::vector deltas = create_deltas(db, fresh); - state_history_log_header header{.magic = ship_magic(ship_current_version), - .block_id = block_state->id}; + std::vector deltas = create_deltas(db, fresh); + state_history_log_header header{.magic = ship_magic(ship_current_version), .block_id = block_state->id}; - this->write_entry(header, block_state->block->previous, [&deltas](auto& stream) { - zlib_pack(stream, deltas); - }); + this->write_entry(header, block_state->block->previous, [&deltas](auto& stream) { zlib_pack(stream, deltas); }); } } // namespace eosio diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index 26ccc57cd64..907609a9a82 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -393,7 +393,7 @@ namespace eosio { namespace testing { static std::pair default_config(const fc::temp_directory& tempdir) { controller::config cfg; - cfg.blocks_dir = tempdir.path() / config::default_blocks_dir_name; + cfg.blog.log_dir = tempdir.path() / config::default_blocks_dir_name; cfg.state_dir = tempdir.path() / config::default_state_dir_name; cfg.state_size = 1024*1024*16; cfg.state_guard_size = 0; @@ -482,6 +482,9 @@ namespace eosio { namespace testing { } } + tester(const std::function& control_setup, setup_policy policy = setup_policy::full, + db_read_mode read_mode = db_read_mode::SPECULATIVE); + using base_tester::produce_block; signed_block_ptr produce_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override { @@ -532,11 +535,11 @@ namespace eosio { namespace testing { } static void config_validator(controller::config& vcfg) { - FC_ASSERT( vcfg.blocks_dir.filename().generic_string() != "." + FC_ASSERT( vcfg.blog.log_dir.filename().generic_string() != "." && vcfg.state_dir.filename().generic_string() != ".", "invalid path names in controller::config" ); - vcfg.blocks_dir = vcfg.blocks_dir.parent_path() / std::string("v_").append( vcfg.blocks_dir.filename().generic_string() ); - vcfg.state_dir = vcfg.state_dir.parent_path() / std::string("v_").append( vcfg.state_dir.filename().generic_string() ); + vcfg.blog.log_dir = vcfg.blog.log_dir.parent_path() / std::string("v_").append( vcfg.blog.log_dir.filename().generic_string() ); + vcfg.state_dir = vcfg.state_dir.parent_path() / std::string("v_").append( vcfg.state_dir.filename().generic_string() ); vcfg.contracts_console = false; } diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index cdd7636ac56..d9031658632 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -273,8 +273,8 @@ namespace eosio { namespace testing { if( !expected_chain_id ) { expected_chain_id = controller::extract_chain_id_from_db( cfg.state_dir ); if( !expected_chain_id ) { - if( fc::is_regular_file( cfg.blocks_dir / "blocks.log" ) ) { - expected_chain_id = block_log::extract_chain_id( cfg.blocks_dir ); + if( fc::is_regular_file( cfg.blog.log_dir / "blocks.log" ) ) { + expected_chain_id = block_log::extract_chain_id( cfg.blog.log_dir ); } else { expected_chain_id = genesis_state().compute_chain_id(); } @@ -1189,6 +1189,20 @@ namespace eosio { namespace testing { preactivate_protocol_features( preactivations ); } + tester::tester(const std::function& control_setup, setup_policy policy, db_read_mode read_mode) { + auto def_conf = default_config(tempdir); + def_conf.first.read_mode = read_mode; + cfg = def_conf.first; + + base_tester::open(make_protocol_feature_set(), def_conf.second.compute_chain_id(), + [&genesis = def_conf.second, &control = this->control, &control_setup]() { + control_setup(*control); + control->startup([]() {}, []() { return false; }, genesis); + }); + + execute_setup_policy(policy); + } + bool fc_exception_code_is::operator()( const fc::exception& ex ) { bool match = (ex.code() == expected); if( !match ) { diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index ca66643e9a5..edbce800703 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -228,7 +228,7 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip ("blocks-dir", bpo::value()->default_value("blocks"), "the location of the blocks directory (absolute path or relative to application data dir)") ("blocks-log-stride", bpo::value()->default_value(config::default_blocks_log_stride), - "split the block log file when the head block number is the multiple of the split factor\n" + "split the block log file when the head block number is the multiple of the stride\n" "When the stride is reached, the current block log and index will be renamed 'blocks--.log/index'\n" "and a new current block log and index will be created with the most recent block. All files following\n" "this format will be used to construct an extended block log.") @@ -633,13 +633,13 @@ chain_plugin::do_hard_replay(const variables_map& options) { // Do not try to recover reversible blocks if the directory does not exist, unless the option was explicitly provided. if( !recover_reversible_blocks( backup_dir / config::reversible_blocks_dir_name, my->chain_config->reversible_cache_size, - my->chain_config->blocks_dir / config::reversible_blocks_dir_name, + my->chain_config->blog.log_dir / config::reversible_blocks_dir_name, options.at( "truncate-at-block" ).as())) { ilog( "Reversible blocks database was not corrupted. Copying from backup to blocks directory." ); fc::copy( backup_dir / config::reversible_blocks_dir_name, - my->chain_config->blocks_dir / config::reversible_blocks_dir_name ); + my->chain_config->blog.log_dir / config::reversible_blocks_dir_name ); fc::copy( backup_dir / config::reversible_blocks_dir_name / "shared_memory.bin", - my->chain_config->blocks_dir / config::reversible_blocks_dir_name / "shared_memory.bin" ); + my->chain_config->blog.log_dir / config::reversible_blocks_dir_name / "shared_memory.bin" ); } } } @@ -754,16 +754,16 @@ void chain_plugin::plugin_initialize(const variables_map& options) { my->chain_config->abi_serializer_max_time_us = my->abi_serializer_max_time_us; } - my->chain_config->blocks_dir = my->blocks_dir; + my->chain_config->blog.log_dir = my->blocks_dir; my->chain_config->state_dir = app().data_dir() / config::default_state_dir_name; my->chain_config->read_only = my->readonly; - my->chain_config->blocks_archive_dir = options.at("blocks-archive-dir").as(); - my->chain_config->blocks_log_stride = options.at("blocks-log-stride").as(); - my->chain_config->max_retained_block_files = options.at("max-retained-block-files").as(); - my->chain_config->fix_irreversible_blocks = options.at("fix-irreversible-blocks").as(); + my->chain_config->blog.archive_dir = options.at("blocks-archive-dir").as(); + my->chain_config->blog.stride = options.at("blocks-log-stride").as(); + my->chain_config->blog.max_retained_files = options.at("max-retained-block-files").as(); + my->chain_config->blog.fix_irreversible_blocks = options.at("fix-irreversible-blocks").as(); if (auto resmon_plugin = app().find_plugin()) { - resmon_plugin->monitor_directory(my->chain_config->blocks_dir); + resmon_plugin->monitor_directory(my->chain_config->blog.log_dir); resmon_plugin->monitor_directory(my->chain_config->state_dir); } @@ -854,7 +854,7 @@ void chain_plugin::plugin_initialize(const variables_map& options) { p = bfs::current_path() / p; } - if( export_reversible_blocks( my->chain_config->blocks_dir/config::reversible_blocks_dir_name, p ) ) + if( export_reversible_blocks( my->chain_config->blog.log_dir/config::reversible_blocks_dir_name, p ) ) ilog( "Saved all blocks from reversible block database into '${path}'", ("path", p.generic_string()) ); else ilog( "Saved recovered blocks from reversible block database into '${path}'", ("path", p.generic_string()) ); @@ -876,13 +876,13 @@ void chain_plugin::plugin_initialize(const variables_map& options) { wlog( "The --truncate-at-block option does not work for a regular replay of the blockchain." ); clear_chainbase_files( my->chain_config->state_dir ); if( options.at( "fix-reversible-blocks" ).as()) { - if( !recover_reversible_blocks( my->chain_config->blocks_dir / config::reversible_blocks_dir_name, + if( !recover_reversible_blocks( my->chain_config->blog.log_dir / config::reversible_blocks_dir_name, my->chain_config->reversible_cache_size )) { ilog( "Reversible blocks database was not corrupted." ); } } } else if( options.at( "fix-reversible-blocks" ).as()) { - if( !recover_reversible_blocks( my->chain_config->blocks_dir / config::reversible_blocks_dir_name, + if( !recover_reversible_blocks( my->chain_config->blog.log_dir / config::reversible_blocks_dir_name, my->chain_config->reversible_cache_size, optional(), options.at( "truncate-at-block" ).as())) { @@ -896,9 +896,9 @@ void chain_plugin::plugin_initialize(const variables_map& options) { } else if( options.count("import-reversible-blocks") ) { auto reversible_blocks_file = options.at("import-reversible-blocks").as(); ilog("Importing reversible blocks from '${file}'", ("file", reversible_blocks_file.generic_string()) ); - fc::remove_all( my->chain_config->blocks_dir/config::reversible_blocks_dir_name ); + fc::remove_all( my->chain_config->blog.log_dir/config::reversible_blocks_dir_name ); - import_reversible_blocks( my->chain_config->blocks_dir/config::reversible_blocks_dir_name, + import_reversible_blocks( my->chain_config->blog.log_dir/config::reversible_blocks_dir_name, my->chain_config->reversible_cache_size, reversible_blocks_file ); EOS_THROW( node_management_success, "imported reversible blocks" ); diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index f3bc50e9d69..f8de60ff7c3 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -60,16 +60,22 @@ struct state_history_plugin_impl : std::enable_shared_from_this get_block_id(uint32_t block_num) { - if (trace_log && block_num >= trace_log->begin_block() && block_num < trace_log->end_block()) - return trace_log->get_block_id(block_num); - if (chain_state_log && block_num >= chain_state_log->begin_block() && block_num < chain_state_log->end_block()) - return chain_state_log->get_block_id(block_num); - try { - return chain_plug->chain().get_block_id_for_num(block_num); - } catch (...) { + std::optional get_block_id(uint32_t block_num) { + std::optional result; + + if (trace_log) + result = trace_log->get_block_id(block_num); + + if (!result && chain_state_log) + result = chain_state_log->get_block_id(block_num); + + if (!result) { + try { + return chain_plug->chain().get_block_id_for_num(block_num); + } catch (...) { + } } - return {}; + return result; } struct session : std::enable_shared_from_this { @@ -374,6 +380,19 @@ void state_history_plugin::set_program_options(options_description& cli, options auto options = cfg.add_options(); options("state-history-dir", bpo::value()->default_value("state-history"), "the location of the state-history directory (absolute path or relative to application data dir)"); + options("state-history-archive-dir", bpo::value()->default_value("archive"), + "the location of the state history archive directory (absolute path or relative to state-history dir).\n" + "If the value is empty, blocks files beyond the retained limit will be deleted.\n" + "All files in the archive directory are completely under user's control, i.e. they won't be accessed by nodeos anymore."); + options("state-history-stride", bpo::value()->default_value(UINT32_MAX), + "split the state history log file when the block number is the multiple of the stride\n" + "When the stride is reached, the current history log and index will be renamed '*-history--.log/index'\n" + "and a new current history log and index will be created with the most recent blocks. All files following\n" + "this format will be used to construct an extended history log."); + options("max-retained-history-files", bpo::value()->default_value(10), + "the maximum number of history file groups to retain so that the blocks in those files can be queried.\n" + "When the number is reached, the oldest history file would be moved to archive dir or deleted if the archive dir is empty.\n" + "The retained history log files should not be manipulated by users." ); cli.add_options()("delete-state-history", bpo::bool_switch()->default_value(false), "clear state history files"); options("trace-history", bpo::bool_switch()->default_value(false), "enable trace history"); options("chain-state-history", bpo::bool_switch()->default_value(false), "enable chain state history"); @@ -403,14 +422,20 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { my->block_start_connection.emplace( chain.block_start.connect([&](uint32_t block_num) { my->on_block_start(block_num); })); - auto dir_option = options.at("state-history-dir").as(); - boost::filesystem::path state_history_dir; + auto dir_option = options.at("state-history-dir").as(); + + static eosio::state_history_config config; + if (dir_option.is_relative()) - state_history_dir = app().data_dir() / dir_option; + config.log_dir = app().data_dir() / dir_option; else - state_history_dir = dir_option; + config.log_dir = dir_option; if (auto resmon_plugin = app().find_plugin()) - resmon_plugin->monitor_directory(state_history_dir); + resmon_plugin->monitor_directory(config.log_dir); + + config.archive_dir = options.at("state-history-archive-dir").as(); + config.stride = options.at("state-history-stride").as(); + config.max_retained_files = options.at("max-retained-history-files").as(); auto ip_port = options.at("state-history-endpoint").as(); auto port = ip_port.substr(ip_port.find(':') + 1, ip_port.size()); @@ -421,12 +446,12 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { if (options.at("delete-state-history").as()) { fc_ilog(_log, "Deleting state history"); - boost::filesystem::remove_all(state_history_dir); + boost::filesystem::remove_all(config.log_dir); } - boost::filesystem::create_directories(state_history_dir); + boost::filesystem::create_directories(config.log_dir); if (options.at("trace-history").as()) { - my->trace_log.emplace(state_history_dir); + my->trace_log.emplace(config); if (options.at("trace-history-debug-mode").as()) my->trace_log->trace_debug_mode = true; @@ -441,7 +466,7 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { } if (options.at("chain-state-history").as()) - my->chain_state_log.emplace(state_history_dir); + my->chain_state_log.emplace(config); } FC_LOG_AND_RETHROW() } // state_history_plugin::plugin_initialize diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp index 229e3b30518..0b0077e7092 100644 --- a/programs/eosio-blocklog/main.cpp +++ b/programs/eosio-blocklog/main.cpp @@ -71,7 +71,8 @@ struct report_time { void blocklog::read_log() { report_time rt("reading log"); - block_log block_logger(blocks_dir); + + block_log block_logger({ .log_dir = blocks_dir }); const auto end = block_logger.head(); EOS_ASSERT( end, block_log_exception, "No blocks found in block log" ); EOS_ASSERT( end->block_num() > 1, block_log_exception, "Only one block found in block log" ); @@ -238,10 +239,11 @@ bool trim_blocklog_front(bfs::path block_dir, uint32_t n) { //n is first } void fix_irreversible_blocks(bfs::path block_dir) { - std::cout << "\nfix_irreversible_blocks of blocks.log and blocks.index in directory " << block_dir << '\n'; - block_log block_logger(block_dir, fc::path(), 0, 0, true); - std::cout << "\nSmoke test of blocks.log and blocks.index in directory " << block_dir << '\n'; - block_log::smoke_test(block_dir, 0); + std::cout << "\nfix_irreversible_blocks of blocks.log and blocks.index in directory " << block_dir << '\n'; + block_log block_logger({ .log_dir = block_dir, .fix_irreversible_blocks = true }); + + std::cout << "\nSmoke test of blocks.log and blocks.index in directory " << block_dir << '\n'; + block_log::smoke_test(block_dir, 0); } void smoke_test(bfs::path block_dir) { @@ -256,7 +258,7 @@ int prune_transactions(const char* type, bfs::path dir, uint32_t block_num, std::vector unpruned_ids) { using namespace std; if (Log::exists(dir)) { - Log log(dir); + Log log( { .log_dir = dir }); log.prune_transactions(block_num, unpruned_ids); if (unpruned_ids.size()) { cerr << "block " << block_num << " in " << type << " does not contain the following transactions: "; diff --git a/programs/eosio-tester/main.cpp b/programs/eosio-tester/main.cpp index d7d0e1d2e73..e15f3ae6f8c 100644 --- a/programs/eosio-tester/main.cpp +++ b/programs/eosio-tester/main.cpp @@ -195,7 +195,7 @@ struct test_chain { eosio::chain::genesis_state genesis; genesis.initial_timestamp = fc::time_point::from_iso_string("2020-01-01T00:00:00.000"); cfg = std::make_unique(); - cfg->blocks_dir = dir.path() / "blocks"; + cfg->blog.log_dir = dir.path() / "blocks"; cfg->state_dir = dir.path() / "state"; cfg->contracts_console = true; cfg->wasm_runtime = eosio::chain::wasm_interface::vm_type::eos_vm_jit; diff --git a/tests/rodeos_test.py b/tests/rodeos_test.py index 02e3cb1c714..b4a79b8c911 100755 --- a/tests/rodeos_test.py +++ b/tests/rodeos_test.py @@ -154,7 +154,7 @@ def get_info(self): useBiosBootFile=False, loadSystemContract=False, specificExtraNodeosArgs={ - 0: ("--plugin eosio::state_history_plugin --trace-history --chain-state-history --disable-replay-opts " + 0: ("--plugin eosio::state_history_plugin --trace-history --chain-state-history --disable-replay-opts --state-history-stride 20 --max-retained-history-files 3 " "--state-history-endpoint {} --plugin eosio::net_api_plugin --wasm-runtime eos-vm-jit -l logging.json").format(stateHistoryEndpoint)}) producerNodeIndex = 0 diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index 55b49ef920d..6349efcb149 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -19,9 +19,9 @@ using namespace testing; using namespace chain; void remove_existing_blocks(controller::config& config) { - auto block_log_path = config.blocks_dir / "blocks.log"; + auto block_log_path = config.blog.log_dir / "blocks.log"; remove(block_log_path); - auto block_index_path = config.blocks_dir / "blocks.index"; + auto block_index_path = config.blog.log_dir / "blocks.index"; remove(block_index_path); } @@ -81,8 +81,8 @@ struct restart_from_block_log_test_fixture { } ~restart_from_block_log_test_fixture() { controller::config copied_config = chain.get_config(); - copied_config.fix_irreversible_blocks = this->fix_irreversible_blocks; - auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blocks_dir); + copied_config.blog.fix_irreversible_blocks = this->fix_irreversible_blocks; + auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blog.log_dir); BOOST_REQUIRE(genesis); // remove the state files to make sure we are starting from block log @@ -94,68 +94,84 @@ struct restart_from_block_log_test_fixture { } }; -struct light_validation_restart_from_block_log_test_fixture { - tester chain; + + +void light_validation_restart_from_block_log_test_case(bool do_prune, uint32_t stride) { + + fc::temp_directory temp_dir; + auto [ config, gen] = tester::default_config(temp_dir); + config.read_mode = db_read_mode::SPECULATIVE; + config.blog.stride = stride; + tester chain(config, gen); + chain.execute_setup_policy(setup_policy::full); + eosio::chain::transaction_trace_ptr trace; - light_validation_restart_from_block_log_test_fixture() - : chain(setup_policy::full) { - deploy_test_api(chain); - trace = push_test_cfd_transaction(chain); - chain.produce_blocks(10); + deploy_test_api(chain); + chain.produce_blocks(10); + trace = push_test_cfd_transaction(chain); + chain.produce_blocks(10); - BOOST_REQUIRE(trace->receipt); - BOOST_CHECK_EQUAL(trace->receipt->status, transaction_receipt::executed); - BOOST_CHECK_EQUAL(2, trace->action_traces.size()); + BOOST_REQUIRE(trace->receipt); + BOOST_CHECK_EQUAL(trace->receipt->status, transaction_receipt::executed); + BOOST_CHECK_EQUAL(2, trace->action_traces.size()); - BOOST_CHECK(trace->action_traces.at(0).context_free); // cfa - BOOST_CHECK_EQUAL("test\n", trace->action_traces.at(0).console); // cfa executed + BOOST_CHECK(trace->action_traces.at(0).context_free); // cfa + BOOST_CHECK_EQUAL("test\n", trace->action_traces.at(0).console); // cfa executed - BOOST_CHECK(!trace->action_traces.at(1).context_free); // non-cfa - BOOST_CHECK_EQUAL("", trace->action_traces.at(1).console); + BOOST_CHECK(!trace->action_traces.at(1).context_free); // non-cfa + BOOST_CHECK_EQUAL("", trace->action_traces.at(1).console); - chain.close(); + chain.close(); + + const auto& blocks_dir = chain.get_config().blog.log_dir; + + if (do_prune) { + block_log blog(chain.get_config().blog); + std::vector ids{trace->id}; + BOOST_CHECK(blog.prune_transactions(trace->block_num, ids) == 1); + BOOST_CHECK(ids.empty()); + BOOST_REQUIRE_NO_THROW(block_log::repair_log(blocks_dir)); } - ~light_validation_restart_from_block_log_test_fixture() { - controller::config copied_config = chain.get_config(); - auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blocks_dir); - BOOST_REQUIRE(genesis); + controller::config copied_config = chain.get_config(); + auto genesis = chain::block_log::extract_genesis_state(blocks_dir); + BOOST_REQUIRE(genesis); - // remove the state files to make sure we are starting from block log - remove_existing_states(copied_config); + // remove the state files to make sure we are starting from block log + remove_existing_states(copied_config); + + transaction_trace_ptr other_trace; + + replay_tester from_block_log_chain(copied_config, *genesis, + [&](std::tuple x) { + auto& t = std::get<0>(x); + if (t && t->id == trace->id) { + other_trace = t; + } + }); + + BOOST_REQUIRE(other_trace); + BOOST_REQUIRE(other_trace->receipt); + BOOST_CHECK_EQUAL(other_trace->receipt->status, transaction_receipt::executed); + BOOST_CHECK(*trace->receipt == *other_trace->receipt); + BOOST_CHECK_EQUAL(2, other_trace->action_traces.size()); + + BOOST_CHECK(other_trace->action_traces.at(0).context_free); // cfa + BOOST_CHECK_EQUAL("", other_trace->action_traces.at(0).console); // cfa not executed for replay + BOOST_CHECK_EQUAL(trace->action_traces.at(0).receipt->global_sequence, + other_trace->action_traces.at(0).receipt->global_sequence); + BOOST_CHECK_EQUAL(trace->action_traces.at(0).receipt->digest(), + other_trace->action_traces.at(0).receipt->digest()); + + BOOST_CHECK(!other_trace->action_traces.at(1).context_free); // non-cfa + BOOST_CHECK_EQUAL("", other_trace->action_traces.at(1).console); + BOOST_CHECK_EQUAL(trace->action_traces.at(1).receipt->global_sequence, + other_trace->action_traces.at(1).receipt->global_sequence); + BOOST_CHECK_EQUAL(trace->action_traces.at(1).receipt->digest(), + other_trace->action_traces.at(1).receipt->digest()); +} - transaction_trace_ptr other_trace; - - replay_tester from_block_log_chain(copied_config, *genesis, - [&](std::tuple x) { - auto& t = std::get<0>(x); - if (t && t->id == trace->id) { - other_trace = t; - } - }); - - BOOST_REQUIRE(other_trace); - BOOST_REQUIRE(other_trace->receipt); - BOOST_CHECK_EQUAL(other_trace->receipt->status, transaction_receipt::executed); - BOOST_CHECK(*trace->receipt == *other_trace->receipt); - BOOST_CHECK_EQUAL(2, other_trace->action_traces.size()); - - BOOST_CHECK(other_trace->action_traces.at(0).context_free); // cfa - BOOST_CHECK_EQUAL("", other_trace->action_traces.at(0).console); // cfa not executed for replay - BOOST_CHECK_EQUAL(trace->action_traces.at(0).receipt->global_sequence, - other_trace->action_traces.at(0).receipt->global_sequence); - BOOST_CHECK_EQUAL(trace->action_traces.at(0).receipt->digest(), - other_trace->action_traces.at(0).receipt->digest()); - - BOOST_CHECK(!other_trace->action_traces.at(1).context_free); // non-cfa - BOOST_CHECK_EQUAL("", other_trace->action_traces.at(1).console); - BOOST_CHECK_EQUAL(trace->action_traces.at(1).receipt->global_sequence, - other_trace->action_traces.at(1).receipt->global_sequence); - BOOST_CHECK_EQUAL(trace->action_traces.at(1).receipt->digest(), - other_trace->action_traces.at(1).receipt->digest()); - } -}; BOOST_AUTO_TEST_SUITE(restart_chain_tests) @@ -213,29 +229,35 @@ BOOST_AUTO_TEST_CASE(test_restart_with_different_chain_id) { } BOOST_FIXTURE_TEST_CASE(test_restart_from_block_log, restart_from_block_log_test_fixture) { - BOOST_REQUIRE_NO_THROW(block_log::smoke_test(chain.get_config().blocks_dir, 1)); + BOOST_REQUIRE_NO_THROW(block_log::smoke_test(chain.get_config().blog.log_dir, 1)); } BOOST_FIXTURE_TEST_CASE(test_restart_from_trimmed_block_log, restart_from_block_log_test_fixture) { auto& config = chain.get_config(); - auto blocks_path = config.blocks_dir; + auto blocks_path = config.blog.log_dir; remove_all(blocks_path/"reversible"); - BOOST_REQUIRE_NO_THROW(block_log::trim_blocklog_end(config.blocks_dir, cutoff_block_num)); - block_log log(chain.get_config().blocks_dir); + BOOST_REQUIRE_NO_THROW(block_log::trim_blocklog_end(config.blog.log_dir, cutoff_block_num)); + block_log log(chain.get_config().blog); BOOST_CHECK(log.head()->block_num() == cutoff_block_num); BOOST_CHECK(fc::file_size(blocks_path / "blocks.index") == cutoff_block_num * sizeof(uint64_t)); } -BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log, light_validation_restart_from_block_log_test_fixture) { +BOOST_AUTO_TEST_CASE(test_light_validation_restart_from_block_log) { + bool do_prune = false; + uint32_t blocks_log_stride = UINT32_MAX; + light_validation_restart_from_block_log_test_case(do_prune, blocks_log_stride); } -BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log_with_pruned_trx, light_validation_restart_from_block_log_test_fixture) { - const auto& blocks_dir = chain.get_config().blocks_dir; - block_log blog(blocks_dir); - std::vector ids{trace->id}; - BOOST_CHECK(blog.prune_transactions(trace->block_num, ids) == 1); - BOOST_CHECK(ids.empty()); - BOOST_REQUIRE_NO_THROW(block_log::repair_log(blocks_dir)); +BOOST_AUTO_TEST_CASE(test_light_validation_restart_from_block_log_with_pruned_trx) { + bool do_prune = true; + uint32_t blocks_log_stride = UINT32_MAX; + light_validation_restart_from_block_log_test_case(do_prune, blocks_log_stride); +} + +BOOST_AUTO_TEST_CASE(test_light_validation_restart_from_block_log_with_pruned_trx_and_split_log) { + bool do_prune = true; + uint32_t blocks_log_stride = 10; + light_validation_restart_from_block_log_test_case(do_prune, blocks_log_stride); } BOOST_AUTO_TEST_CASE(test_split_log) { @@ -245,14 +267,15 @@ BOOST_AUTO_TEST_CASE(test_split_log) { tester chain( temp_dir, [](controller::config& config) { - config.blocks_log_stride = 20; - config.max_retained_block_files = 5; + config.blog.archive_dir = "archive"; + config.blog.stride = 20; + config.blog.max_retained_files = 5; }, true); chain.produce_blocks(150); - auto blocks_dir = chain.get_config().blocks_dir; - auto blocks_archive_dir = chain.get_config().blocks_dir / chain.get_config().blocks_archive_dir; + auto blocks_dir = chain.get_config().blog.log_dir; + auto blocks_archive_dir = blocks_dir / chain.get_config().blog.archive_dir; BOOST_CHECK(bfs::exists( blocks_archive_dir / "blocks-1-20.log" )); BOOST_CHECK(bfs::exists( blocks_archive_dir / "blocks-1-20.index" )); @@ -297,15 +320,15 @@ BOOST_AUTO_TEST_CASE(test_split_log_no_archive) { tester chain( temp_dir, [](controller::config& config) { - config.blocks_archive_dir = ""; - config.blocks_log_stride = 10; - config.max_retained_block_files = 5; + config.blog.archive_dir = ""; + config.blog.stride = 10; + config.blog.max_retained_files = 5; }, true); chain.produce_blocks(75); - auto blocks_dir = chain.get_config().blocks_dir; - auto blocks_archive_dir = chain.get_config().blocks_dir / chain.get_config().blocks_archive_dir; + auto blocks_dir = chain.get_config().blog.log_dir; + auto blocks_archive_dir = chain.get_config().blog.log_dir / chain.get_config().blog.archive_dir; BOOST_CHECK(!bfs::exists( blocks_archive_dir / "blocks-1-10.log" )); BOOST_CHECK(!bfs::exists( blocks_archive_dir / "blocks-1-10.index" )); @@ -341,14 +364,14 @@ void split_log_replay(uint32_t replay_max_retained_block_files) { tester chain( temp_dir, [](controller::config& config) { - config.blocks_log_stride = stride; - config.max_retained_block_files = 10; + config.blog.stride = stride; + config.blog.max_retained_files = 10; }, true); chain.produce_blocks(150); controller::config copied_config = chain.get_config(); - auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blocks_dir); + auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blog.log_dir); BOOST_REQUIRE(genesis); chain.close(); @@ -356,9 +379,9 @@ void split_log_replay(uint32_t replay_max_retained_block_files) { // remove the state files to make sure we are starting from block log remove_existing_states(copied_config); // we need to remove the reversible blocks so that new blocks can be produced from the new chain - bfs::remove_all(copied_config.blocks_dir/"reversible"); - copied_config.blocks_log_stride = stride; - copied_config.max_retained_block_files = replay_max_retained_block_files; + bfs::remove_all(copied_config.blog.log_dir/"reversible"); + copied_config.blog.stride = stride; + copied_config.blog.max_retained_files = replay_max_retained_block_files; tester from_block_log_chain(copied_config, *genesis); BOOST_CHECK( from_block_log_chain.control->fetch_block_by_number(1)->block_num() == 1); BOOST_CHECK( from_block_log_chain.control->fetch_block_by_number(75)->block_num() == 75); @@ -405,14 +428,14 @@ BOOST_AUTO_TEST_CASE(test_restart_without_blocks_log_file) { tester chain( temp_dir, [](controller::config& config) { - config.blocks_log_stride = stride; - config.max_retained_block_files = 10; + config.blog.stride = stride; + config.blog.max_retained_files = 10; }, true); chain.produce_blocks(160); controller::config copied_config = chain.get_config(); - auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blocks_dir); + auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blog.log_dir); BOOST_REQUIRE(genesis); chain.close(); @@ -420,11 +443,11 @@ BOOST_AUTO_TEST_CASE(test_restart_without_blocks_log_file) { // remove the state files to make sure we are starting from block log remove_existing_states(copied_config); // we need to remove the reversible blocks so that new blocks can be produced from the new chain - bfs::remove_all(copied_config.blocks_dir/"reversible"); - bfs::remove(copied_config.blocks_dir/"blocks.log"); - bfs::remove(copied_config.blocks_dir/"blocks.index"); - copied_config.blocks_log_stride = stride; - copied_config.max_retained_block_files = 10; + bfs::remove_all(copied_config.blog.log_dir/"reversible"); + bfs::remove(copied_config.blog.log_dir/"blocks.log"); + bfs::remove(copied_config.blog.log_dir/"blocks.index"); + copied_config.blog.stride = stride; + copied_config.blog.max_retained_files = 10; tester from_block_log_chain(copied_config, *genesis); BOOST_CHECK( from_block_log_chain.control->fetch_block_by_number(1)->block_num() == 1); BOOST_CHECK( from_block_log_chain.control->fetch_block_by_number(75)->block_num() == 75); @@ -436,10 +459,10 @@ BOOST_AUTO_TEST_CASE(test_restart_without_blocks_log_file) { BOOST_FIXTURE_TEST_CASE(auto_fix_with_incomplete_head,restart_from_block_log_test_fixture) { auto& config = chain.get_config(); - auto blocks_path = config.blocks_dir; + auto blocks_path = config.blog.log_dir; // write a few random bytes to block log indicating the last block entry is incomplete fc::cfile logfile; - logfile.set_file_path(config.blocks_dir / "blocks.log"); + logfile.set_file_path(config.blog.log_dir / "blocks.log"); logfile.open("ab"); const char random_data[] = "12345678901231876983271649837"; logfile.write(random_data, sizeof(random_data)); @@ -447,11 +470,11 @@ BOOST_FIXTURE_TEST_CASE(auto_fix_with_incomplete_head,restart_from_block_log_tes } BOOST_FIXTURE_TEST_CASE(auto_fix_with_corrupted_index,restart_from_block_log_test_fixture) { - auto& config = chain.get_config(); - auto blocks_path = config.blocks_dir; + auto& config = chain.get_config(); + auto blocks_path = config.blog.log_dir; // write a few random index to block log indicating the index is corrupted fc::cfile indexfile; - indexfile.set_file_path(config.blocks_dir / "blocks.index"); + indexfile.set_file_path(config.blog.log_dir / "blocks.index"); indexfile.open("ab"); uint64_t data = UINT64_MAX; indexfile.write(reinterpret_cast(&data), sizeof(data)); @@ -459,17 +482,17 @@ BOOST_FIXTURE_TEST_CASE(auto_fix_with_corrupted_index,restart_from_block_log_tes } BOOST_FIXTURE_TEST_CASE(auto_fix_with_corrupted_log_and_index,restart_from_block_log_test_fixture) { - auto& config = chain.get_config(); - auto blocks_path = config.blocks_dir; + auto& config = chain.get_config(); + auto blocks_path = config.blog.log_dir; // write a few random bytes to block log and index fc::cfile indexfile; - indexfile.set_file_path(config.blocks_dir / "blocks.index"); + indexfile.set_file_path(config.blog.log_dir / "blocks.index"); indexfile.open("ab"); const char random_index[] = "1234"; indexfile.write(reinterpret_cast(&random_index), sizeof(random_index)); fc::cfile logfile; - logfile.set_file_path(config.blocks_dir / "blocks.log"); + logfile.set_file_path(config.blog.log_dir / "blocks.log"); logfile.open("ab"); const char random_data[] = "12345678901231876983271649837"; logfile.write(random_data, sizeof(random_data)); @@ -489,8 +512,8 @@ BOOST_AUTO_TEST_CASE(test_split_from_v1_log) { tester chain( temp_dir, [](controller::config& config) { - config.blocks_log_stride = 20; - config.max_retained_block_files = 5; + config.blog.stride = 20; + config.blog.max_retained_files = 5; }, true); chain.produce_blocks(75); @@ -510,7 +533,7 @@ void trim_blocklog_front(uint32_t version) { namespace bfs = boost::filesystem; - auto blocks_dir = chain.get_config().blocks_dir; + auto blocks_dir = chain.get_config().blog.log_dir; auto old_index_size = fc::file_size(blocks_dir / "blocks.index"); scoped_temp_path temp1, temp2; @@ -520,8 +543,8 @@ void trim_blocklog_front(uint32_t version) { BOOST_REQUIRE_NO_THROW(block_log::trim_blocklog_front(temp1.path, temp2.path, 10)); BOOST_REQUIRE_NO_THROW(block_log::smoke_test(temp1.path, 1)); - block_log old_log(blocks_dir); - block_log new_log(temp1.path); + block_log old_log(chain.get_config().blog); + block_log new_log({ .log_dir = temp1.path}); // double check if the version has been set to the desired version BOOST_CHECK(old_log.version() == version); BOOST_CHECK(new_log.first_block_num() == 10); diff --git a/unittests/snapshot_tests.cpp b/unittests/snapshot_tests.cpp index 7e3e15e8da7..a2dc5d73a40 100644 --- a/unittests/snapshot_tests.cpp +++ b/unittests/snapshot_tests.cpp @@ -33,17 +33,17 @@ chainbase::bfs::path get_parent_path(chainbase::bfs::path blocks_dir, int ordina controller::config copy_config(const controller::config& config, int ordinal) { controller::config copied_config = config; - auto parent_path = get_parent_path(config.blocks_dir, ordinal); - copied_config.blocks_dir = parent_path / config.blocks_dir.filename().generic_string(); + auto parent_path = get_parent_path(config.blog.log_dir, ordinal); + copied_config.blog.log_dir = parent_path / config.blog.log_dir.filename().generic_string(); copied_config.state_dir = parent_path / config.state_dir.filename().generic_string(); return copied_config; } controller::config copy_config_and_files(const controller::config& config, int ordinal) { controller::config copied_config = copy_config(config, ordinal); - fc::create_directories(copied_config.blocks_dir); - fc::copy(config.blocks_dir / "blocks.log", copied_config.blocks_dir / "blocks.log"); - fc::copy(config.blocks_dir / config::reversible_blocks_dir_name, copied_config.blocks_dir / config::reversible_blocks_dir_name ); + fc::create_directories(copied_config.blog.log_dir); + fc::copy(config.blog.log_dir / "blocks.log", copied_config.blog.log_dir / "blocks.log"); + fc::copy(config.blog.log_dir / config::reversible_blocks_dir_name, copied_config.blog.log_dir / config::reversible_blocks_dir_name ); return copied_config; } @@ -52,7 +52,7 @@ class snapshotted_tester : public base_tester { enum config_file_handling { dont_copy_config_files, copy_config_files }; snapshotted_tester(controller::config config, const snapshot_reader_ptr& snapshot, int ordinal, config_file_handling copy_files_from_config = config_file_handling::dont_copy_config_files) { - FC_ASSERT(config.blocks_dir.filename().generic_string() != "." + FC_ASSERT(config.blog.log_dir.filename().generic_string() != "." && config.state_dir.filename().generic_string() != ".", "invalid path names in controller::config"); controller::config copied_config = (copy_files_from_config == copy_config_files) @@ -323,7 +323,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_exhaustive_snapshot, SNAPSHOT_SUITE, snapshot BOOST_AUTO_TEST_CASE_TEMPLATE(test_replay_over_snapshot, SNAPSHOT_SUITE, snapshot_suites) { tester chain; - const chainbase::bfs::path parent_path = chain.get_config().blocks_dir.parent_path(); + const chainbase::bfs::path parent_path = chain.get_config().blog.log_dir.parent_path(); chain.create_account(N(snapshot)); chain.produce_blocks(1); @@ -403,7 +403,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_replay_over_snapshot, SNAPSHOT_SUITE, snapsho // verifies that chain's block_log has a genesis_state (and blocks starting at 1) controller::config copied_config = copy_config_and_files(chain.get_config(), ordinal++); - auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blocks_dir); + auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blog.log_dir); BOOST_REQUIRE(genesis); tester from_block_log_chain(copied_config, *genesis); const auto from_block_log_head = from_block_log_chain.control->head_block_num(); @@ -418,7 +418,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_replay_over_snapshot, SNAPSHOT_SUITE, snapsho BOOST_AUTO_TEST_CASE_TEMPLATE(test_chain_id_in_snapshot, SNAPSHOT_SUITE, snapshot_suites) { tester chain; - const chainbase::bfs::path parent_path = chain.get_config().blocks_dir.parent_path(); + const chainbase::bfs::path parent_path = chain.get_config().blog.log_dir.parent_path(); chain.create_account(N(snapshot)); chain.produce_blocks(1); @@ -501,7 +501,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_compatible_versions, SNAPSHOT_SUITE, snapshot BOOST_AUTO_TEST_CASE_TEMPLATE(test_pending_schedule_snapshot, SNAPSHOT_SUITE, snapshot_suites) { tester chain(setup_policy::preactivate_feature_and_new_bios); - auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blocks_dir); + auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blog.log_dir); BOOST_REQUIRE(genesis); BOOST_REQUIRE_EQUAL(genesis->compute_chain_id(), chain.control->get_chain_id()); const auto& gpo = chain.control->get_global_properties(); @@ -571,7 +571,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_pending_schedule_snapshot, SNAPSHOT_SUITE, sn BOOST_AUTO_TEST_CASE_TEMPLATE(test_restart_with_existing_state_and_truncated_block_log, SNAPSHOT_SUITE, snapshot_suites) { tester chain; - const chainbase::bfs::path parent_path = chain.get_config().blocks_dir.parent_path(); + const chainbase::bfs::path parent_path = chain.get_config().blog.log_dir.parent_path(); chain.create_account(N(snapshot)); chain.produce_blocks(1); diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp index 0842f816ccc..3db8a934415 100644 --- a/unittests/state_history_tests.cpp +++ b/unittests/state_history_tests.cpp @@ -65,7 +65,7 @@ struct state_history_abi_serializer { BOOST_AUTO_TEST_SUITE(test_state_history) - BOOST_AUTO_TEST_CASE(test_trace_converter) { +BOOST_AUTO_TEST_CASE(test_trace_converter) { tester chain; using namespace eosio::state_history; @@ -116,7 +116,7 @@ BOOST_AUTO_TEST_CASE(test_trace_log) { scoped_temp_path state_history_dir; fc::create_directories(state_history_dir.path); - state_history_traces_log log(state_history_dir.path); + state_history_traces_log log({ .log_dir = state_history_dir.path }); chain.control->applied_transaction.connect( [&](std::tuple t) { @@ -141,7 +141,7 @@ BOOST_AUTO_TEST_CASE(test_trace_log) { // we assume the nodeos has to be stopped while running, it can only be read // correctly with restart - state_history_traces_log new_log(state_history_dir.path); + state_history_traces_log new_log({ .log_dir = state_history_dir.path }); auto pruned_traces = new_log.get_traces(cfd_trace->block_num); BOOST_REQUIRE(pruned_traces.size()); @@ -155,7 +155,7 @@ BOOST_AUTO_TEST_CASE(test_chain_state_log) { scoped_temp_path state_history_dir; fc::create_directories(state_history_dir.path); - state_history_chain_state_log log(state_history_dir.path); + state_history_chain_state_log log({ .log_dir = state_history_dir.path }); uint32_t last_accepted_block_num = 0; @@ -172,6 +172,113 @@ BOOST_AUTO_TEST_CASE(test_chain_state_log) { BOOST_CHECK_NO_THROW(from_bin(deltas, deltas_bin)); } +BOOST_AUTO_TEST_CASE(test_splitted_log) { + namespace bfs = boost::filesystem; + + scoped_temp_path state_history_dir; + fc::create_directories(state_history_dir.path); + + state_history_config config{ + .log_dir = state_history_dir.path, + .archive_dir = "archive", + .stride = 20, + .max_retained_files = 5 + }; + + state_history_traces_log traces_log(config); + state_history_chain_state_log chain_state_log(config); + + uint32_t last_accepted_block_num = 0; + + tester chain([&](eosio::chain::controller& control) { + control.applied_transaction.connect( + [&](std::tuple t) { + traces_log.add_transaction(std::get<0>(t), std::get<1>(t)); + }); + + control.accepted_block.connect([&](const block_state_ptr& bs) { + traces_log.store(control.db(), bs); + chain_state_log.store(control.db(), bs); + last_accepted_block_num = bs->block_num; + }); + control.block_start.connect([&](uint32_t block_num) { traces_log.block_start(block_num); } ); + }); + + + chain.produce_blocks(50); + + deploy_test_api(chain); + auto cfd_trace = push_test_cfd_transaction(chain); + + chain.produce_blocks(100); + + + auto log_dir = state_history_dir.path; + auto archive_dir = log_dir / "archive"; + + BOOST_CHECK(bfs::exists( archive_dir / "trace_history-2-20.log" )); + BOOST_CHECK(bfs::exists( archive_dir / "trace_history-2-20.index" )); + BOOST_CHECK(bfs::exists( archive_dir / "trace_history-21-40.log" )); + BOOST_CHECK(bfs::exists( archive_dir / "trace_history-21-40.index" )); + + BOOST_CHECK(bfs::exists( archive_dir / "chain_state_history-2-20.log" )); + BOOST_CHECK(bfs::exists( archive_dir / "chain_state_history-2-20.index" )); + BOOST_CHECK(bfs::exists( archive_dir / "chain_state_history-21-40.log" )); + BOOST_CHECK(bfs::exists( archive_dir / "chain_state_history-21-40.index" )); + + BOOST_CHECK(bfs::exists( log_dir / "trace_history-41-60.log" )); + BOOST_CHECK(bfs::exists( log_dir / "trace_history-41-60.index" )); + BOOST_CHECK(bfs::exists( log_dir / "trace_history-61-80.log" )); + BOOST_CHECK(bfs::exists( log_dir / "trace_history-61-80.index" )); + BOOST_CHECK(bfs::exists( log_dir / "trace_history-81-100.log" )); + BOOST_CHECK(bfs::exists( log_dir / "trace_history-81-100.index" )); + BOOST_CHECK(bfs::exists( log_dir / "trace_history-101-120.log" )); + BOOST_CHECK(bfs::exists( log_dir / "trace_history-101-120.index" )); + BOOST_CHECK(bfs::exists( log_dir / "trace_history-121-140.log" )); + BOOST_CHECK(bfs::exists( log_dir / "trace_history-121-140.index" )); + + BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-41-60.log" )); + BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-41-60.index" )); + BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-61-80.log" )); + BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-61-80.index" )); + BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-81-100.log" )); + BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-81-100.index" )); + BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-101-120.log" )); + BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-101-120.index" )); + BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-121-140.log" )); + BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-121-140.index" )); + + BOOST_CHECK(traces_log.get_traces(10).empty()); + BOOST_CHECK(traces_log.get_traces(100).size()); + BOOST_CHECK(traces_log.get_traces(140).size()); + BOOST_CHECK(traces_log.get_traces(150).size()); + BOOST_CHECK(traces_log.get_traces(160).empty()); + + BOOST_CHECK(chain_state_log.get_log_entry(10).empty()); + BOOST_CHECK(chain_state_log.get_log_entry(100).size()); + BOOST_CHECK(chain_state_log.get_log_entry(140).size()); + BOOST_CHECK(chain_state_log.get_log_entry(150).size()); + BOOST_CHECK(chain_state_log.get_log_entry(160).empty()); + + auto traces = traces_log.get_traces(cfd_trace->block_num); + BOOST_REQUIRE(traces.size()); + + BOOST_REQUIRE(!get_prunable_data_from_traces(traces, cfd_trace->id).contains()); + + std::vector ids{cfd_trace->id}; + traces_log.prune_transactions(cfd_trace->block_num, ids); + BOOST_REQUIRE(ids.empty()); + + // we assume the nodeos has to be stopped while running, it can only be read + // correctly with restart + state_history_traces_log new_log({ .log_dir = state_history_dir.path }); + auto pruned_traces = new_log.get_traces(cfd_trace->block_num); + BOOST_REQUIRE(pruned_traces.size()); + + BOOST_CHECK(get_prunable_data_from_traces(pruned_traces, cfd_trace->id).contains()); +} + + BOOST_AUTO_TEST_CASE(test_state_result_abi) { using namespace state_history; From c2e1188fa871e9617b49b95ffeedafa4a74eca03 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 2 Jul 2020 16:50:53 -0500 Subject: [PATCH 2/6] fix namespace problem --- libraries/chain/include/eosio/chain/log_catalog.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/include/eosio/chain/log_catalog.hpp b/libraries/chain/include/eosio/chain/log_catalog.hpp index 38b9f00baf8..d2d9a7acf9b 100644 --- a/libraries/chain/include/eosio/chain/log_catalog.hpp +++ b/libraries/chain/include/eosio/chain/log_catalog.hpp @@ -9,7 +9,7 @@ namespace eosio { namespace chain { -namespace bfs = bfs; +namespace bfs = boost::filesystem; template void for_each_file_in_dir_matches(const bfs::path& dir, std::string pattern, Lambda&& lambda) { From 692d80b790b25b9f8b2476110a2902607fb20160 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 6 Jul 2020 09:20:06 -0500 Subject: [PATCH 3/6] Avoid designated initializers For gcc 8 compatibility --- plugins/state_history_plugin/state_history_plugin.cpp | 2 +- programs/eosio-blocklog/main.cpp | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index f8de60ff7c3..2c6e9a3c730 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -385,7 +385,7 @@ void state_history_plugin::set_program_options(options_description& cli, options "If the value is empty, blocks files beyond the retained limit will be deleted.\n" "All files in the archive directory are completely under user's control, i.e. they won't be accessed by nodeos anymore."); options("state-history-stride", bpo::value()->default_value(UINT32_MAX), - "split the state history log file when the block number is the multiple of the stride\n" + "split the state history log files when the block number is the multiple of the stride\n" "When the stride is reached, the current history log and index will be renamed '*-history--.log/index'\n" "and a new current history log and index will be created with the most recent blocks. All files following\n" "this format will be used to construct an extended history log."); diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp index 0b0077e7092..0f9f37a2d2e 100644 --- a/programs/eosio-blocklog/main.cpp +++ b/programs/eosio-blocklog/main.cpp @@ -240,7 +240,10 @@ bool trim_blocklog_front(bfs::path block_dir, uint32_t n) { //n is first void fix_irreversible_blocks(bfs::path block_dir) { std::cout << "\nfix_irreversible_blocks of blocks.log and blocks.index in directory " << block_dir << '\n'; - block_log block_logger({ .log_dir = block_dir, .fix_irreversible_blocks = true }); + block_log::config_type config; + config.log_dir = block_dir; + config.fix_irreversible_blocks = true; + block_log block_logger(config); std::cout << "\nSmoke test of blocks.log and blocks.index in directory " << block_dir << '\n'; block_log::smoke_test(block_dir, 0); From bac62391a523afeaace8714c00f106ccc58a54f9 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 6 Jul 2020 13:15:05 -0500 Subject: [PATCH 4/6] Add retained directory --- libraries/chain/block_log.cpp | 2 +- .../include/eosio/chain/block_log_config.hpp | 1 + .../chain/include/eosio/chain/log_catalog.hpp | 40 +++-- .../include/eosio/state_history/log.hpp | 1 + libraries/state_history/log.cpp | 2 +- plugins/chain_plugin/chain_plugin.cpp | 18 ++- .../state_history_plugin.cpp | 4 + unittests/state_history_tests.cpp | 153 +++++++++++------- 8 files changed, 144 insertions(+), 77 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 1b20aa55a84..8d041000d1f 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -567,7 +567,7 @@ namespace eosio { namespace chain { if (!fc::is_directory(config.log_dir)) fc::create_directories(config.log_dir); - catalog.open(config.log_dir, config.archive_dir, "blocks"); + catalog.open(config.log_dir, config.retained_dir, config.archive_dir, "blocks"); catalog.max_retained_files = config.max_retained_files; this->stride = config.stride; diff --git a/libraries/chain/include/eosio/chain/block_log_config.hpp b/libraries/chain/include/eosio/chain/block_log_config.hpp index 86f78464c58..5e3401bc61d 100644 --- a/libraries/chain/include/eosio/chain/block_log_config.hpp +++ b/libraries/chain/include/eosio/chain/block_log_config.hpp @@ -8,6 +8,7 @@ namespace bfs = boost::filesystem; struct block_log_config { bfs::path log_dir; + bfs::path retained_dir; bfs::path archive_dir; uint32_t stride = UINT32_MAX; uint16_t max_retained_files = 10; diff --git a/libraries/chain/include/eosio/chain/log_catalog.hpp b/libraries/chain/include/eosio/chain/log_catalog.hpp index d2d9a7acf9b..2fe6a84fcee 100644 --- a/libraries/chain/include/eosio/chain/log_catalog.hpp +++ b/libraries/chain/include/eosio/chain/log_catalog.hpp @@ -46,6 +46,7 @@ struct log_catalog { using mapmode = boost::iostreams::mapped_file::mapmode; + bfs::path retained_dir; bfs::path archive_dir; size_type max_retained_files = 10; collection_t collection; @@ -62,20 +63,25 @@ struct log_catalog { return collection.begin()->first; } - void open(const bfs::path& log_dir, const bfs::path& archive_dir, const char* name, + static bfs::path make_abosolute_dir(const bfs::path& base_dir, bfs::path new_dir) { + if (new_dir.is_relative()) + new_dir = base_dir / new_dir; + + if (!bfs::is_directory(new_dir)) + bfs::create_directories(new_dir); + + return new_dir; + } + + void open(const bfs::path& log_dir, const bfs::path& retained_dir, const bfs::path& archive_dir, const char* name, const char* suffix_pattern = R"(-\d+-\d+\.log)") { + this->retained_dir = make_abosolute_dir(log_dir, retained_dir.empty() ? log_dir : retained_dir); if (!archive_dir.empty()) { - if (archive_dir.is_relative()) - this->archive_dir = log_dir / archive_dir; - else - this->archive_dir = archive_dir; - - if (!bfs::is_directory(this->archive_dir)) - bfs::create_directories(this->archive_dir); + this->archive_dir = make_abosolute_dir(log_dir, archive_dir); } - for_each_file_in_dir_matches(log_dir, std::string(name) + suffix_pattern, [this](bfs::path path) { + for_each_file_in_dir_matches(this->retained_dir, std::string(name) + suffix_pattern, [this](bfs::path path) { auto log_path = path; auto index_path = path.replace_extension("index"); auto path_without_extension = log_path.parent_path() / log_path.stem().string(); @@ -184,9 +190,19 @@ struct log_catalog { return {}; } + static void rename_if_not_exists(bfs::path old_name, bfs::path new_name) { + if (!bfs::exists(new_name)) { + bfs::rename(old_name, new_name); + } + else { + bfs::remove(old_name); + wlog("${new_name} already exists, just remove ${old_name}", ("old_name", old_name.string())("new_name", new_name.string())); + } + } + static void rename_bundle(bfs::path orig_path, bfs::path new_path) { - bfs::rename(orig_path.replace_extension(".log"), new_path.replace_extension(".log")); - bfs::rename(orig_path.replace_extension(".index"), new_path.replace_extension(".index")); + rename_if_not_exists(orig_path.replace_extension(".log"), new_path.replace_extension(".log")); + rename_if_not_exists(orig_path.replace_extension(".index"), new_path.replace_extension(".index")); } /// Add a new entry into the catalog. @@ -201,7 +217,7 @@ struct log_catalog { const int bufsize = 64; char buf[bufsize]; snprintf(buf, bufsize, "%s-%d-%d", name, start_block_num, end_block_num); - bfs::path new_path = dir / buf; + bfs::path new_path = retained_dir / buf; rename_bundle(dir / name, new_path); if (this->collection.size() >= max_retained_files) { diff --git a/libraries/state_history/include/eosio/state_history/log.hpp b/libraries/state_history/include/eosio/state_history/log.hpp index 936e8083deb..fb0f2d7eeb2 100644 --- a/libraries/state_history/include/eosio/state_history/log.hpp +++ b/libraries/state_history/include/eosio/state_history/log.hpp @@ -98,6 +98,7 @@ class state_history_log_data : public chain::log_data_basestride = config.stride; open_log(config.log_dir / (std::string(name) + ".log")); diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index edbce800703..fd68e21b77f 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -229,13 +229,16 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip "the location of the blocks directory (absolute path or relative to application data dir)") ("blocks-log-stride", bpo::value()->default_value(config::default_blocks_log_stride), "split the block log file when the head block number is the multiple of the stride\n" - "When the stride is reached, the current block log and index will be renamed 'blocks--.log/index'\n" + "When the stride is reached, the current block log and index will be renamed '/blocks--.log/index'\n" "and a new current block log and index will be created with the most recent block. All files following\n" "this format will be used to construct an extended block log.") ("max-retained-block-files", bpo::value()->default_value(config::default_max_retained_block_files), "the maximum number of blocks files to retain so that the blocks in those files can be queried.\n" "When the number is reached, the oldest block file would be moved to archive dir or deleted if the archive dir is empty.\n" "The retained block log files should not be manipulated by users." ) + ("blocks-retained-dir", bpo::value()->default_value(""), + "the location of the blocks retained directory (absolute path or relative to blocks dir).\n" + "If the value is empty, it is set to the value of blocks dir.") ("blocks-archive-dir", bpo::value()->default_value(config::default_blocks_archive_dir_name), "the location of the blocks archive directory (absolute path or relative to blocks dir).\n" "If the value is empty, blocks files beyond the retained limit will be deleted.\n" @@ -754,12 +757,13 @@ void chain_plugin::plugin_initialize(const variables_map& options) { my->chain_config->abi_serializer_max_time_us = my->abi_serializer_max_time_us; } - my->chain_config->blog.log_dir = my->blocks_dir; - my->chain_config->state_dir = app().data_dir() / config::default_state_dir_name; - my->chain_config->read_only = my->readonly; - my->chain_config->blog.archive_dir = options.at("blocks-archive-dir").as(); - my->chain_config->blog.stride = options.at("blocks-log-stride").as(); - my->chain_config->blog.max_retained_files = options.at("max-retained-block-files").as(); + my->chain_config->blog.log_dir = my->blocks_dir; + my->chain_config->state_dir = app().data_dir() / config::default_state_dir_name; + my->chain_config->read_only = my->readonly; + my->chain_config->blog.retained_dir = options.at("blocks-retained-dir").as(); + my->chain_config->blog.archive_dir = options.at("blocks-archive-dir").as(); + my->chain_config->blog.stride = options.at("blocks-log-stride").as(); + my->chain_config->blog.max_retained_files = options.at("max-retained-block-files").as(); my->chain_config->blog.fix_irreversible_blocks = options.at("fix-irreversible-blocks").as(); if (auto resmon_plugin = app().find_plugin()) { diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 2c6e9a3c730..cd3c86dde09 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -380,6 +380,9 @@ void state_history_plugin::set_program_options(options_description& cli, options auto options = cfg.add_options(); options("state-history-dir", bpo::value()->default_value("state-history"), "the location of the state-history directory (absolute path or relative to application data dir)"); + options("state-history-retained-dir", bpo::value()->default_value(""), + "the location of the state history retained directory (absolute path or relative to state-history dir).\n" + "If the value is empty, it is set to the value of state-history directory."); options("state-history-archive-dir", bpo::value()->default_value("archive"), "the location of the state history archive directory (absolute path or relative to state-history dir).\n" "If the value is empty, blocks files beyond the retained limit will be deleted.\n" @@ -433,6 +436,7 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { if (auto resmon_plugin = app().find_plugin()) resmon_plugin->monitor_directory(config.log_dir); + config.retained_dir = options.at("state-history-retained-dir").as(); config.archive_dir = options.at("state-history-archive-dir").as(); config.stride = options.at("state-history-stride").as(); config.max_retained_files = options.at("max-retained-history-files").as(); diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp index 3db8a934415..cf64d4c3eec 100644 --- a/unittests/state_history_tests.cpp +++ b/unittests/state_history_tests.cpp @@ -172,25 +172,18 @@ BOOST_AUTO_TEST_CASE(test_chain_state_log) { BOOST_CHECK_NO_THROW(from_bin(deltas, deltas_bin)); } -BOOST_AUTO_TEST_CASE(test_splitted_log) { - namespace bfs = boost::filesystem; - - scoped_temp_path state_history_dir; - fc::create_directories(state_history_dir.path); - - state_history_config config{ - .log_dir = state_history_dir.path, - .archive_dir = "archive", - .stride = 20, - .max_retained_files = 5 - }; - state_history_traces_log traces_log(config); - state_history_chain_state_log chain_state_log(config); +struct state_history_tester_logs { + state_history_tester_logs(const state_history_config& config) + : traces_log(config) , chain_state_log(config) {} - uint32_t last_accepted_block_num = 0; + state_history_traces_log traces_log; + state_history_chain_state_log chain_state_log; +}; - tester chain([&](eosio::chain::controller& control) { +struct state_history_tester : state_history_tester_logs, tester { + state_history_tester(const state_history_config& config) + : state_history_tester_logs(config), tester ([&](eosio::chain::controller& control) { control.applied_transaction.connect( [&](std::tuple t) { traces_log.add_transaction(std::get<0>(t), std::get<1>(t)); @@ -199,12 +192,26 @@ BOOST_AUTO_TEST_CASE(test_splitted_log) { control.accepted_block.connect([&](const block_state_ptr& bs) { traces_log.store(control.db(), bs); chain_state_log.store(control.db(), bs); - last_accepted_block_num = bs->block_num; }); control.block_start.connect([&](uint32_t block_num) { traces_log.block_start(block_num); } ); - }); + }) {} +}; +BOOST_AUTO_TEST_CASE(test_splitted_log) { + namespace bfs = boost::filesystem; + + scoped_temp_path state_history_dir; + fc::create_directories(state_history_dir.path); + + state_history_config config{ + .log_dir = state_history_dir.path, + .retained_dir = "retained", + .archive_dir = "archive", + .stride = 20, + .max_retained_files = 5 + }; + state_history_tester chain(config); chain.produce_blocks(50); deploy_test_api(chain); @@ -214,7 +221,8 @@ BOOST_AUTO_TEST_CASE(test_splitted_log) { auto log_dir = state_history_dir.path; - auto archive_dir = log_dir / "archive"; + auto archive_dir = log_dir / "archive"; + auto retained_dir = log_dir / "retained"; BOOST_CHECK(bfs::exists( archive_dir / "trace_history-2-20.log" )); BOOST_CHECK(bfs::exists( archive_dir / "trace_history-2-20.index" )); @@ -226,58 +234,91 @@ BOOST_AUTO_TEST_CASE(test_splitted_log) { BOOST_CHECK(bfs::exists( archive_dir / "chain_state_history-21-40.log" )); BOOST_CHECK(bfs::exists( archive_dir / "chain_state_history-21-40.index" )); - BOOST_CHECK(bfs::exists( log_dir / "trace_history-41-60.log" )); - BOOST_CHECK(bfs::exists( log_dir / "trace_history-41-60.index" )); - BOOST_CHECK(bfs::exists( log_dir / "trace_history-61-80.log" )); - BOOST_CHECK(bfs::exists( log_dir / "trace_history-61-80.index" )); - BOOST_CHECK(bfs::exists( log_dir / "trace_history-81-100.log" )); - BOOST_CHECK(bfs::exists( log_dir / "trace_history-81-100.index" )); - BOOST_CHECK(bfs::exists( log_dir / "trace_history-101-120.log" )); - BOOST_CHECK(bfs::exists( log_dir / "trace_history-101-120.index" )); - BOOST_CHECK(bfs::exists( log_dir / "trace_history-121-140.log" )); - BOOST_CHECK(bfs::exists( log_dir / "trace_history-121-140.index" )); - - BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-41-60.log" )); - BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-41-60.index" )); - BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-61-80.log" )); - BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-61-80.index" )); - BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-81-100.log" )); - BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-81-100.index" )); - BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-101-120.log" )); - BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-101-120.index" )); - BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-121-140.log" )); - BOOST_CHECK(bfs::exists( log_dir / "chain_state_history-121-140.index" )); - - BOOST_CHECK(traces_log.get_traces(10).empty()); - BOOST_CHECK(traces_log.get_traces(100).size()); - BOOST_CHECK(traces_log.get_traces(140).size()); - BOOST_CHECK(traces_log.get_traces(150).size()); - BOOST_CHECK(traces_log.get_traces(160).empty()); - - BOOST_CHECK(chain_state_log.get_log_entry(10).empty()); - BOOST_CHECK(chain_state_log.get_log_entry(100).size()); - BOOST_CHECK(chain_state_log.get_log_entry(140).size()); - BOOST_CHECK(chain_state_log.get_log_entry(150).size()); - BOOST_CHECK(chain_state_log.get_log_entry(160).empty()); - - auto traces = traces_log.get_traces(cfd_trace->block_num); + BOOST_CHECK(bfs::exists( retained_dir / "trace_history-41-60.log" )); + BOOST_CHECK(bfs::exists( retained_dir / "trace_history-41-60.index" )); + BOOST_CHECK(bfs::exists( retained_dir / "trace_history-61-80.log" )); + BOOST_CHECK(bfs::exists( retained_dir / "trace_history-61-80.index" )); + BOOST_CHECK(bfs::exists( retained_dir / "trace_history-81-100.log" )); + BOOST_CHECK(bfs::exists( retained_dir / "trace_history-81-100.index" )); + BOOST_CHECK(bfs::exists( retained_dir / "trace_history-101-120.log" )); + BOOST_CHECK(bfs::exists( retained_dir / "trace_history-101-120.index" )); + BOOST_CHECK(bfs::exists( retained_dir / "trace_history-121-140.log" )); + BOOST_CHECK(bfs::exists( retained_dir / "trace_history-121-140.index" )); + + BOOST_CHECK(bfs::exists( retained_dir / "chain_state_history-41-60.log" )); + BOOST_CHECK(bfs::exists( retained_dir / "chain_state_history-41-60.index" )); + BOOST_CHECK(bfs::exists( retained_dir / "chain_state_history-61-80.log" )); + BOOST_CHECK(bfs::exists( retained_dir / "chain_state_history-61-80.index" )); + BOOST_CHECK(bfs::exists( retained_dir / "chain_state_history-81-100.log" )); + BOOST_CHECK(bfs::exists( retained_dir / "chain_state_history-81-100.index" )); + BOOST_CHECK(bfs::exists( retained_dir / "chain_state_history-101-120.log" )); + BOOST_CHECK(bfs::exists( retained_dir / "chain_state_history-101-120.index" )); + BOOST_CHECK(bfs::exists( retained_dir / "chain_state_history-121-140.log" )); + BOOST_CHECK(bfs::exists( retained_dir / "chain_state_history-121-140.index" )); + + BOOST_CHECK(chain.traces_log.get_traces(10).empty()); + BOOST_CHECK(chain.traces_log.get_traces(100).size()); + BOOST_CHECK(chain.traces_log.get_traces(140).size()); + BOOST_CHECK(chain.traces_log.get_traces(150).size()); + BOOST_CHECK(chain.traces_log.get_traces(160).empty()); + + BOOST_CHECK(chain.chain_state_log.get_log_entry(10).empty()); + BOOST_CHECK(chain.chain_state_log.get_log_entry(100).size()); + BOOST_CHECK(chain.chain_state_log.get_log_entry(140).size()); + BOOST_CHECK(chain.chain_state_log.get_log_entry(150).size()); + BOOST_CHECK(chain.chain_state_log.get_log_entry(160).empty()); + + auto traces = chain.traces_log.get_traces(cfd_trace->block_num); BOOST_REQUIRE(traces.size()); BOOST_REQUIRE(!get_prunable_data_from_traces(traces, cfd_trace->id).contains()); std::vector ids{cfd_trace->id}; - traces_log.prune_transactions(cfd_trace->block_num, ids); + chain.traces_log.prune_transactions(cfd_trace->block_num, ids); BOOST_REQUIRE(ids.empty()); // we assume the nodeos has to be stopped while running, it can only be read // correctly with restart - state_history_traces_log new_log({ .log_dir = state_history_dir.path }); + state_history_traces_log new_log(config); auto pruned_traces = new_log.get_traces(cfd_trace->block_num); BOOST_REQUIRE(pruned_traces.size()); BOOST_CHECK(get_prunable_data_from_traces(pruned_traces, cfd_trace->id).contains()); } +BOOST_AUTO_TEST_CASE(test_corrupted_log_recovery) { + namespace bfs = boost::filesystem; + + scoped_temp_path state_history_dir; + fc::create_directories(state_history_dir.path); + + state_history_config config{ + .log_dir = state_history_dir.path, + .archive_dir = "archive", + .stride = 100, + .max_retained_files = 5 + }; + + state_history_tester chain(config); + chain.produce_blocks(50); + chain.close(); + + // write a few random bytes to block log indicating the last block entry is incomplete + fc::cfile logfile; + logfile.set_file_path(state_history_dir.path / "trace_history.log"); + logfile.open("ab"); + const char random_data[] = "12345678901231876983271649837"; + logfile.write(random_data, sizeof(random_data)); + + bfs::remove_all(chain.get_config().blog.log_dir/"reversible"); + + state_history_tester new_chain(config); + new_chain.produce_blocks(50); + + BOOST_CHECK(new_chain.traces_log.get_traces(10).size()); + BOOST_CHECK(new_chain.chain_state_log.get_log_entry(10).size()); +} + BOOST_AUTO_TEST_CASE(test_state_result_abi) { using namespace state_history; From 1e938deb53a2922288665ba7e79657fc94a1366f Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 13 Jul 2020 09:55:11 -0500 Subject: [PATCH 5/6] Address some PR comments --- .../chain/include/eosio/chain/log_catalog.hpp | 72 +++++++++---------- .../chain/include/eosio/chain/log_index.hpp | 2 +- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/libraries/chain/include/eosio/chain/log_catalog.hpp b/libraries/chain/include/eosio/chain/log_catalog.hpp index 2fe6a84fcee..86584715869 100644 --- a/libraries/chain/include/eosio/chain/log_catalog.hpp +++ b/libraries/chain/include/eosio/chain/log_catalog.hpp @@ -20,7 +20,7 @@ void for_each_file_in_dir_matches(const bfs::path& dir, std::string pattern, Lam // Skip if not a file if (!bfs::is_regular_file(p->status())) continue; - // skip if it's not match blocks-*-*.log + // skip if it does not match the pattern if (!std::regex_match(p->path().filename().string(), what, my_filter)) continue; lambda(p->path()); @@ -97,8 +97,8 @@ struct log_catalog { auto existing_itr = collection.find(log.first_block_num()); if (existing_itr != collection.end()) { if (log.last_block_num() <= existing_itr->second.last_block_num) { - wlog("${log_path} contains the overlapping range with ${existing_path}.log, droping ${log_path} " - "from catelog", + wlog("${log_path} contains the overlapping range with ${existing_path}.log, dropping ${log_path} " + "from catalog", ("log_path", log_path.string())("existing_path", existing_itr->second.filename_base.string())); return; } else { @@ -114,41 +114,33 @@ struct log_catalog { } bool index_matches_data(const bfs::path& index_path, const LogData& log) const { - if (bfs::exists(index_path) && bfs::file_size(index_path) / sizeof(uint64_t) != log.num_blocks()) { - // make sure the last 8 bytes of index and log matches - - fc::cfile index_file; - index_file.set_file_path(index_path); - index_file.open("r"); - index_file.seek_end(-sizeof(uint64_t)); - uint64_t pos; - index_file.read(reinterpret_cast(&pos), sizeof(pos)); - return pos == log.last_block_position(); - } - return false; - } + if (!bfs::exists(index_path)) return false; - std::string filebase_for_block(uint32_t block_num) { - if (collection.empty() || block_num < collection.begin()->first) - return ""; - auto it = --collection.upper_bound(block_num); + auto num_blocks_in_index = bfs::file_size(index_path) / sizeof(uint64_t); + if (num_blocks_in_index != log.num_blocks()) + return false; - if (block_num <= it->second.last_block_num) - return it->second.filename_base; - return ""; + // make sure the last 8 bytes of index and log matches + fc::cfile index_file; + index_file.set_file_path(index_path); + index_file.open("r"); + index_file.seek_end(-sizeof(uint64_t)); + uint64_t pos; + index_file.read(reinterpret_cast(&pos), sizeof(pos)); + return pos == log.last_block_position(); } - bool set_active_item(uint32_t block_num, mapmode mode = mapmode::readonly) { + std::optional get_block_position(uint32_t block_num, mapmode mode = mapmode::readonly) { try { if (active_index != npos) { auto active_item = collection.nth(active_index); if (active_item->first <= block_num && block_num <= active_item->second.last_block_num && log_data.flags() == mode) { - return true; + return log_index.nth_block_position(block_num - log_data.first_block_num()); } } if (collection.empty() || block_num < collection.begin()->first) - return false; + return {}; auto it = --collection.upper_bound(block_num); @@ -157,35 +149,35 @@ struct log_catalog { log_data.open(name.replace_extension("log"), mode); log_index.open(name.replace_extension("index")); this->active_index = collection.index_of(it); - return true; + return log_index.nth_block_position(block_num - log_data.first_block_num()); } - return false; + return {}; } catch (...) { this->active_index = npos; - return false; + return {}; } } std::pair, uint32_t> ro_stream_for_block(uint32_t block_num) { - if (set_active_item(block_num, mapmode::readonly)) { - auto pos = log_index.nth_block_position(block_num - log_data.first_block_num()); - return std::make_pair(log_data.ro_stream_at(pos), log_data.version()); + auto pos = get_block_position(block_num, mapmode::readonly); + if (pos) { + return std::make_pair(log_data.ro_stream_at(*pos), log_data.version()); } return {fc::datastream(nullptr, 0), static_cast(0)}; } std::pair, uint32_t> rw_stream_for_block(uint32_t block_num) { - if (set_active_item(block_num, mapmode::readwrite)) { - auto pos = log_index.nth_block_position(block_num - log_data.first_block_num()); - return std::make_pair(log_data.rw_stream_at(pos), log_data.version()); + auto pos = get_block_position(block_num, mapmode::readwrite); + if (pos) { + return std::make_pair(log_data.rw_stream_at(*pos), log_data.version()); } return {fc::datastream(nullptr, 0), static_cast(0)}; } std::optional id_for_block(uint32_t block_num) { - if (set_active_item(block_num, mapmode::readonly)) { - auto pos = log_index.nth_block_position(block_num - log_data.first_block_num()); - return log_data.block_id_at(pos); + auto pos = get_block_position(block_num, mapmode::readonly); + if (pos) { + return log_data.block_id_at(*pos); } return {}; } @@ -196,7 +188,7 @@ struct log_catalog { } else { bfs::remove(old_name); - wlog("${new_name} already exists, just remove ${old_name}", ("old_name", old_name.string())("new_name", new_name.string())); + wlog("${new_name} already exists, just removing ${old_name}", ("old_name", old_name.string())("new_name", new_name.string())); } } @@ -221,7 +213,7 @@ struct log_catalog { rename_bundle(dir / name, new_path); if (this->collection.size() >= max_retained_files) { - auto items_to_erase = + const auto items_to_erase = max_retained_files > 0 ? this->collection.size() - max_retained_files + 1 : this->collection.size(); for (auto it = this->collection.begin(); it < this->collection.begin() + items_to_erase; ++it) { auto orig_name = it->second.filename_base; diff --git a/libraries/chain/include/eosio/chain/log_index.hpp b/libraries/chain/include/eosio/chain/log_index.hpp index fed17c23164..4ff5f33fdde 100644 --- a/libraries/chain/include/eosio/chain/log_index.hpp +++ b/libraries/chain/include/eosio/chain/log_index.hpp @@ -19,7 +19,7 @@ class log_index { file.close(); file.open(path.generic_string()); EOS_ASSERT(file.size() % sizeof(uint64_t) == 0, Exception, - "The size of ${file} is not the multiple of sizeof(uint64_t)", ("file", path.generic_string())); + "The size of ${file} is not a multiple of sizeof(uint64_t)", ("file", path.generic_string())); } bool is_open() const { return file.is_open(); } From 083cf2dc390d1928fc6b0e726b652a37b1f106c4 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 23 Jul 2020 14:03:55 -0500 Subject: [PATCH 6/6] Address PR comment --- .../state_history_plugin/state_history_plugin.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index cd3c86dde09..bc35687873c 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -69,13 +69,14 @@ struct state_history_plugin_impl : std::enable_shared_from_thisget_block_id(block_num); - if (!result) { - try { - return chain_plug->chain().get_block_id_for_num(block_num); - } catch (...) { - } + if (result) + return result; + + try { + return chain_plug->chain().get_block_id_for_num(block_num); + } catch (...) { + return {}; } - return result; } struct session : std::enable_shared_from_this {