diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index d025153d095..b91a1ee62bf 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -320,12 +320,12 @@ namespace eosio { namespace chain { return fc::endian_reverse_u32(prev_block_num) + 1; } - fc::datastream ro_stream_at(uint64_t pos) { - return fc::datastream(file.const_data() + pos, file.size() - pos); + std::pair,uint32_t> ro_stream_at(uint64_t pos) { + return std::make_pair(fc::datastream(file.const_data() + pos, file.size() - pos), version()); } - fc::datastream rw_stream_at(uint64_t pos) { - return fc::datastream(file.data() + pos, file.size() - pos); + std::pair,uint32_t> rw_stream_at(uint64_t pos) { + return std::make_pair(fc::datastream(file.data() + pos, file.size() - pos), version()); } /** diff --git a/libraries/chain/include/eosio/chain/log_catalog.hpp b/libraries/chain/include/eosio/chain/log_catalog.hpp index 6b78f3e5cff..48f17cd156e 100644 --- a/libraries/chain/include/eosio/chain/log_catalog.hpp +++ b/libraries/chain/include/eosio/chain/log_catalog.hpp @@ -162,7 +162,7 @@ struct log_catalog { std::pair, uint32_t> ro_stream_for_block(uint32_t block_num) { auto pos = get_block_position(block_num, mapmode::readonly); if (pos) { - return std::make_pair(log_data.ro_stream_at(*pos), log_data.version()); + return log_data.ro_stream_at(*pos); } return {fc::datastream(nullptr, 0), static_cast(0)}; } @@ -170,7 +170,7 @@ struct log_catalog { std::pair, uint32_t> rw_stream_for_block(uint32_t block_num) { auto pos = get_block_position(block_num, mapmode::readwrite); if (pos) { - return std::make_pair(log_data.rw_stream_at(*pos), log_data.version()); + return log_data.rw_stream_at(*pos); } return {fc::datastream(nullptr, 0), static_cast(0)}; } @@ -272,4 +272,4 @@ struct log_catalog { }; } // namespace chain -} // namespace eosio \ No newline at end of file +} // namespace eosio diff --git a/libraries/state_history/include/eosio/state_history/log.hpp b/libraries/state_history/include/eosio/state_history/log.hpp index ae5e4fef52d..595628da155 100644 --- a/libraries/state_history/include/eosio/state_history/log.hpp +++ b/libraries/state_history/include/eosio/state_history/log.hpp @@ -80,13 +80,17 @@ class state_history_log_data : public chain::log_data_base ro_stream_at(uint64_t pos) const { - return fc::datastream(file.const_data() + pos + sizeof(state_history_log_header), - payload_size_at(pos)); + std::pair, uint32_t> ro_stream_at(uint64_t pos) const { + uint32_t ver = get_ship_version(chain::read_buffer(file.const_data() + pos)); + return std::make_pair( + fc::datastream(file.const_data() + pos + sizeof(state_history_log_header), payload_size_at(pos)), + ver); } - fc::datastream rw_stream_at(uint64_t pos) const { - return fc::datastream(file.data() + pos + sizeof(state_history_log_header), payload_size_at(pos)); + std::pair, uint32_t> rw_stream_at(uint64_t pos) const { + uint32_t ver = get_ship_version(chain::read_buffer(file.const_data() + pos)); + return std::make_pair( + fc::datastream(file.data() + pos + sizeof(state_history_log_header), payload_size_at(pos)), ver); } uint32_t block_num_at(uint64_t position) const { @@ -197,7 +201,7 @@ class state_history_traces_log : public state_history_log { cache.add_transaction(trace, transaction); } - std::vector get_traces(block_num_type block_num); + chain::bytes get_log_entry(block_num_type block_num); void block_start(uint32_t block_num) { cache.clear(); } diff --git a/libraries/state_history/log.cpp b/libraries/state_history/log.cpp index 78cefc8b3dc..c49c7593c96 100644 --- a/libraries/state_history/log.cpp +++ b/libraries/state_history/log.cpp @@ -292,28 +292,39 @@ void state_history_log::split_log() { state_history_traces_log::state_history_traces_log(const state_history_config& config) : state_history_log("trace_history", config) {} -std::vector state_history_traces_log::get_traces(block_num_type block_num) { +chain::bytes state_history_traces_log::get_log_entry(block_num_type block_num) { - auto [ds, _] = catalog.ro_stream_for_block(block_num); + auto get_traces_bin = [](auto& ds, uint32_t version) { + if (version == 0) { + return state_history::zlib_decompress(ds); + } + else { + std::vector traces; + state_history::trace_converter::unpack(ds, traces); + return fc::raw::pack(traces); + } + }; + + auto [ds, version] = catalog.ro_stream_for_block(block_num); if (ds.remaining()) { - std::vector traces; - state_history::trace_converter::unpack(ds, traces); - return traces; + return get_traces_bin(ds, version); } if (block_num < begin_block() || block_num >= end_block()) return {}; state_history_log_header header; get_entry_header(block_num, header); - std::vector traces; - state_history::trace_converter::unpack(read_log, traces); - return traces; + return get_traces_bin(read_log, get_ship_version(header.magic)); } + void state_history_traces_log::prune_transactions(state_history_log::block_num_type block_num, std::vector& ids) { - auto [ds, _] = catalog.rw_stream_for_block(block_num); + auto [ds, version] = catalog.rw_stream_for_block(block_num); + if (ds.remaining()) { + EOS_ASSERT(version > 0, chain::state_history_exception, + "The trace log version 0 does not support transaction pruning."); state_history::trace_converter::prune_traces(ds, ds.remaining(), ids); return; } @@ -322,6 +333,8 @@ void state_history_traces_log::prune_transactions(state_history_log::block_num_t return; state_history_log_header header; get_entry_header(block_num, header); + EOS_ASSERT(get_ship_version(header.magic) > 0, chain::state_history_exception, + "The trace log version 0 does not support transaction pruning."); write_log.seek(read_log.tellp()); state_history::trace_converter::prune_traces(write_log, header.payload_size, ids); write_log.flush(); diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index e957c973ac9..a510e9da94a 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -224,7 +224,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisfetch_block) result.block = plugin->get_block(current_request->start_block_num); if (current_request->fetch_traces && plugin->trace_log) { - result.traces = plugin->trace_log->get_traces(current_request->start_block_num); + result.traces = plugin->trace_log->get_log_entry(current_request->start_block_num); } if (current_request->fetch_deltas && plugin->chain_state_log) { result.deltas = plugin->chain_state_log->get_log_entry(current_request->start_block_num); diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 88c67577694..a5e0d1fa6aa 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -32,6 +32,9 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts.hpp.in ${CMAKE_CURRENT_BINA add_subdirectory(snapshots) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/snapshots.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/snapshots.hpp ESCAPE_QUOTES) +add_subdirectory(state-history-data) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/utilities.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/utilities.hpp ESCAPE_QUOTES) + add_custom_command( OUTPUT protocol_feature_digest_tests.cpp COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/gen_protocol_feature_digest_tests.py ${CMAKE_CURRENT_SOURCE_DIR}/../libraries/chain/protocol_feature_manager.cpp > protocol_feature_digest_tests.cpp diff --git a/unittests/state-history-data/CMakeLists.txt b/unittests/state-history-data/CMakeLists.txt new file mode 100644 index 00000000000..9d1b970e287 --- /dev/null +++ b/unittests/state-history-data/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(trace-log-v0) +add_subdirectory(trace-log-v1) \ No newline at end of file diff --git a/unittests/state-history-data/trace-log-v0/CMakeLists.txt b/unittests/state-history-data/trace-log-v0/CMakeLists.txt new file mode 100644 index 00000000000..25a384937d0 --- /dev/null +++ b/unittests/state-history-data/trace-log-v0/CMakeLists.txt @@ -0,0 +1,2 @@ +configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/trace_history.index ${CMAKE_CURRENT_BINARY_DIR}/trace_history.index COPYONLY ) +configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/trace_history.log ${CMAKE_CURRENT_BINARY_DIR}/trace_history.log COPYONLY ) \ No newline at end of file diff --git a/unittests/state-history-data/trace-log-v0/trace_history.index b/unittests/state-history-data/trace-log-v0/trace_history.index new file mode 100644 index 00000000000..4d7eda8534a Binary files /dev/null and b/unittests/state-history-data/trace-log-v0/trace_history.index differ diff --git a/unittests/state-history-data/trace-log-v0/trace_history.log b/unittests/state-history-data/trace-log-v0/trace_history.log new file mode 100644 index 00000000000..1d8fba5775e Binary files /dev/null and b/unittests/state-history-data/trace-log-v0/trace_history.log differ diff --git a/unittests/state-history-data/trace-log-v1/CMakeLists.txt b/unittests/state-history-data/trace-log-v1/CMakeLists.txt new file mode 100644 index 00000000000..25a384937d0 --- /dev/null +++ b/unittests/state-history-data/trace-log-v1/CMakeLists.txt @@ -0,0 +1,2 @@ +configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/trace_history.index ${CMAKE_CURRENT_BINARY_DIR}/trace_history.index COPYONLY ) +configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/trace_history.log ${CMAKE_CURRENT_BINARY_DIR}/trace_history.log COPYONLY ) \ No newline at end of file diff --git a/unittests/state-history-data/trace-log-v1/trace_history.index b/unittests/state-history-data/trace-log-v1/trace_history.index new file mode 100644 index 00000000000..f9a40fb098d Binary files /dev/null and b/unittests/state-history-data/trace-log-v1/trace_history.index differ diff --git a/unittests/state-history-data/trace-log-v1/trace_history.log b/unittests/state-history-data/trace-log-v1/trace_history.log new file mode 100644 index 00000000000..e60d20f3d75 Binary files /dev/null and b/unittests/state-history-data/trace-log-v1/trace_history.log differ diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp index 7cb83364a30..cbce759dab2 100644 --- a/unittests/state_history_tests.cpp +++ b/unittests/state_history_tests.cpp @@ -1,20 +1,18 @@ -#include #include #include +#include #include #include #include +#include #include #include #include "test_cfd_transaction.hpp" #include -#pragma push_macro("N") -#undef N -#include #include -#pragma pop_macro("N") +#include using namespace eosio::chain; using namespace eosio::testing; @@ -24,11 +22,13 @@ using prunable_data_type = eosio::chain::packed_transaction::prunable_data_type; namespace bio = boost::iostreams; extern const char* const state_history_plugin_abi; -prunable_data_type::prunable_data_t get_prunable_data_from_traces(std::vector& traces, - const transaction_id_type& id) { - auto cfd_trace_itr = std::find_if(traces.begin(), traces.end(), [id](const eosio::state_history::transaction_trace& v) { - return std::get(v).id == id; - }); +prunable_data_type::prunable_data_t +get_prunable_data_from_traces(std::vector& traces, + const transaction_id_type& id) { + auto cfd_trace_itr = + std::find_if(traces.begin(), traces.end(), [id](const eosio::state_history::transaction_trace& v) { + return std::get(v).id == id; + }); // make sure the trace with cfd can be found BOOST_REQUIRE(cfd_trace_itr != traces.end()); @@ -39,14 +39,45 @@ prunable_data_type::prunable_data_t get_prunable_data_from_traces(std::vector(*trace_v0.partial).prunable_data->prunable_data; } +bool operator==(const eosio::checksum256& lhs, const transaction_id_type& rhs) { + return memcmp(lhs.extract_as_byte_array().data(), rhs.data(), rhs.data_size()) == 0; +} + +auto get_prunable_data_from_traces(std::vector& traces, + const transaction_id_type& id) { + auto cfd_trace_itr = + std::find_if(traces.begin(), traces.end(), [id](const eosio::ship_protocol::transaction_trace& v) { + return std::get(v).id == id; + }); + + // make sure the trace with cfd can be found + BOOST_REQUIRE(cfd_trace_itr != traces.end()); + BOOST_REQUIRE(std::holds_alternative(*cfd_trace_itr)); + auto trace_v0 = std::get(*cfd_trace_itr); + BOOST_REQUIRE(trace_v0.partial); + BOOST_REQUIRE(std::holds_alternative(*trace_v0.partial)); + return std::get(*trace_v0.partial).prunable_data->prunable_data; +} + prunable_data_type::prunable_data_t get_prunable_data_from_traces_bin(const std::vector& entry, const transaction_id_type& id) { - fc::datastream strm(entry.data(), entry.size()); + fc::datastream strm(entry.data(), entry.size()); std::vector traces; eosio::state_history::trace_converter::unpack(strm, traces); return get_prunable_data_from_traces(traces, id); } +std::vector get_traces(eosio::state_history_traces_log& log, + block_num_type block_num) { + auto entry = log.get_log_entry(block_num); + std::vector traces; + if (entry.size()) { + eosio::input_stream traces_bin{entry.data(), entry.data() + entry.size()}; + BOOST_REQUIRE_NO_THROW(from_bin(traces, traces_bin)); + } + return traces; +} + struct state_history_abi_serializer { tester& chain; abi_serializer sr; @@ -132,10 +163,11 @@ BOOST_AUTO_TEST_CASE(test_trace_log) { auto cfd_trace = push_test_cfd_transaction(chain); chain.produce_blocks(1); - auto traces = log.get_traces(cfd_trace->block_num); + auto traces = get_traces(log, cfd_trace->block_num); BOOST_REQUIRE(traces.size()); - BOOST_REQUIRE(!std::holds_alternative(get_prunable_data_from_traces(traces, cfd_trace->id))); + BOOST_REQUIRE(!std::holds_alternative( + get_prunable_data_from_traces(traces, cfd_trace->id))); std::vector ids{cfd_trace->id}; log.prune_transactions(cfd_trace->block_num, ids); @@ -144,10 +176,24 @@ BOOST_AUTO_TEST_CASE(test_trace_log) { // we assume the nodeos has to be stopped while running, it can only be read // correctly with restart eosio::state_history_traces_log new_log({ .log_dir = state_history_dir.path }); - auto pruned_traces = new_log.get_traces(cfd_trace->block_num); + auto pruned_traces = get_traces(new_log, cfd_trace->block_num); BOOST_REQUIRE(pruned_traces.size()); - BOOST_CHECK(std::holds_alternative(get_prunable_data_from_traces(pruned_traces, cfd_trace->id))); + BOOST_CHECK(std::holds_alternative( + get_prunable_data_from_traces(pruned_traces, cfd_trace->id))); +} + +BOOST_AUTO_TEST_CASE(test_trace_log_versions) { + namespace bfs = boost::filesystem; + + for(std::string version : {"v0", "v1"}) { + tester chain; + eosio::state_history_traces_log log({ .log_dir = state_history_data_base_path() + "/trace-log-"+ version }); + + for(int i = 2; i <= 11; i++) { + auto traces = get_traces(log, i); + } + } } BOOST_AUTO_TEST_CASE(test_chain_state_log) { @@ -258,11 +304,11 @@ BOOST_AUTO_TEST_CASE(test_splitted_log) { BOOST_CHECK(bfs::exists( retained_dir / "chain_state_history-121-140.log" )); BOOST_CHECK(bfs::exists( retained_dir / "chain_state_history-121-140.index" )); - BOOST_CHECK(chain.traces_log.get_traces(10).empty()); - BOOST_CHECK(chain.traces_log.get_traces(100).size()); - BOOST_CHECK(chain.traces_log.get_traces(140).size()); - BOOST_CHECK(chain.traces_log.get_traces(150).size()); - BOOST_CHECK(chain.traces_log.get_traces(160).empty()); + BOOST_CHECK(get_traces(chain.traces_log, 10).empty()); + BOOST_CHECK(get_traces(chain.traces_log, 100).size()); + BOOST_CHECK(get_traces(chain.traces_log, 140).size()); + BOOST_CHECK(get_traces(chain.traces_log, 150).size()); + BOOST_CHECK(get_traces(chain.traces_log, 160).empty()); BOOST_CHECK(chain.chain_state_log.get_log_entry(10).empty()); BOOST_CHECK(chain.chain_state_log.get_log_entry(100).size()); @@ -270,10 +316,11 @@ BOOST_AUTO_TEST_CASE(test_splitted_log) { BOOST_CHECK(chain.chain_state_log.get_log_entry(150).size()); BOOST_CHECK(chain.chain_state_log.get_log_entry(160).empty()); - auto traces = chain.traces_log.get_traces(cfd_trace->block_num); + auto traces = get_traces(chain.traces_log, cfd_trace->block_num); BOOST_REQUIRE(traces.size()); - BOOST_REQUIRE(!std::holds_alternative(get_prunable_data_from_traces(traces, cfd_trace->id))); + BOOST_REQUIRE(!std::holds_alternative( + get_prunable_data_from_traces(traces, cfd_trace->id))); std::vector ids{cfd_trace->id}; chain.traces_log.prune_transactions(cfd_trace->block_num, ids); @@ -282,10 +329,11 @@ BOOST_AUTO_TEST_CASE(test_splitted_log) { // we assume the nodeos has to be stopped while running, it can only be read // correctly with restart eosio::state_history_traces_log new_log(config); - auto pruned_traces = new_log.get_traces(cfd_trace->block_num); + auto pruned_traces = get_traces(new_log, cfd_trace->block_num); BOOST_REQUIRE(pruned_traces.size()); - BOOST_CHECK(std::holds_alternative(get_prunable_data_from_traces(pruned_traces, cfd_trace->id))); + BOOST_CHECK(std::holds_alternative( + get_prunable_data_from_traces(pruned_traces, cfd_trace->id))); } void push_blocks( tester& from, tester& to ) { @@ -335,10 +383,10 @@ bool test_fork(uint32_t stride, uint32_t max_retained_files) { auto fb = chain2.control->fetch_block_by_number( start ); chain1.push_block( fb ); } - auto traces = chain1.traces_log.get_traces(b->block_num()); + auto traces = get_traces(chain1.traces_log, b->block_num()); bool trace_found = std::find_if(traces.begin(), traces.end(), [create_account_trace_id](const auto& v) { - return std::get(v).id == create_account_trace_id; + return std::get(v).id == create_account_trace_id; }) != traces.end(); return trace_found; @@ -390,7 +438,7 @@ BOOST_AUTO_TEST_CASE(test_corrupted_log_recovery) { state_history_tester new_chain(config); new_chain.produce_blocks(50); - BOOST_CHECK(new_chain.traces_log.get_traces(10).size()); + BOOST_CHECK(get_traces(new_chain.traces_log, 10).size()); BOOST_CHECK(new_chain.chain_state_log.get_log_entry(10).size()); } @@ -510,25 +558,26 @@ BOOST_AUTO_TEST_CASE(test_traces_present) BOOST_CHECK(onblock_test_executed); - auto traces = log.get_traces(tr_ptr->block_num); + auto traces = get_traces(log, tr_ptr->block_num); BOOST_REQUIRE_EQUAL(traces.size(), 1); - auto trace_itr = std::find_if(traces.begin(), traces.end(), [tr_ptr](const eosio::state_history::transaction_trace& v) { - return std::get(v).id == tr_ptr->id; - }); + auto trace_itr = + std::find_if(traces.begin(), traces.end(), [tr_ptr](const eosio::ship_protocol::transaction_trace& v) { + return std::get(v).id == tr_ptr->id; + }); BOOST_REQUIRE(trace_itr != traces.end()); - auto &action_traces = std::get(*trace_itr).action_traces; + auto& action_traces = std::get(*trace_itr).action_traces; - auto new_account_action_itr = std::find_if(action_traces.begin(), action_traces.end(), [](const eosio::state_history::action_trace& v) { - return std::get(v).act.name == "newaccount"_n.to_uint64_t(); - }); + auto new_account_action_itr = + std::find_if(action_traces.begin(), action_traces.end(), [](const eosio::ship_protocol::action_trace& v) { + return std::get(v).act.name.value == "newaccount"_n.to_uint64_t(); + }); BOOST_REQUIRE(new_account_action_itr != action_traces.end()); } - class table_deltas_tester : public tester { public: using tester::tester; diff --git a/unittests/utilities.hpp.in b/unittests/utilities.hpp.in new file mode 100644 index 00000000000..ff71a5ffbd1 --- /dev/null +++ b/unittests/utilities.hpp.in @@ -0,0 +1,9 @@ +#pragma once + +namespace eosio { + namespace testing { + std::string state_history_data_base_path() { + return "${CMAKE_BINARY_DIR}/unittests/state-history-data"; + } + } +}