From f6038e36a0d7e75fc0c8a71d39b7b92741fd1126 Mon Sep 17 00:00:00 2001 From: Jeeyong Um Date: Sat, 8 Sep 2018 00:40:55 +0900 Subject: [PATCH 001/369] Fix wrong full-board check in Tic-Tac-Toe contract --- contracts/tic_tac_toe/tic_tac_toe.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/tic_tac_toe/tic_tac_toe.cpp b/contracts/tic_tac_toe/tic_tac_toe.cpp index cdbdc3d098d..ebc5cc46203 100644 --- a/contracts/tic_tac_toe/tic_tac_toe.cpp +++ b/contracts/tic_tac_toe/tic_tac_toe.cpp @@ -51,7 +51,7 @@ account_name get_winner(const tic_tac_toe::game& current_game) { uint32_t consecutive_diagonal_backslash = 3; uint32_t consecutive_diagonal_slash = 3; for (uint32_t i = 0; i < board.size(); i++) { - is_board_full &= is_empty_cell(board[i]); + is_board_full &= !is_empty_cell(board[i]); uint16_t row = uint16_t(i / tic_tac_toe::game::board_width); uint16_t column = uint16_t(i % tic_tac_toe::game::board_width); From a8784aa84ea38d57a55c024044780f8c5c1ff69e Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Tue, 25 Sep 2018 17:57:17 -0400 Subject: [PATCH 002/369] state_history_plugin --- libraries/chain/include/eosio/chain/types.hpp | 1 + plugins/CMakeLists.txt | 1 + plugins/state_history_plugin/.clang-format | 8 + plugins/state_history_plugin/CMakeLists.txt | 7 + .../state_history_plugin.hpp | 45 ++ .../state_history_plugin.cpp | 431 ++++++++++++++++++ programs/nodeos/CMakeLists.txt | 1 + programs/nodeos/main.cpp | 2 + 8 files changed, 496 insertions(+) create mode 100644 plugins/state_history_plugin/.clang-format create mode 100644 plugins/state_history_plugin/CMakeLists.txt create mode 100644 plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp create mode 100644 plugins/state_history_plugin/state_history_plugin.cpp diff --git a/libraries/chain/include/eosio/chain/types.hpp b/libraries/chain/include/eosio/chain/types.hpp index 2a9117a99fc..431db291e70 100644 --- a/libraries/chain/include/eosio/chain/types.hpp +++ b/libraries/chain/include/eosio/chain/types.hpp @@ -153,6 +153,7 @@ namespace eosio { namespace chain { account_history_object_type, ///< Defined by history_plugin action_history_object_type, ///< Defined by history_plugin reversible_block_object_type, + state_history_object_type, ///< Defined by state_history_plugin OBJECT_TYPE_COUNT ///< Sentry value which contains the number of different object types }; diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 9b0b17b9d0a..8c93df9c48e 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -9,6 +9,7 @@ add_subdirectory(producer_plugin) add_subdirectory(producer_api_plugin) add_subdirectory(history_plugin) add_subdirectory(history_api_plugin) +add_subdirectory(state_history_plugin) add_subdirectory(wallet_plugin) add_subdirectory(wallet_api_plugin) diff --git a/plugins/state_history_plugin/.clang-format b/plugins/state_history_plugin/.clang-format new file mode 100644 index 00000000000..42dd5b7832c --- /dev/null +++ b/plugins/state_history_plugin/.clang-format @@ -0,0 +1,8 @@ +BasedOnStyle: LLVM +IndentWidth: 3 +ColumnLimit: 120 +PointerAlignment: Left +AlwaysBreakTemplateDeclarations: true +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: true +BreakConstructorInitializers: BeforeComma diff --git a/plugins/state_history_plugin/CMakeLists.txt b/plugins/state_history_plugin/CMakeLists.txt new file mode 100644 index 00000000000..3a6ef327305 --- /dev/null +++ b/plugins/state_history_plugin/CMakeLists.txt @@ -0,0 +1,7 @@ +file(GLOB HEADERS "include/eosio/state_history_plugin/*.hpp") +add_library( state_history_plugin + state_history_plugin.cpp + ${HEADERS} ) + +target_link_libraries( state_history_plugin chain_plugin eosio_chain appbase ) +target_include_directories( state_history_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp new file mode 100644 index 00000000000..a13a6978446 --- /dev/null +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -0,0 +1,45 @@ +/** + * @file + * @copyright defined in eos/LICENSE.txt + */ +#pragma once +#include + +#include + +namespace fc { +class variant; +} + +namespace eosio { +using chain::bytes; +using std::shared_ptr; + +typedef shared_ptr impl_ptr; + +struct table_delta { + std::string name{}; + std::vector> rows{}; + std::vector removed{}; +}; + +class state_history_plugin : public plugin { + public: + APPBASE_PLUGIN_REQUIRES((chain_plugin)) + + state_history_plugin(); + virtual ~state_history_plugin(); + + virtual void set_program_options(options_description& cli, options_description& cfg) override; + + void plugin_initialize(const variables_map& options); + void plugin_startup(); + void plugin_shutdown(); + + private: + impl_ptr my; +}; + +} // namespace eosio + +FC_REFLECT(eosio::table_delta, (name)(rows)(removed)); diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp new file mode 100644 index 00000000000..c1af7273ad1 --- /dev/null +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -0,0 +1,431 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +template +struct history_serial_wrapper { + const T& obj; +}; + +template +static history_serial_wrapper make_history_serial_wrapper(const T& obj) { + return {obj}; +} + +namespace fc { + +template +static const T& as_type(const T& x) { + return x; +} + +template +static void serialize_shared_vector(datastream& ds, const T& v) { + ds << fc::unsigned_int(v.size()); + for (auto& x : v) + ds << make_history_serial_wrapper(x); +} + +template +static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.name.value)); + fc::raw::pack(ds, as_type(obj.obj.vm_type)); + fc::raw::pack(ds, as_type(obj.obj.vm_version)); + fc::raw::pack(ds, as_type(obj.obj.privileged)); + fc::raw::pack(ds, as_type(obj.obj.last_code_update)); + fc::raw::pack(ds, as_type(obj.obj.code_version)); + fc::raw::pack(ds, as_type(obj.obj.creation_date)); + fc::raw::pack(ds, as_type(obj.obj.code)); + fc::raw::pack(ds, as_type(obj.obj.abi)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.name.value)); + fc::raw::pack(ds, as_type(obj.obj.recv_sequence)); + fc::raw::pack(ds, as_type(obj.obj.auth_sequence)); + fc::raw::pack(ds, as_type(obj.obj.code_sequence)); + fc::raw::pack(ds, as_type(obj.obj.abi_sequence)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.code.value)); + fc::raw::pack(ds, as_type(obj.obj.scope.value)); + fc::raw::pack(ds, as_type(obj.obj.table.value)); + fc::raw::pack(ds, as_type(obj.obj.payer.value)); + fc::raw::pack(ds, as_type(obj.obj.count)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.t_id._id)); + fc::raw::pack(ds, as_type(obj.obj.primary_key)); + fc::raw::pack(ds, as_type(obj.obj.payer.value)); + fc::raw::pack(ds, as_type(obj.obj.value)); + return ds; +} + +template +static void serialize_secondary_index_data(datastream& ds, const T& obj) { + fc::raw::pack(ds, obj); +} + +template +static void serialize_secondary_index_data(datastream& ds, const float64_t& obj) { + uint64_t i; + memcpy(&i, &obj, sizeof(i)); + fc::raw::pack(ds, i); +} + +template +static void serialize_secondary_index_data(datastream& ds, const float128_t& obj) { + __uint128_t i; + memcpy(&i, &obj, sizeof(i)); + fc::raw::pack(ds, i); +} + +template +static void serialize_secondary_index_data(datastream& ds, const eosio::chain::key256_t& obj) { + fc::raw::pack(ds, obj[0]); + fc::raw::pack(ds, obj[1]); +} + +template +static datastream& serialize_secondary_index(datastream& ds, const T& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.id._id)); + fc::raw::pack(ds, as_type(obj.t_id._id)); + fc::raw::pack(ds, as_type(obj.primary_key)); + fc::raw::pack(ds, as_type(obj.payer.value)); + serialize_secondary_index_data(ds, obj.secondary_key); + return ds; +} + +template +static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + return serialize_secondary_index(ds, obj.obj); +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + return serialize_secondary_index(ds, obj.obj); +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + return serialize_secondary_index(ds, obj.obj); +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + return serialize_secondary_index(ds, obj.obj); +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + return serialize_secondary_index(ds, obj.obj); +} + +template +static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.producer_name.value)); + fc::raw::pack(ds, as_type(obj.obj.block_signing_key)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.version)); + serialize_shared_vector(ds, obj.obj.producers); + return ds; +} + +template +static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.max_block_net_usage)); + fc::raw::pack(ds, as_type(obj.obj.target_block_net_usage_pct)); + fc::raw::pack(ds, as_type(obj.obj.max_transaction_net_usage)); + fc::raw::pack(ds, as_type(obj.obj.base_per_transaction_net_usage)); + fc::raw::pack(ds, as_type(obj.obj.net_usage_leeway)); + fc::raw::pack(ds, as_type(obj.obj.context_free_discount_net_usage_num)); + fc::raw::pack(ds, as_type(obj.obj.context_free_discount_net_usage_den)); + fc::raw::pack(ds, as_type(obj.obj.max_block_cpu_usage)); + fc::raw::pack(ds, as_type(obj.obj.target_block_cpu_usage_pct)); + fc::raw::pack(ds, as_type(obj.obj.max_transaction_cpu_usage)); + fc::raw::pack(ds, as_type(obj.obj.min_transaction_cpu_usage)); + fc::raw::pack(ds, as_type(obj.obj.max_transaction_lifetime)); + fc::raw::pack(ds, as_type(obj.obj.deferred_trx_expiration_window)); + fc::raw::pack(ds, as_type(obj.obj.max_transaction_delay)); + fc::raw::pack(ds, as_type(obj.obj.max_inline_action_size)); + fc::raw::pack(ds, as_type(obj.obj.max_inline_action_depth)); + fc::raw::pack(ds, as_type(obj.obj.max_authority_depth)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + + fc::raw::pack(ds, as_type>(obj.obj.proposed_schedule_block_num)); + fc::raw::pack(ds, make_history_serial_wrapper( + as_type(obj.obj.proposed_schedule))); + fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.configuration))); + + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.global_action_sequence)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.block_id)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.expiration)); + fc::raw::pack(ds, as_type(obj.obj.trx_id)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.trx_id)); + fc::raw::pack(ds, as_type(obj.obj.sender.value)); + fc::raw::pack(ds, as_type<__uint128_t>(obj.obj.sender_id)); + fc::raw::pack(ds, as_type(obj.obj.payer.value)); + fc::raw::pack(ds, as_type(obj.obj.delay_until)); + fc::raw::pack(ds, as_type(obj.obj.expiration)); + fc::raw::pack(ds, as_type(obj.obj.published)); + fc::raw::pack(ds, as_type(obj.obj.packed_trx)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.key)); + fc::raw::pack(ds, as_type(obj.obj.weight)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.actor.value)); + fc::raw::pack(ds, as_type(obj.obj.permission.value)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.permission)); + fc::raw::pack(ds, as_type(obj.obj.weight)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.wait_sec)); + fc::raw::pack(ds, as_type(obj.obj.weight)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.threshold)); + serialize_shared_vector(ds, obj.obj.keys); + serialize_shared_vector(ds, obj.obj.accounts); + serialize_shared_vector(ds, obj.obj.waits); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.usage_id._id)); + fc::raw::pack(ds, as_type(obj.obj.parent._id)); + fc::raw::pack(ds, as_type(obj.obj.owner.value)); + fc::raw::pack(ds, as_type(obj.obj.name.value)); + fc::raw::pack(ds, as_type(obj.obj.last_updated)); + fc::raw::pack(ds, as_type(obj.obj.auth)); + return ds; +} + +} // namespace fc + +namespace eosio { +using namespace chain; +using boost::signals2::scoped_connection; + +static appbase::abstract_plugin& _state_history_plugin = app().register_plugin(); + +struct state_history_object : public chainbase::object { + OBJECT_CTOR(state_history_object, (deltas)) + + id_type id; + shared_string deltas; +}; + +using state_history_index = chainbase::shared_multi_index_container< + state_history_object, + indexed_by, + member>>>; + +} // namespace eosio + +CHAINBASE_SET_INDEX_TYPE(eosio::state_history_object, eosio::state_history_index) + +namespace eosio { + +template +static void for_each_table(chainbase::database& db, F f) { + f("account_index", db.get_index()); + f("account_sequence_index", db.get_index()); + + f("table_id_multi_index", db.get_index()); + f("key_value_index", db.get_index()); + f("index64_index", db.get_index()); + f("index128_index", db.get_index()); + f("index256_index", db.get_index()); + f("index_double_index", db.get_index()); + f("index_long_double_index", db.get_index()); + + f("global_property_multi_index", db.get_index()); + f("dynamic_global_property_multi_index", db.get_index()); + f("block_summary_multi_index", db.get_index()); + f("transaction_multi_index", db.get_index()); + f("generated_transaction_multi_index", db.get_index()); + + f("permission_index", db.get_index()); + // f("permission_usage_index", db.get_index()); + // f("permission_link_index", db.get_index()); + + // f("resource_limits_index", db.get_index()); + // f("resource_usage_index", db.get_index()); + // f("resource_limits_state_index", db.get_index()); + // f("resource_limits_config_index", db.get_index()); +} + +class state_history_plugin_impl { + public: + chain_plugin* chain_plug = nullptr; + fc::optional accepted_block_connection; + + void on_accepted_block(const block_state_ptr& block_state) { + auto& chain = chain_plug->chain(); + auto& db = chain.db(); + dlog("${i}: ${n} transactions", + ("i", block_state->block->block_num())("n", block_state->block->transactions.size())); + db.create([&](state_history_object& hist) { + hist.id = block_state->block->block_num(); + std::vector deltas; + for_each_table(db, [&](auto* name, auto& index) { + if (index.stack().empty()) + return; + auto& undo = index.stack().back(); + if (undo.old_values.empty() && undo.new_ids.empty() && undo.removed_values.empty()) + return; + deltas.push_back({}); + auto& delta = deltas.back(); + delta.name = name; + dlog(" ${name} rev=${r} old=${o} new=${n} rem=${rem}", + ("name", name)("r", undo.revision)("o", undo.old_values.size())("n", undo.new_ids.size())( + "rem", undo.removed_values.size())); + for (auto& old : undo.old_values) + delta.rows.emplace_back(old.first._id, fc::raw::pack(make_history_serial_wrapper(index.get(old.first)))); + for (auto id : undo.new_ids) + delta.rows.emplace_back(id._id, fc::raw::pack(make_history_serial_wrapper(index.get(id)))); + for (auto& old : undo.removed_values) + delta.removed.push_back(old.first._id); + }); + auto bin = fc::raw::pack(deltas); + dlog(" ${s} bytes", ("s", bin.size())); + hist.deltas.insert(hist.deltas.end(), bin.begin(), bin.end()); + }); + } +}; + +state_history_plugin::state_history_plugin() + : my(std::make_shared()) {} + +state_history_plugin::~state_history_plugin() {} + +void state_history_plugin::set_program_options(options_description& cli, options_description& cfg) {} + +void state_history_plugin::plugin_initialize(const variables_map& options) { + try { + my->chain_plug = app().find_plugin(); + EOS_ASSERT(my->chain_plug, chain::missing_chain_plugin_exception, ""); + auto& chain = my->chain_plug->chain(); + chain.db().add_index(); + my->accepted_block_connection.emplace( + chain.accepted_block.connect([&](const block_state_ptr& p) { my->on_accepted_block(p); })); + } + FC_LOG_AND_RETHROW() +} + +void state_history_plugin::plugin_startup() {} + +void state_history_plugin::plugin_shutdown() { my->accepted_block_connection.reset(); } + +} // namespace eosio diff --git a/programs/nodeos/CMakeLists.txt b/programs/nodeos/CMakeLists.txt index 82ce6470789..6999d738901 100644 --- a/programs/nodeos/CMakeLists.txt +++ b/programs/nodeos/CMakeLists.txt @@ -51,6 +51,7 @@ target_link_libraries( ${NODE_EXECUTABLE_NAME} PRIVATE appbase PRIVATE -Wl,${whole_archive_flag} login_plugin -Wl,${no_whole_archive_flag} PRIVATE -Wl,${whole_archive_flag} history_plugin -Wl,${no_whole_archive_flag} + PRIVATE -Wl,${whole_archive_flag} state_history_plugin -Wl,${no_whole_archive_flag} PRIVATE -Wl,${whole_archive_flag} bnet_plugin -Wl,${no_whole_archive_flag} PRIVATE -Wl,${whole_archive_flag} history_api_plugin -Wl,${no_whole_archive_flag} PRIVATE -Wl,${whole_archive_flag} chain_api_plugin -Wl,${no_whole_archive_flag} diff --git a/programs/nodeos/main.cpp b/programs/nodeos/main.cpp index f17aa231105..73eb8329da7 100644 --- a/programs/nodeos/main.cpp +++ b/programs/nodeos/main.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -95,6 +96,7 @@ int main(int argc, char** argv) try { app().set_version(eosio::nodeos::config::version); app().register_plugin(); + app().register_plugin(); auto root = fc::app_path(); app().set_default_data_dir(root / "eosio/nodeos/data" ); From 708e574000e31279d35e632960590d010c182b2c Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Wed, 26 Sep 2018 09:33:56 -0400 Subject: [PATCH 003/369] state_history_plugin --- .../state_history_plugin.hpp | 3 +- .../state_history_plugin.cpp | 133 +++++++++++++++++- 2 files changed, 129 insertions(+), 7 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index a13a6978446..5799cef7771 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -18,6 +18,7 @@ using std::shared_ptr; typedef shared_ptr impl_ptr; struct table_delta { + fc::unsigned_int struct_version = 0; std::string name{}; std::vector> rows{}; std::vector removed{}; @@ -42,4 +43,4 @@ class state_history_plugin : public plugin { } // namespace eosio -FC_REFLECT(eosio::table_delta, (name)(rows)(removed)); +FC_REFLECT(eosio::table_delta, (struct_version)(name)(rows)(removed)); diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index c1af7273ad1..f992d0b22c7 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -3,7 +3,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -311,6 +314,124 @@ static datastream& operator<<(datastream& return ds; } +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.last_used)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.account.value)); + fc::raw::pack(ds, as_type(obj.obj.code.value)); + fc::raw::pack(ds, as_type(obj.obj.message_type.value)); + fc::raw::pack(ds, as_type(obj.obj.required_permission.value)); + return ds; +} + +template +static datastream& +operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.owner.value)); + fc::raw::pack(ds, as_type(obj.obj.pending)); + fc::raw::pack(ds, as_type(obj.obj.net_weight)); + fc::raw::pack(ds, as_type(obj.obj.cpu_weight)); + fc::raw::pack(ds, as_type(obj.obj.ram_bytes)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.last_ordinal)); + fc::raw::pack(ds, as_type(obj.obj.value_ex)); + fc::raw::pack(ds, as_type(obj.obj.consumed)); + return ds; +} + +template +static datastream& +operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.owner.value)); + fc::raw::pack( + ds, make_history_serial_wrapper(as_type(obj.obj.net_usage))); + fc::raw::pack( + ds, make_history_serial_wrapper(as_type(obj.obj.cpu_usage))); + fc::raw::pack(ds, as_type(obj.obj.ram_usage)); + return ds; +} + +template +static datastream& +operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, make_history_serial_wrapper( + as_type(obj.obj.average_block_net_usage))); + fc::raw::pack(ds, make_history_serial_wrapper( + as_type(obj.obj.average_block_cpu_usage))); + fc::raw::pack(ds, as_type(obj.obj.pending_net_usage)); + fc::raw::pack(ds, as_type(obj.obj.pending_cpu_usage)); + fc::raw::pack(ds, as_type(obj.obj.total_net_weight)); + fc::raw::pack(ds, as_type(obj.obj.total_cpu_weight)); + fc::raw::pack(ds, as_type(obj.obj.total_ram_bytes)); + fc::raw::pack(ds, as_type(obj.obj.virtual_net_limit)); + fc::raw::pack(ds, as_type(obj.obj.virtual_cpu_limit)); + return ds; +} + +template +static datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.numerator)); + fc::raw::pack(ds, as_type(obj.obj.denominator)); + return ds; +} + +template +static datastream& +operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.target)); + fc::raw::pack(ds, as_type(obj.obj.max)); + fc::raw::pack(ds, as_type(obj.obj.periods)); + fc::raw::pack(ds, as_type(obj.obj.max_multiplier)); + fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.contract_rate))); + fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.expand_rate))); + return ds; +} + +template +static datastream& +operator<<(datastream& ds, + const history_serial_wrapper& obj) { + ds << fc::unsigned_int(0); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, make_history_serial_wrapper(as_type( + obj.obj.cpu_limit_parameters))); + fc::raw::pack(ds, make_history_serial_wrapper(as_type( + obj.obj.net_limit_parameters))); + fc::raw::pack(ds, as_type(obj.obj.account_cpu_usage_average_window)); + fc::raw::pack(ds, as_type(obj.obj.account_net_usage_average_window)); + return ds; +}; + } // namespace fc namespace eosio { @@ -357,13 +478,13 @@ static void for_each_table(chainbase::database& db, F f) { f("generated_transaction_multi_index", db.get_index()); f("permission_index", db.get_index()); - // f("permission_usage_index", db.get_index()); - // f("permission_link_index", db.get_index()); + f("permission_usage_index", db.get_index()); + f("permission_link_index", db.get_index()); - // f("resource_limits_index", db.get_index()); - // f("resource_usage_index", db.get_index()); - // f("resource_limits_state_index", db.get_index()); - // f("resource_limits_config_index", db.get_index()); + f("resource_limits_index", db.get_index()); + f("resource_usage_index", db.get_index()); + f("resource_limits_state_index", db.get_index()); + f("resource_limits_config_index", db.get_index()); } class state_history_plugin_impl { From 5391c891b9316cadb3e9ecc57fda6422308ed37f Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Wed, 26 Sep 2018 18:36:48 -0400 Subject: [PATCH 004/369] state_history_plugin --- .../state_history_plugin.hpp | 15 +- .../state_history_plugin.cpp | 215 +++++++++++++++++- 2 files changed, 216 insertions(+), 14 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index 5799cef7771..e7142d90eba 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -15,7 +15,7 @@ namespace eosio { using chain::bytes; using std::shared_ptr; -typedef shared_ptr impl_ptr; +typedef shared_ptr state_history_ptr; struct table_delta { fc::unsigned_int struct_version = 0; @@ -24,6 +24,15 @@ struct table_delta { std::vector removed{}; }; +struct get_state_request_v0 {}; + +struct get_state_result_v0 { + uint32_t last_block_num = 0; +}; + +using state_request = fc::static_variant; +using state_result = fc::static_variant; + class state_history_plugin : public plugin { public: APPBASE_PLUGIN_REQUIRES((chain_plugin)) @@ -38,9 +47,11 @@ class state_history_plugin : public plugin { void plugin_shutdown(); private: - impl_ptr my; + state_history_ptr my; }; } // namespace eosio FC_REFLECT(eosio::table_delta, (struct_version)(name)(rows)(removed)); +FC_REFLECT_EMPTY(eosio::get_state_request_v0); +FC_REFLECT(eosio::get_state_result_v0, (last_block_num)); diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index f992d0b22c7..cf047b859c5 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -12,8 +12,17 @@ #include #include +#include +#include +#include +#include +#include +#include #include +using tcp = boost::asio::ip::tcp; +namespace ws = boost::beast::websocket; + template struct history_serial_wrapper { const T& obj; @@ -487,20 +496,170 @@ static void for_each_table(chainbase::database& db, F f) { f("resource_limits_config_index", db.get_index()); } -class state_history_plugin_impl { - public: +template +auto catch_and_log(F f) { + try { + return f(); + } catch (const fc::exception& e) { + elog("${e}", ("e", e.to_detail_string())); + } catch (const std::exception& e) { + elog("${e}", ("e", e.what())); + } catch (...) { + elog("unknown exception"); + } +} + +struct state_history_plugin_impl : std::enable_shared_from_this { chain_plugin* chain_plug = nullptr; fc::optional accepted_block_connection; + string endpoint_address = "0.0.0.0"; + uint16_t endpoint_port = 4321; + std::unique_ptr acceptor; + + struct session : std::enable_shared_from_this { + std::shared_ptr plugin; + std::unique_ptr> stream; + boost::beast::flat_buffer in_buffer; + + session(std::shared_ptr plugin) + : plugin(std::move(plugin)) {} + + void start(tcp::socket socket) { + ilog("incoming connection"); + stream = std::make_unique>(std::move(socket)); + stream->binary(true); + stream->next_layer().set_option(boost::asio::ip::tcp::no_delay(true)); + stream->next_layer().set_option(boost::asio::socket_base::send_buffer_size(1024 * 1024)); + stream->next_layer().set_option(boost::asio::socket_base::receive_buffer_size(1024 * 1024)); + stream->async_accept([self = shared_from_this(), this](boost::system::error_code ec) { + callback(ec, "async_accept", [&] { start_read(); }); + }); + } + + void start_read() { + stream->async_read(in_buffer, [self = shared_from_this(), this](boost::system::error_code ec, size_t bytes) { + callback(ec, "async_read", [&] { + ilog("read"); + auto d = boost::asio::buffer_cast(boost::beast::buffers_front(in_buffer.data())); + auto s = boost::asio::buffer_size(in_buffer.data()); + fc::datastream ds(d, s); + state_request req; + fc::raw::unpack(ds, req); + req.visit(*this); + start_read(); + }); + }); + } + + template + void send(const T obj) { + auto bin = std::make_shared>(fc::raw::pack(state_result{std::move(obj)})); + stream->async_write( // + boost::asio::buffer(*bin), + [self = shared_from_this(), this, bin](boost::system::error_code ec, size_t bytes) { + callback(ec, "async_write", [] {}); + }); + } + + using result_type = void; + void operator()(get_state_request_v0&) { + ilog("get_state_request_v0"); + get_state_result_v0 result; + auto& ind = plugin->chain_plug->chain().db().get_index().indices(); + if (!ind.empty()) + result.last_block_num = (--ind.end())->id._id; + send(std::move(result)); + } + + template + void catch_and_close(F f) { + try { + f(); + } catch (const fc::exception& e) { + elog("${e}", ("e", e.to_detail_string())); + close(); + } catch (const std::exception& e) { + elog("${e}", ("e", e.what())); + close(); + } catch (...) { + elog("unknown exception"); + close(); + } + } + + template + void callback(boost::system::error_code ec, const char* what, F f) { + if (ec) + return on_fail(ec, what); + catch_and_close(f); + } + + void on_fail(boost::system::error_code ec, const char* what) { + try { + elog("${w}: ${m}", ("w", what)("m", ec.message())); + close(); + } catch (...) { + elog("uncaught exception on close"); + } + } + + void close() { + stream->next_layer().close(); + plugin->sessions.erase(this); + } + }; + std::map> sessions; + + void listen() { + boost::system::error_code ec; + auto address = boost::asio::ip::make_address(endpoint_address); + auto endpoint = tcp::endpoint{address, endpoint_port}; + acceptor = std::make_unique(app().get_io_service()); + + auto check_ec = [&](const char* what) { + if (!ec) + return; + elog("${w}: ${m}", ("w", what)("m", ec.message())); + EOS_ASSERT(false, plugin_exception, "unable top open listen socket"); + }; + + acceptor->open(endpoint.protocol(), ec); + check_ec("open"); + acceptor->bind(endpoint, ec); + check_ec("bind"); + acceptor->listen(boost::asio::socket_base::max_listen_connections, ec); + check_ec("listen"); + do_accept(); + } + + void do_accept() { + auto socket = std::make_shared(app().get_io_service()); + acceptor->async_accept(*socket, [self = shared_from_this(), socket, this](auto ec) { + if (ec) { + if (ec == boost::system::errc::too_many_files_open) + catch_and_log([&] { do_accept(); }); + return; + } + catch_and_log([&] { + auto s = std::make_shared(self); + sessions[s.get()] = s; + s->start(std::move(*socket)); + }); + do_accept(); + }); + } + + void on_accept(boost::system::error_code ec) {} void on_accepted_block(const block_state_ptr& block_state) { auto& chain = chain_plug->chain(); auto& db = chain.db(); - dlog("${i}: ${n} transactions", - ("i", block_state->block->block_num())("n", block_state->block->transactions.size())); + // dlog("${i}: ${n} transactions", + // ("i", block_state->block->block_num())("n", block_state->block->transactions.size())); db.create([&](state_history_object& hist) { hist.id = block_state->block->block_num(); std::vector deltas; - for_each_table(db, [&](auto* name, auto& index) { + for_each_table(db, [&, this](auto* name, auto& index) { if (index.stack().empty()) return; auto& undo = index.stack().back(); @@ -509,9 +668,9 @@ class state_history_plugin_impl { deltas.push_back({}); auto& delta = deltas.back(); delta.name = name; - dlog(" ${name} rev=${r} old=${o} new=${n} rem=${rem}", - ("name", name)("r", undo.revision)("o", undo.old_values.size())("n", undo.new_ids.size())( - "rem", undo.removed_values.size())); + // dlog(" ${name} rev=${r} old=${o} new=${n} rem=${rem}", + // ("name", name)("r", undo.revision)("o", undo.old_values.size())("n", undo.new_ids.size())( + // "rem", undo.removed_values.size())); for (auto& old : undo.old_values) delta.rows.emplace_back(old.first._id, fc::raw::pack(make_history_serial_wrapper(index.get(old.first)))); for (auto id : undo.new_ids) @@ -520,7 +679,7 @@ class state_history_plugin_impl { delta.removed.push_back(old.first._id); }); auto bin = fc::raw::pack(deltas); - dlog(" ${s} bytes", ("s", bin.size())); + // dlog(" ${s} bytes", ("s", bin.size())); hist.deltas.insert(hist.deltas.end(), bin.begin(), bin.end()); }); } @@ -531,7 +690,10 @@ state_history_plugin::state_history_plugin() state_history_plugin::~state_history_plugin() {} -void state_history_plugin::set_program_options(options_description& cli, options_description& cfg) {} +void state_history_plugin::set_program_options(options_description& cli, options_description& cfg) { + cfg.add_options()("state-endpoint", bpo::value()->default_value("0.0.0.0:8080"), + "the endpoint upon which to listen for incoming connections"); +} void state_history_plugin::plugin_initialize(const variables_map& options) { try { @@ -541,12 +703,41 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { chain.db().add_index(); my->accepted_block_connection.emplace( chain.accepted_block.connect([&](const block_state_ptr& p) { my->on_accepted_block(p); })); + + if (options.count("state-endpoint")) { + auto ip_port = options.at("state-endpoint").as(); + auto port = ip_port.substr(ip_port.find(':') + 1, ip_port.size()); + auto host = ip_port.substr(0, ip_port.find(':')); + my->endpoint_address = host; + my->endpoint_port = std::stoi(port); + idump((ip_port)(host)(port)); + } + + // auto& ind = chain.db().get_index().indices(); + // size_t total = 0, max_size = 0, max_block = 0, num = 0; + // for (auto& x : ind) { + // if (!(x.id._id % 10000)) + // dlog(" ${i}: ${s} bytes", ("i", x.id._id)("s", x.deltas.size())); + // total += x.deltas.size(); + // if (x.deltas.size() > max_size) { + // max_block = x.id._id; + // max_size = x.deltas.size(); + // } + // ++num; + // } + // dlog("total: ${s} bytes", ("s", total)); + // dlog("max: ${s} bytes in ${b}", ("s", max_size)("b", max_block)); + // dlog("num: ${s}", ("s", num)); } FC_LOG_AND_RETHROW() } -void state_history_plugin::plugin_startup() {} +void state_history_plugin::plugin_startup() { my->listen(); } -void state_history_plugin::plugin_shutdown() { my->accepted_block_connection.reset(); } +void state_history_plugin::plugin_shutdown() { + my->accepted_block_connection.reset(); + while (!my->sessions.empty()) + my->sessions.begin()->second->close(); +} } // namespace eosio From 6db3ddefbeb6ee9b3d1a5350ceaf6d408a541ed1 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Thu, 27 Sep 2018 17:57:03 -0400 Subject: [PATCH 005/369] state_history_plugin --- plugins/state_history_plugin/CMakeLists.txt | 1 + .../state_history_plugin.hpp | 16 +- .../state_history_plugin.cpp | 119 ++++-- .../state_history_plugin_abi.cpp | 389 ++++++++++++++++++ 4 files changed, 479 insertions(+), 46 deletions(-) create mode 100644 plugins/state_history_plugin/state_history_plugin_abi.cpp diff --git a/plugins/state_history_plugin/CMakeLists.txt b/plugins/state_history_plugin/CMakeLists.txt index 3a6ef327305..21f0e10a900 100644 --- a/plugins/state_history_plugin/CMakeLists.txt +++ b/plugins/state_history_plugin/CMakeLists.txt @@ -1,6 +1,7 @@ file(GLOB HEADERS "include/eosio/state_history_plugin/*.hpp") add_library( state_history_plugin state_history_plugin.cpp + state_history_plugin_abi.cpp ${HEADERS} ) target_link_libraries( state_history_plugin chain_plugin eosio_chain appbase ) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index e7142d90eba..14cd1947b7d 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -30,8 +30,18 @@ struct get_state_result_v0 { uint32_t last_block_num = 0; }; -using state_request = fc::static_variant; -using state_result = fc::static_variant; +struct get_block_request_v0 { + uint32_t block_num = 0; +}; + +struct get_block_result_v0 { + uint32_t block_num = 0; + bool found = false; + bytes deltas; +}; + +using state_request = fc::static_variant; +using state_result = fc::static_variant; class state_history_plugin : public plugin { public: @@ -55,3 +65,5 @@ class state_history_plugin : public plugin { FC_REFLECT(eosio::table_delta, (struct_version)(name)(rows)(removed)); FC_REFLECT_EMPTY(eosio::get_state_request_v0); FC_REFLECT(eosio::get_state_result_v0, (last_block_num)); +FC_REFLECT(eosio::get_block_request_v0, (block_num)); +FC_REFLECT(eosio::get_block_result_v0, (block_num)(found)(deltas)); diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index cf047b859c5..44e4c066392 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -23,6 +23,8 @@ using tcp = boost::asio::ip::tcp; namespace ws = boost::beast::websocket; +extern const char* const state_history_plugin_abi; + template struct history_serial_wrapper { const T& obj; @@ -42,14 +44,14 @@ static const T& as_type(const T& x) { template static void serialize_shared_vector(datastream& ds, const T& v) { - ds << fc::unsigned_int(v.size()); + fc::raw::pack(ds, unsigned_int(v.size())); for (auto& x : v) ds << make_history_serial_wrapper(x); } template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.name.value)); fc::raw::pack(ds, as_type(obj.obj.vm_type)); @@ -66,7 +68,7 @@ static datastream& operator<<(datastream& ds, const history_serial_wrapp template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.name.value)); fc::raw::pack(ds, as_type(obj.obj.recv_sequence)); @@ -79,7 +81,7 @@ static datastream& operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.code.value)); fc::raw::pack(ds, as_type(obj.obj.scope.value)); @@ -92,7 +94,7 @@ static datastream& operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.t_id._id)); fc::raw::pack(ds, as_type(obj.obj.primary_key)); @@ -128,7 +130,7 @@ static void serialize_secondary_index_data(datastream& ds, const eosio::chai template static datastream& serialize_secondary_index(datastream& ds, const T& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.id._id)); fc::raw::pack(ds, as_type(obj.t_id._id)); fc::raw::pack(ds, as_type(obj.primary_key)); @@ -168,7 +170,7 @@ static datastream& operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.producer_name.value)); fc::raw::pack(ds, as_type(obj.obj.block_signing_key)); return ds; @@ -177,7 +179,7 @@ static datastream& operator<<(datastream& ds, const history_serial_wrapp template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.version)); serialize_shared_vector(ds, obj.obj.producers); return ds; @@ -185,7 +187,7 @@ static datastream& operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.max_block_net_usage)); fc::raw::pack(ds, as_type(obj.obj.target_block_net_usage_pct)); fc::raw::pack(ds, as_type(obj.obj.max_transaction_net_usage)); @@ -209,7 +211,7 @@ static datastream& operator<<(datastream& ds, const history_serial_wrapp template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type>(obj.obj.proposed_schedule_block_num)); @@ -223,7 +225,7 @@ static datastream& operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.global_action_sequence)); return ds; @@ -232,7 +234,7 @@ static datastream& operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.block_id)); return ds; @@ -241,7 +243,7 @@ static datastream& operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.expiration)); fc::raw::pack(ds, as_type(obj.obj.trx_id)); @@ -251,7 +253,7 @@ static datastream& operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.trx_id)); fc::raw::pack(ds, as_type(obj.obj.sender.value)); @@ -266,7 +268,7 @@ static datastream& operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.key)); fc::raw::pack(ds, as_type(obj.obj.weight)); return ds; @@ -275,7 +277,7 @@ static datastream& operator<<(datastream& ds, const history_serial_wrapp template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.actor.value)); fc::raw::pack(ds, as_type(obj.obj.permission.value)); return ds; @@ -284,7 +286,7 @@ static datastream& operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.permission)); fc::raw::pack(ds, as_type(obj.obj.weight)); return ds; @@ -292,7 +294,7 @@ static datastream& operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.wait_sec)); fc::raw::pack(ds, as_type(obj.obj.weight)); return ds; @@ -301,7 +303,7 @@ static datastream& operator<<(datastream& ds, const history_serial_wrapp template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.threshold)); serialize_shared_vector(ds, obj.obj.keys); serialize_shared_vector(ds, obj.obj.accounts); @@ -312,21 +314,21 @@ static datastream& operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.usage_id._id)); fc::raw::pack(ds, as_type(obj.obj.parent._id)); fc::raw::pack(ds, as_type(obj.obj.owner.value)); fc::raw::pack(ds, as_type(obj.obj.name.value)); fc::raw::pack(ds, as_type(obj.obj.last_updated)); - fc::raw::pack(ds, as_type(obj.obj.auth)); + fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.auth))); return ds; } template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.last_used)); return ds; @@ -335,7 +337,7 @@ static datastream& operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.account.value)); fc::raw::pack(ds, as_type(obj.obj.code.value)); @@ -348,7 +350,7 @@ template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.owner.value)); fc::raw::pack(ds, as_type(obj.obj.pending)); @@ -361,7 +363,7 @@ operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.last_ordinal)); fc::raw::pack(ds, as_type(obj.obj.value_ex)); fc::raw::pack(ds, as_type(obj.obj.consumed)); @@ -372,7 +374,7 @@ template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.owner.value)); fc::raw::pack( @@ -387,7 +389,7 @@ template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, make_history_serial_wrapper( as_type(obj.obj.average_block_net_usage))); @@ -406,7 +408,7 @@ operator<<(datastream& template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.numerator)); fc::raw::pack(ds, as_type(obj.obj.denominator)); return ds; @@ -416,7 +418,7 @@ template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.target)); fc::raw::pack(ds, as_type(obj.obj.max)); fc::raw::pack(ds, as_type(obj.obj.periods)); @@ -430,7 +432,7 @@ template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - ds << fc::unsigned_int(0); + fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, make_history_serial_wrapper(as_type( obj.obj.cpu_limit_parameters))); @@ -519,7 +521,6 @@ struct state_history_plugin_impl : std::enable_shared_from_this { std::shared_ptr plugin; std::unique_ptr> stream; - boost::beast::flat_buffer in_buffer; session(std::shared_ptr plugin) : plugin(std::move(plugin)) {} @@ -532,28 +533,39 @@ struct state_history_plugin_impl : std::enable_shared_from_thisnext_layer().set_option(boost::asio::socket_base::send_buffer_size(1024 * 1024)); stream->next_layer().set_option(boost::asio::socket_base::receive_buffer_size(1024 * 1024)); stream->async_accept([self = shared_from_this(), this](boost::system::error_code ec) { - callback(ec, "async_accept", [&] { start_read(); }); + callback(ec, "async_accept", [&] { + start_read(); + stream->async_write( // + boost::asio::buffer(std::string{state_history_plugin_abi}), + [self = shared_from_this(), this](boost::system::error_code ec, size_t bytes) { + callback(ec, "async_write", [] {}); + }); + }); }); } void start_read() { - stream->async_read(in_buffer, [self = shared_from_this(), this](boost::system::error_code ec, size_t bytes) { - callback(ec, "async_read", [&] { - ilog("read"); - auto d = boost::asio::buffer_cast(boost::beast::buffers_front(in_buffer.data())); - auto s = boost::asio::buffer_size(in_buffer.data()); - fc::datastream ds(d, s); - state_request req; - fc::raw::unpack(ds, req); - req.visit(*this); - start_read(); - }); - }); + auto in_buffer = std::make_shared(); + stream->async_read( + *in_buffer, [self = shared_from_this(), this, in_buffer](boost::system::error_code ec, size_t) { + callback(ec, "async_read", [&] { + ilog("read"); + auto d = boost::asio::buffer_cast(boost::beast::buffers_front(in_buffer->data())); + auto s = boost::asio::buffer_size(in_buffer->data()); + fc::datastream ds(d, s); + state_request req; + // ilog("request: ${b}", ("b", bytes(d, d + s))); + fc::raw::unpack(ds, req); + req.visit(*this); + start_read(); + }); + }); } template void send(const T obj) { auto bin = std::make_shared>(fc::raw::pack(state_result{std::move(obj)})); + // dlog(" result: ${b}", ("b", *bin)); stream->async_write( // boost::asio::buffer(*bin), [self = shared_from_this(), this, bin](boost::system::error_code ec, size_t bytes) { @@ -571,6 +583,19 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain_plug->chain().db().get_index().indices(); + auto it = ind.find(req.block_num); + if (it != ind.end()) { + result.found = true; + result.deltas.assign(it->deltas.begin(), it->deltas.end()); + // dlog(" bytes: ${b}", ("b", result.deltas)); + } + send(std::move(result)); + } + template void catch_and_close(F f) { try { @@ -625,6 +650,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisopen(endpoint.protocol(), ec); check_ec("open"); + acceptor->set_option(boost::asio::socket_base::reuse_address(true)); acceptor->bind(endpoint, ec); check_ec("bind"); acceptor->listen(boost::asio::socket_base::max_listen_connections, ec); @@ -677,6 +703,11 @@ struct state_history_plugin_impl : std::enable_shared_from_this Date: Thu, 27 Sep 2018 18:41:39 -0400 Subject: [PATCH 006/369] state_history_plugin --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index 9bfe5043f54..6eff122be85 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 9bfe5043f5484e00f89387091e9b5beb90b88c62 +Subproject commit 6eff122be853cd592ded7cb1c8b8ff8fe18143b4 From 609955ac5f3ce4604b5a61457ee0be7011e3e83b Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Fri, 28 Sep 2018 14:49:05 -0400 Subject: [PATCH 007/369] state_history_plugin --- libraries/chainbase | 2 +- .../state_history_plugin.cpp | 40 ++++++++++++------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/libraries/chainbase b/libraries/chainbase index 6eff122be85..373a692f09d 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 6eff122be853cd592ded7cb1c8b8ff8fe18143b4 +Subproject commit 373a692f09da8edcfa20b1460dae4aa6a288b783 diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 44e4c066392..dd854574b1f 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -521,6 +521,8 @@ struct state_history_plugin_impl : std::enable_shared_from_this { std::shared_ptr plugin; std::unique_ptr> stream; + bool sending = false; + std::vector> send_queue; session(std::shared_ptr plugin) : plugin(std::move(plugin)) {} @@ -535,11 +537,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisasync_accept([self = shared_from_this(), this](boost::system::error_code ec) { callback(ec, "async_accept", [&] { start_read(); - stream->async_write( // - boost::asio::buffer(std::string{state_history_plugin_abi}), - [self = shared_from_this(), this](boost::system::error_code ec, size_t bytes) { - callback(ec, "async_write", [] {}); - }); + send(state_history_plugin_abi); }); }); } @@ -549,12 +547,10 @@ struct state_history_plugin_impl : std::enable_shared_from_thisasync_read( *in_buffer, [self = shared_from_this(), this, in_buffer](boost::system::error_code ec, size_t) { callback(ec, "async_read", [&] { - ilog("read"); auto d = boost::asio::buffer_cast(boost::beast::buffers_front(in_buffer->data())); auto s = boost::asio::buffer_size(in_buffer->data()); fc::datastream ds(d, s); state_request req; - // ilog("request: ${b}", ("b", bytes(d, d + s))); fc::raw::unpack(ds, req); req.visit(*this); start_read(); @@ -562,20 +558,34 @@ struct state_history_plugin_impl : std::enable_shared_from_this - void send(const T obj) { - auto bin = std::make_shared>(fc::raw::pack(state_result{std::move(obj)})); - // dlog(" result: ${b}", ("b", *bin)); + void send(T obj) { + send_queue.push_back(fc::raw::pack(state_result{std::move(obj)})); + send(); + } + + void send() { + if (sending || send_queue.empty()) + return; + sending = true; stream->async_write( // - boost::asio::buffer(*bin), - [self = shared_from_this(), this, bin](boost::system::error_code ec, size_t bytes) { - callback(ec, "async_write", [] {}); + boost::asio::buffer(send_queue[0]), + [self = shared_from_this(), this](boost::system::error_code ec, size_t) { + callback(ec, "async_write", [&] { + send_queue.erase(send_queue.begin()); + sending = false; + send(); + }); }); } using result_type = void; void operator()(get_state_request_v0&) { - ilog("get_state_request_v0"); get_state_result_v0 result; auto& ind = plugin->chain_plug->chain().db().get_index().indices(); if (!ind.empty()) @@ -584,7 +594,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain_plug->chain().db().get_index().indices(); auto it = ind.find(req.block_num); From 92bbacf80996527d7a88a38f5293b7a25b2c9c9d Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Tue, 2 Oct 2018 10:04:18 -0400 Subject: [PATCH 008/369] state history plugin --- plugins/state_history_plugin/state_history_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index dd854574b1f..1a6d00542b1 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -287,7 +287,7 @@ template static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.permission)); + fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.permission))); fc::raw::pack(ds, as_type(obj.obj.weight)); return ds; } From 5355d706869df694cf6eded15f405a39738fd06c Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Tue, 2 Oct 2018 10:52:40 -0400 Subject: [PATCH 009/369] state history plugin --- .../state_history_plugin.cpp | 44 +++--- .../state_history_plugin_abi.cpp | 126 +++++++++--------- 2 files changed, 86 insertions(+), 84 deletions(-) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 1a6d00542b1..878b72e58f7 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -471,31 +471,31 @@ namespace eosio { template static void for_each_table(chainbase::database& db, F f) { - f("account_index", db.get_index()); - f("account_sequence_index", db.get_index()); + f("account", db.get_index()); + f("account_sequence", db.get_index()); - f("table_id_multi_index", db.get_index()); - f("key_value_index", db.get_index()); - f("index64_index", db.get_index()); - f("index128_index", db.get_index()); - f("index256_index", db.get_index()); - f("index_double_index", db.get_index()); - f("index_long_double_index", db.get_index()); + f("table_id", db.get_index()); + f("key_value", db.get_index()); + f("index64", db.get_index()); + f("index128", db.get_index()); + f("index256", db.get_index()); + f("index_double", db.get_index()); + f("index_long_double", db.get_index()); - f("global_property_multi_index", db.get_index()); - f("dynamic_global_property_multi_index", db.get_index()); - f("block_summary_multi_index", db.get_index()); - f("transaction_multi_index", db.get_index()); - f("generated_transaction_multi_index", db.get_index()); + f("global_property", db.get_index()); + f("dynamic_global_property", db.get_index()); + f("block_summary", db.get_index()); + f("transaction", db.get_index()); + f("generated_transaction", db.get_index()); - f("permission_index", db.get_index()); - f("permission_usage_index", db.get_index()); - f("permission_link_index", db.get_index()); + f("permission", db.get_index()); + f("permission_usage", db.get_index()); + f("permission_link", db.get_index()); - f("resource_limits_index", db.get_index()); - f("resource_usage_index", db.get_index()); - f("resource_limits_state_index", db.get_index()); - f("resource_limits_config_index", db.get_index()); + f("resource_limits", db.get_index()); + f("resource_usage", db.get_index()); + f("resource_limits_state", db.get_index()); + f("resource_limits_config", db.get_index()); } template @@ -738,6 +738,8 @@ void state_history_plugin::set_program_options(options_description& cli, options void state_history_plugin::plugin_initialize(const variables_map& options) { try { + // todo: check for --disable-replay-opts + my->chain_plug = app().find_plugin(); EOS_ASSERT(my->chain_plug, chain::missing_chain_plugin_exception, ""); auto& chain = my->chain_plug->chain(); diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index 1cbf234bae9..881c63c3084 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -35,7 +35,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "account_object_v0", "fields": [ + "name": "account_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "name", "name": "name" }, { "type": "uint8", "name": "vm_type" }, @@ -49,7 +49,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "account_sequence_object_v0", "fields": [ + "name": "account_sequence_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "name", "name": "name" }, { "type": "uint64", "name": "recv_sequence" }, @@ -59,7 +59,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "table_id_object_v0", "fields": [ + "name": "table_id_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "name", "name": "code" }, { "type": "name", "name": "scope" }, @@ -69,7 +69,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "key_value_object_v0", "fields": [ + "name": "key_value_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "uint64", "name": "t_id" }, { "type": "uint64", "name": "primary_key" }, @@ -78,7 +78,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "index64_object_v0", "fields": [ + "name": "index64_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "uint64", "name": "t_id" }, { "type": "uint64", "name": "primary_key" }, @@ -87,7 +87,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "index128_object_v0", "fields": [ + "name": "index128_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "uint64", "name": "t_id" }, { "type": "uint64", "name": "primary_key" }, @@ -102,7 +102,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "index256_object_v0", "fields": [ + "name": "index256_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "uint64", "name": "t_id" }, { "type": "uint64", "name": "primary_key" }, @@ -111,7 +111,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "index_double_object_v0", "fields": [ + "name": "index_double_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "uint64", "name": "t_id" }, { "type": "uint64", "name": "primary_key" }, @@ -120,7 +120,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "index_long_double_object_v0", "fields": [ + "name": "index_long_double_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "uint64", "name": "t_id" }, { "type": "uint64", "name": "primary_key" }, @@ -162,7 +162,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "global_property_object_v0", "fields": [ + "name": "global_property_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "uint32?", "name": "proposed_schedule_block_num" }, { "type": "shared_producer_schedule_type", "name": "proposed_schedule" }, @@ -170,26 +170,26 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "dynamic_global_property_object_v0", "fields": [ + "name": "dynamic_global_property_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "uint64", "name": "global_action_sequence" } ] }, { - "name": "block_summary_object_v0", "fields": [ + "name": "block_summary_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "checksum256", "name": "block_id" } ] }, { - "name": "transaction_object_v0", "fields": [ + "name": "transaction_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "time_point_sec", "name": " expiration" }, { "type": "checksum256", "name": "trx_id" } ] }, { - "name": "generated_transaction_object_v0", "fields": [ + "name": "generated_transaction_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "checksum256", "name": "trx_id" }, { "type": "name", "name": "sender" }, @@ -234,7 +234,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "permission_object_v0", "fields": [ + "name": "permission_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "uint64", "name": "usage_id" }, { "type": "uint64", "name": "parent" }, @@ -245,13 +245,13 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "permission_usage_object_v0", "fields": [ + "name": "permission_usage_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "time_point", "name": " last_used" } ] }, { - "name": "permission_link_object_v0", "fields": [ + "name": "permission_link_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "name", "name": "account" }, { "type": "name", "name": "code" }, @@ -260,7 +260,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "resource_limits_object_v0", "fields": [ + "name": "resource_limits_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "name", "name": "owner" }, { "type": "bool", "name": "pending" }, @@ -277,7 +277,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "resource_usage_object_v0", "fields": [ + "name": "resource_usage_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "name", "name": "owner" }, { "type": "usage_accumulator", "name": "net_usage" }, @@ -286,7 +286,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "resource_limits_state_object_v0", "fields": [ + "name": "resource_limits_state_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "usage_accumulator", "name": "average_block_net_usage" }, { "type": "usage_accumulator", "name": "average_block_cpu_usage" }, @@ -316,7 +316,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "resource_limits_config_object_v0", "fields": [ + "name": "resource_limits_config_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "elastic_limit_parameters", "name": "cpu_limit_parameters" }, { "type": "elastic_limit_parameters", "name": "net_limit_parameters" }, @@ -330,60 +330,60 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "result", "types": ["get_state_result_v0", "get_block_result_v0"] }, { "name": "table_delta", "types": ["table_delta_v0"] }, - { "name": "account_object", "types": ["account_object_v0"] }, - { "name": "account_sequence_object", "types": ["account_sequence_object_v0"] }, - { "name": "table_id_object", "types": ["table_id_object_v0"] }, - { "name": "key_value_object", "types": ["key_value_object_v0"] }, - { "name": "index64_object", "types": ["index64_object_v0"] }, - { "name": "index128_object", "types": ["index128_object_v0"] }, - { "name": "index256_object", "types": ["index256_object_v0"] }, - { "name": "index_double_object", "types": ["index_double_object_v0"] }, - { "name": "index_long_double_object", "types": ["index_long_double_object_v0"] }, + { "name": "account", "types": ["account_v0"] }, + { "name": "account_sequence", "types": ["account_sequence_v0"] }, + { "name": "table_id", "types": ["table_id_v0"] }, + { "name": "key_value", "types": ["key_value_v0"] }, + { "name": "index64", "types": ["index64_v0"] }, + { "name": "index128", "types": ["index128_v0"] }, + { "name": "index256", "types": ["index256_v0"] }, + { "name": "index_double", "types": ["index_double_v0"] }, + { "name": "index_long_double", "types": ["index_long_double_v0"] }, { "name": "producer_key", "types": ["producer_key_v0"] }, { "name": "shared_producer_schedule_type", "types": ["shared_producer_schedule_type_v0"] }, { "name": "chain_config", "types": ["chain_config_v0"] }, - { "name": "global_property_object", "types": ["global_property_object_v0"] }, - { "name": "dynamic_global_property_object", "types": ["dynamic_global_property_object_v0"] }, - { "name": "block_summary_object", "types": ["block_summary_object_v0"] }, - { "name": "transaction_object", "types": ["transaction_object_v0"] }, - { "name": "generated_transaction_object", "types": ["generated_transaction_object_v0"] }, + { "name": "global_property", "types": ["global_property_v0"] }, + { "name": "dynamic_global_property", "types": ["dynamic_global_property_v0"] }, + { "name": "block_summary", "types": ["block_summary_v0"] }, + { "name": "transaction", "types": ["transaction_v0"] }, + { "name": "generated_transaction", "types": ["generated_transaction_v0"] }, { "name": "key_weight", "types": ["key_weight_v0"] }, { "name": "permission_level", "types": ["permission_level_v0"] }, { "name": "permission_level_weight", "types": ["permission_level_weight_v0"] }, { "name": "wait_weight", "types": ["wait_weight_v0"] }, { "name": "shared_authority", "types": ["shared_authority_v0"] }, - { "name": "permission_object", "types": ["permission_object_v0"] }, - { "name": "permission_usage_object", "types": ["permission_usage_object_v0"] }, - { "name": "permission_link_object", "types": ["permission_link_object_v0"] }, - { "name": "resource_limits_object", "types": ["resource_limits_object_v0"] }, + { "name": "permission", "types": ["permission_v0"] }, + { "name": "permission_usage", "types": ["permission_usage_v0"] }, + { "name": "permission_link", "types": ["permission_link_v0"] }, + { "name": "resource_limits", "types": ["resource_limits_v0"] }, { "name": "usage_accumulator", "types": ["usage_accumulator_v0"] }, - { "name": "resource_usage_object", "types": ["resource_usage_object_v0"] }, - { "name": "resource_limits_state_object", "types": ["resource_limits_state_object_v0"] }, + { "name": "resource_usage", "types": ["resource_usage_v0"] }, + { "name": "resource_limits_state", "types": ["resource_limits_state_v0"] }, { "name": "resource_limits_ratio", "types": ["resource_limits_ratio_v0"] }, { "name": "elastic_limit_parameters", "types": ["elastic_limit_parameters_v0"] }, - { "name": "resource_limits_config_object", "types": ["resource_limits_config_object_v0"] } + { "name": "resource_limits_config", "types": ["resource_limits_config_v0"] } ], "tables": [ - { "name": "account_index", "type": "account_object" }, - { "name": "account_sequence_index", "type": "account_sequence_object" }, - { "name": "table_id_multi_index", "type": "table_id_object" }, - { "name": "key_value_index", "type": "key_value_object" }, - { "name": "index64_index", "type": "index64_object" }, - { "name": "index128_index", "type": "index128_object" }, - { "name": "index256_index", "type": "index256_object" }, - { "name": "index_double_index", "type": "index_double_object" }, - { "name": "index_long_double_index", "type": "index_long_double_object" }, - { "name": "global_property_multi_index", "type": "global_property_object" }, - { "name": "dynamic_global_property_multi_index", "type": "dynamic_global_property_object" }, - { "name": "block_summary_multi_index", "type": "block_summary_object" }, - { "name": "transaction_multi_index", "type": "transaction_object" }, - { "name": "generated_transaction_multi_index", "type": "generated_transaction_object" }, - { "name": "permission_index", "type": "permission_object" }, - { "name": "permission_usage_index", "type": "permission_usage_object" }, - { "name": "permission_link_index", "type": "permission_link_object" }, - { "name": "resource_limits_index", "type": "resource_limits_object" }, - { "name": "resource_usage_index", "type": "resource_usage_object" }, - { "name": "resource_limits_state_index", "type": "resource_limits_state_object" }, - { "name": "resource_limits_config_index", "type": "resource_limits_config_object" } + { "name": "account", "type": "account" }, + { "name": "account_sequence", "type": "account_sequence" }, + { "name": "table_id", "type": "table_id" }, + { "name": "key_value", "type": "key_value" }, + { "name": "index64", "type": "index64" }, + { "name": "index128", "type": "index128" }, + { "name": "index256", "type": "index256" }, + { "name": "index_double", "type": "index_double" }, + { "name": "index_long_double", "type": "index_long_double" }, + { "name": "global_property", "type": "global_property" }, + { "name": "dynamic_global_property", "type": "dynamic_global_property" }, + { "name": "block_summary", "type": "block_summary" }, + { "name": "transaction", "type": "transaction" }, + { "name": "generated_transaction", "type": "generated_transaction" }, + { "name": "permission", "type": "permission" }, + { "name": "permission_usage", "type": "permission_usage" }, + { "name": "permission_link", "type": "permission_link" }, + { "name": "resource_limits", "type": "resource_limits" }, + { "name": "resource_usage", "type": "resource_usage" }, + { "name": "resource_limits_state", "type": "resource_limits_state" }, + { "name": "resource_limits_config", "type": "resource_limits_config" } ] })"; From b4d648aa4b1554049f0ca4e6d5e447befd415fc1 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Tue, 2 Oct 2018 14:49:56 -0400 Subject: [PATCH 010/369] state history plugin: move to separate db --- .../state_history_plugin.cpp | 62 ++++++++++++------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 878b72e58f7..9d26c7234c5 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -512,11 +512,12 @@ auto catch_and_log(F f) { } struct state_history_plugin_impl : std::enable_shared_from_this { - chain_plugin* chain_plug = nullptr; - fc::optional accepted_block_connection; - string endpoint_address = "0.0.0.0"; - uint16_t endpoint_port = 4321; - std::unique_ptr acceptor; + chain_plugin* chain_plug = nullptr; + std::unique_ptr db; + fc::optional accepted_block_connection; + string endpoint_address = "0.0.0.0"; + uint16_t endpoint_port = 4321; + std::unique_ptr acceptor; struct session : std::enable_shared_from_this { std::shared_ptr plugin; @@ -559,6 +560,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain_plug->chain().db().get_index().indices(); + auto& ind = plugin->db->get_index().indices(); if (!ind.empty()) result.last_block_num = (--ind.end())->id._id; send(std::move(result)); @@ -596,7 +598,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain_plug->chain().db().get_index().indices(); + auto& ind = plugin->db->get_index().indices(); auto it = ind.find(req.block_num); if (it != ind.end()) { result.found = true; @@ -689,13 +691,12 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain(); - auto& db = chain.db(); // dlog("${i}: ${n} transactions", // ("i", block_state->block->block_num())("n", block_state->block->transactions.size())); - db.create([&](state_history_object& hist) { + db->create([&](state_history_object& hist) { hist.id = block_state->block->block_num(); std::vector deltas; - for_each_table(db, [&, this](auto* name, auto& index) { + for_each_table(chain.db(), [&, this](auto* name, auto& index) { if (index.stack().empty()) return; auto& undo = index.stack().back(); @@ -732,8 +733,14 @@ state_history_plugin::state_history_plugin() state_history_plugin::~state_history_plugin() {} void state_history_plugin::set_program_options(options_description& cli, options_description& cfg) { - cfg.add_options()("state-endpoint", bpo::value()->default_value("0.0.0.0:8080"), - "the endpoint upon which to listen for incoming connections"); + auto options = cfg.add_options(); + options("state-history-dir", bpo::value()->default_value("state-history"), + "the location of the state-history directory (absolute path or relative to application data dir)"); + options("state-history-db-size-mb", bpo::value()->default_value(1024), + "Maximum size (in MiB) of the state history database"); + options("delete-state-history", bpo::bool_switch()->default_value(false), "clear state history database"); + options("state-history-endpoint", bpo::value()->default_value("0.0.0.0:8080"), + "the endpoint upon which to listen for incoming connections"); } void state_history_plugin::plugin_initialize(const variables_map& options) { @@ -743,20 +750,33 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { my->chain_plug = app().find_plugin(); EOS_ASSERT(my->chain_plug, chain::missing_chain_plugin_exception, ""); auto& chain = my->chain_plug->chain(); - chain.db().add_index(); my->accepted_block_connection.emplace( chain.accepted_block.connect([&](const block_state_ptr& p) { my->on_accepted_block(p); })); - if (options.count("state-endpoint")) { - auto ip_port = options.at("state-endpoint").as(); - auto port = ip_port.substr(ip_port.find(':') + 1, ip_port.size()); - auto host = ip_port.substr(0, ip_port.find(':')); - my->endpoint_address = host; - my->endpoint_port = std::stoi(port); - idump((ip_port)(host)(port)); + auto dir_option = options.at("state-history-dir").as(); + boost::filesystem::path state_history_dir; + if (dir_option.is_relative()) + state_history_dir = app().data_dir() / dir_option; + else + state_history_dir = dir_option; + auto state_history_size = options.at("state-history-db-size-mb").as() * 1024 * 1024; + + auto ip_port = options.at("state-history-endpoint").as(); + auto port = ip_port.substr(ip_port.find(':') + 1, ip_port.size()); + auto host = ip_port.substr(0, ip_port.find(':')); + my->endpoint_address = host; + my->endpoint_port = std::stoi(port); + idump((ip_port)(host)(port)); + + if (options.at("delete-state-history").as()) { + ilog("Deleting state history"); + boost::filesystem::remove_all(state_history_dir); } - // auto& ind = chain.db().get_index().indices(); + my->db = std::make_unique(state_history_dir, database::read_write, state_history_size); + my->db->add_index(); + + // auto& ind = my->db->get_index().indices(); // size_t total = 0, max_size = 0, max_block = 0, num = 0; // for (auto& x : ind) { // if (!(x.id._id % 10000)) From 637a7fb122b513c6f5cc6ff8a585a75db754dcbe Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Tue, 2 Oct 2018 16:17:28 -0400 Subject: [PATCH 011/369] state history --- libraries/chain/include/eosio/chain/types.hpp | 1 - plugins/state_history_plugin/state_history_plugin.cpp | 9 ++++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libraries/chain/include/eosio/chain/types.hpp b/libraries/chain/include/eosio/chain/types.hpp index 431db291e70..2a9117a99fc 100644 --- a/libraries/chain/include/eosio/chain/types.hpp +++ b/libraries/chain/include/eosio/chain/types.hpp @@ -153,7 +153,6 @@ namespace eosio { namespace chain { account_history_object_type, ///< Defined by history_plugin action_history_object_type, ///< Defined by history_plugin reversible_block_object_type, - state_history_object_type, ///< Defined by state_history_plugin OBJECT_TYPE_COUNT ///< Sentry value which contains the number of different object types }; diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 9d26c7234c5..2b6bf3a8135 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -451,7 +451,14 @@ using boost::signals2::scoped_connection; static appbase::abstract_plugin& _state_history_plugin = app().register_plugin(); -struct state_history_object : public chainbase::object { +enum class state_history_db_object_type { + null_object_type = 0, + state_history_object_type, +}; + +struct state_history_object + : public chainbase::object<(uint16_t)state_history_db_object_type::state_history_object_type, + state_history_object> { OBJECT_CTOR(state_history_object, (deltas)) id_type id; From f9b08408a74ecf038def884c0d4d66c79912129a Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Tue, 2 Oct 2018 17:26:38 -0400 Subject: [PATCH 012/369] state history: switch forks --- .../state_history_plugin.cpp | 65 ++++++++++++------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 2b6bf3a8135..26372d30477 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -698,34 +698,51 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain(); - // dlog("${i}: ${n} transactions", - // ("i", block_state->block->block_num())("n", block_state->block->transactions.size())); + auto& idx = db->get_index(); + + uint64_t num_removed = 0; + while (!idx.indices().empty() && (--idx.indices().end())->id._id >= block_state->block->block_num()) { + db->remove(*--idx.indices().end()); + ++num_removed; + } + if (num_removed) + ilog("fork: removed ${n} blocks", ("n", num_removed)); + EOS_ASSERT(idx.indices().empty() || (--idx.indices().end())->id._id + 1 == block_state->block->block_num(), + plugin_exception, "missed a fork switch"); + + bool fresh = idx.indices().empty(); + if (fresh) + ilog("start fresh at block ${n}", ("n", block_state->block->block_num())); + db->create([&](state_history_object& hist) { hist.id = block_state->block->block_num(); std::vector deltas; for_each_table(chain.db(), [&, this](auto* name, auto& index) { - if (index.stack().empty()) - return; - auto& undo = index.stack().back(); - if (undo.old_values.empty() && undo.new_ids.empty() && undo.removed_values.empty()) - return; - deltas.push_back({}); - auto& delta = deltas.back(); - delta.name = name; - // dlog(" ${name} rev=${r} old=${o} new=${n} rem=${rem}", - // ("name", name)("r", undo.revision)("o", undo.old_values.size())("n", undo.new_ids.size())( - // "rem", undo.removed_values.size())); - for (auto& old : undo.old_values) - delta.rows.emplace_back(old.first._id, fc::raw::pack(make_history_serial_wrapper(index.get(old.first)))); - for (auto id : undo.new_ids) - delta.rows.emplace_back(id._id, fc::raw::pack(make_history_serial_wrapper(index.get(id)))); - for (auto& old : undo.removed_values) - delta.removed.push_back(old.first._id); - // if (name == std::string("account_index")) { - // dlog("account_sequence_index"); - // for (auto& r : delta.rows) - // dlog(" ${id} ${bytes}", ("id", r.first)("bytes", r.second)); - // } + if (fresh) { + if (index.indices().empty()) + return; + deltas.push_back({}); + auto& delta = deltas.back(); + delta.name = name; + for (auto& row : index.indices()) + delta.rows.emplace_back(row.id._id, fc::raw::pack(make_history_serial_wrapper(row))); + } else { + if (index.stack().empty()) + return; + auto& undo = index.stack().back(); + if (undo.old_values.empty() && undo.new_ids.empty() && undo.removed_values.empty()) + return; + deltas.push_back({}); + auto& delta = deltas.back(); + delta.name = name; + for (auto& old : undo.old_values) + delta.rows.emplace_back(old.first._id, + fc::raw::pack(make_history_serial_wrapper(index.get(old.first)))); + for (auto id : undo.new_ids) + delta.rows.emplace_back(id._id, fc::raw::pack(make_history_serial_wrapper(index.get(id)))); + for (auto& old : undo.removed_values) + delta.removed.push_back(old.first._id); + } }); auto bin = fc::raw::pack(deltas); // dlog(" ${s} bytes", ("s", bin.size())); From df12ea2db8fcb47205a774ce24d1431c7f5080d3 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Tue, 2 Oct 2018 18:45:22 -0400 Subject: [PATCH 013/369] state history plugin --- .../state_history_plugin.cpp | 86 +++++++++++-------- 1 file changed, 52 insertions(+), 34 deletions(-) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 26372d30477..52aeaa597c4 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -477,32 +477,34 @@ CHAINBASE_SET_INDEX_TYPE(eosio::state_history_object, eosio::state_history_index namespace eosio { template -static void for_each_table(chainbase::database& db, F f) { - f("account", db.get_index()); - f("account_sequence", db.get_index()); - - f("table_id", db.get_index()); - f("key_value", db.get_index()); - f("index64", db.get_index()); - f("index128", db.get_index()); - f("index256", db.get_index()); - f("index_double", db.get_index()); - f("index_long_double", db.get_index()); - - f("global_property", db.get_index()); - f("dynamic_global_property", db.get_index()); - f("block_summary", db.get_index()); - f("transaction", db.get_index()); - f("generated_transaction", db.get_index()); - - f("permission", db.get_index()); - f("permission_usage", db.get_index()); - f("permission_link", db.get_index()); - - f("resource_limits", db.get_index()); - f("resource_usage", db.get_index()); - f("resource_limits_state", db.get_index()); - f("resource_limits_config", db.get_index()); +static void for_each_table(const chainbase::database& db, F f) { + f("account", db.get_index(), [](auto&) { return true; }); + f("account_sequence", db.get_index(), [](auto&) { return true; }); + + f("table_id", db.get_index(), [](auto&) { return true; }); + f("key_value", db.get_index(), [](auto&) { return true; }); + f("index64", db.get_index(), [](auto&) { return true; }); + f("index128", db.get_index(), [](auto&) { return true; }); + f("index256", db.get_index(), [](auto&) { return true; }); + f("index_double", db.get_index(), [](auto&) { return true; }); + f("index_long_double", db.get_index(), [](auto&) { return true; }); + + f("global_property", db.get_index(), [](auto&) { return true; }); + f("dynamic_global_property", db.get_index(), [](auto&) { return true; }); + f("block_summary", db.get_index(), + [](auto& row) { return row.block_id != block_id_type(); }); + f("transaction", db.get_index(), [](auto&) { return true; }); + f("generated_transaction", db.get_index(), [](auto&) { return true; }); + + f("permission", db.get_index(), [](auto&) { return true; }); + f("permission_usage", db.get_index(), [](auto&) { return true; }); + f("permission_link", db.get_index(), [](auto&) { return true; }); + + f("resource_limits", db.get_index(), [](auto&) { return true; }); + f("resource_usage", db.get_index(), [](auto&) { return true; }); + f("resource_limits_state", db.get_index(), [](auto&) { return true; }); + f("resource_limits_config", db.get_index(), + [](auto&) { return true; }); } template @@ -543,6 +545,8 @@ struct state_history_plugin_impl : std::enable_shared_from_thisnext_layer().set_option(boost::asio::socket_base::send_buffer_size(1024 * 1024)); stream->next_layer().set_option(boost::asio::socket_base::receive_buffer_size(1024 * 1024)); stream->async_accept([self = shared_from_this(), this](boost::system::error_code ec) { + if (!plugin->db) + return; callback(ec, "async_accept", [&] { start_read(); send(state_history_plugin_abi); @@ -554,6 +558,8 @@ struct state_history_plugin_impl : std::enable_shared_from_this(); stream->async_read( *in_buffer, [self = shared_from_this(), this, in_buffer](boost::system::error_code ec, size_t) { + if (!plugin->db) + return; callback(ec, "async_read", [&] { auto d = boost::asio::buffer_cast(boost::beast::buffers_front(in_buffer->data())); auto s = boost::asio::buffer_size(in_buffer->data()); @@ -585,6 +591,8 @@ struct state_history_plugin_impl : std::enable_shared_from_thisasync_write( // boost::asio::buffer(send_queue[0]), [self = shared_from_this(), this](boost::system::error_code ec, size_t) { + if (!plugin->db) + return; callback(ec, "async_write", [&] { send_queue.erase(send_queue.begin()); sending = false; @@ -680,6 +688,8 @@ struct state_history_plugin_impl : std::enable_shared_from_this(app().get_io_service()); acceptor->async_accept(*socket, [self = shared_from_this(), socket, this](auto ec) { + if (!db) + return; if (ec) { if (ec == boost::system::errc::too_many_files_open) catch_and_log([&] { do_accept(); }); @@ -717,7 +727,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thiscreate([&](state_history_object& hist) { hist.id = block_state->block->block_num(); std::vector deltas; - for_each_table(chain.db(), [&, this](auto* name, auto& index) { + for_each_table(chain.db(), [&, this](auto* name, auto& index, auto filter) { if (fresh) { if (index.indices().empty()) return; @@ -725,7 +735,8 @@ struct state_history_plugin_impl : std::enable_shared_from_thisaccepted_block_connection.reset(); while (!my->sessions.empty()) my->sessions.begin()->second->close(); + my->db.reset(); } } // namespace eosio From ee7e034721c3b9ab350ce4baae301a74a3f14daf Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Wed, 3 Oct 2018 17:17:17 -0400 Subject: [PATCH 014/369] state history: irreversible_state.log --- .../state_history_plugin.cpp | 155 +++++++++++++++++- 1 file changed, 150 insertions(+), 5 deletions(-) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 52aeaa597c4..7ef9a09fe3f 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -520,14 +520,110 @@ auto catch_and_log(F f) { } } +/* + * irreversible_state.log: + * +---------+----------------+-----------+------------------+-----+---------+----------------+ + * | Entry i | Pos of Entry i | Entry i+1 | Pos of Entry i+1 | ... | Entry z | Pos of Entry z | + * +---------+----------------+-----------+------------------+-----+---------+----------------+ + * + * irreversible_state.index: + * +----------------+------------------+-----+----------------+ + * | Pos of Entry i | Pos of Entry i+1 | ... | Pos of Entry z | + * +----------------+------------------+-----+----------------+ + * + * Each entry: + * uint32_t block_num + * uint64_t size of remainder + * char[] deltas + */ + +struct irrev_state_log_header { + uint32_t block_num; + uint64_t size; +}; + struct state_history_plugin_impl : std::enable_shared_from_this { chain_plugin* chain_plug = nullptr; std::unique_ptr db; + std::fstream irreversible_log; + std::fstream irreversible_index; + uint32_t irreversible_begin_block = 0; + uint32_t irreversible_end_block = 0; fc::optional accepted_block_connection; + fc::optional irreversible_block_connection; string endpoint_address = "0.0.0.0"; uint16_t endpoint_port = 4321; std::unique_ptr acceptor; + void open_irreversible(const boost::filesystem::path& path) { + open_irreversible_log((path / "irreversible_state.log").string()); + open_irreversible_index((path / "irreversible_state.index").string()); + } + + void open_irreversible_log(const std::string& filename) { + irreversible_log.open(filename, + std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); + uint64_t size = irreversible_log.tellg(); + if (size >= sizeof(irrev_state_log_header)) { + irrev_state_log_header header; + irreversible_log.seekg(0); + irreversible_log.read((char*)&header, sizeof(header)); + EOS_ASSERT(sizeof(header) + header.size + sizeof(uint64_t) <= size, plugin_exception, + "corrupt irreversible_state.log (1)"); + irreversible_begin_block = header.block_num; + + uint64_t end_pos; + irreversible_log.seekg(size - sizeof(end_pos)); + irreversible_log.read((char*)&end_pos, sizeof(end_pos)); + EOS_ASSERT(end_pos <= size && end_pos + sizeof(header) <= size, plugin_exception, + "corrupt irreversible_state.log (2)"); + irreversible_log.seekg(end_pos); + irreversible_log.read((char*)&header, sizeof(header)); + EOS_ASSERT(end_pos + sizeof(header) + header.size + sizeof(uint64_t) == size, plugin_exception, + "corrupt irreversible_state.log (3)"); + irreversible_end_block = header.block_num + 1; + + EOS_ASSERT(irreversible_begin_block < irreversible_end_block, plugin_exception, + "corrupt irreversible_state.log (4)"); + ilog("irreversible_state.log has blocks ${b}-${e}", + ("b", irreversible_begin_block)("e", irreversible_end_block - 1)); + } else { + EOS_ASSERT(!size, plugin_exception, "corrupt irreversible_state.log (5)"); + ilog("irreversible_state.log is empty"); + } + } + + void open_irreversible_index(const std::string& filename) { + irreversible_index.open(filename, + std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); + if (irreversible_index.tellg() == (irreversible_end_block - irreversible_begin_block) * sizeof(uint64_t)) + return; + ilog("Regenerate irreversible_state.index"); + irreversible_index.close(); + irreversible_index.open(filename, + std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc); + + irreversible_log.seekg(0, std::ios_base::end); + uint64_t size = irreversible_log.tellg(); + uint64_t pos = 0; + while (pos < size) { + irreversible_index.write((char*)&pos, sizeof(pos)); + irrev_state_log_header header; + EOS_ASSERT(pos + sizeof(header) <= size, plugin_exception, "corrupt irreversible_state.log (6)"); + irreversible_log.seekg(pos); + irreversible_log.read((char*)&header, sizeof(header)); + uint64_t suffix_pos = pos + sizeof(header) + header.size; + uint64_t suffix; + EOS_ASSERT(suffix_pos + sizeof(suffix) <= size, plugin_exception, "corrupt irreversible_state.log (7)"); + irreversible_log.seekg(suffix_pos); + irreversible_log.read((char*)&suffix, sizeof(suffix)); + // ilog("block ${b} at ${pos}-${end} suffix=${suffix} file_size=${fs}", + // ("b", header.block_num)("pos", pos)("end", suffix_pos + sizeof(suffix))("suffix", suffix)("fs", size)); + EOS_ASSERT(suffix == pos, plugin_exception, "corrupt irreversible_state.log (8)"); + pos = suffix_pos + sizeof(suffix); + } + } + struct session : std::enable_shared_from_this { std::shared_ptr plugin; std::unique_ptr> stream; @@ -707,6 +803,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisblock->block_num())); auto& chain = chain_plug->chain(); auto& idx = db->get_index(); @@ -722,7 +819,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisblock->block_num())); + ilog("Placing initial state in block ${n}", ("n", block_state->block->block_num())); db->create([&](state_history_object& hist) { hist.id = block_state->block->block_num(); @@ -765,8 +862,43 @@ struct state_history_plugin_impl : std::enable_shared_from_thisblock->block_num())); + auto& idx = db->get_index().indices(); + while (!idx.empty()) { + auto& obj = *idx.begin(); + if (obj.id._id > block_state->block->block_num()) + break; + if (obj.id._id == irreversible_end_block || irreversible_begin_block == irreversible_end_block) { + // ilog("write irreversible ${n}", ("n", obj.id._id)); + irreversible_log.seekg(0, std::ios_base::end); + uint64_t pos = irreversible_log.tellg(); + irrev_state_log_header header{.block_num = (uint32_t)obj.id._id, .size = obj.deltas.size()}; + irreversible_log.write((char*)&header, sizeof(header)); + if (!obj.deltas.empty()) + irreversible_log.write(obj.deltas.data(), obj.deltas.size()); + irreversible_log.write((char*)&pos, sizeof(pos)); + + // ilog("block ${b} at ${pos}-${end}", + // ("b", header.block_num)("pos", pos)("end", pos + sizeof(header) + obj.deltas.size() + sizeof(pos))); + + irreversible_index.seekg(0, std::ios_base::end); + irreversible_index.write((char*)&pos, sizeof(pos)); + + if (irreversible_begin_block == irreversible_end_block) + irreversible_begin_block = obj.id._id; + irreversible_end_block = obj.id._id + 1; + } + if (idx.size() >= 2 && (++idx.begin())->id._id <= block_state->block->block_num()) { + // ilog("drop irreversible ${n}", ("n", obj.id._id)); + db->remove(obj); + } else + break; + } + } // on_irreversible_block +}; // state_history_plugin_impl state_history_plugin::state_history_plugin() : my(std::make_shared()) {} @@ -793,6 +925,8 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { auto& chain = my->chain_plug->chain(); my->accepted_block_connection.emplace( chain.accepted_block.connect([&](const block_state_ptr& p) { my->on_accepted_block(p); })); + my->irreversible_block_connection.emplace( + chain.irreversible_block.connect([&](const block_state_ptr& p) { my->on_irreversible_block(p); })); auto dir_option = options.at("state-history-dir").as(); boost::filesystem::path state_history_dir; @@ -816,8 +950,18 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { my->db = std::make_unique(state_history_dir, database::read_write, state_history_size); my->db->add_index(); + my->open_irreversible(state_history_dir); + + auto& ind = my->db->get_index().indices(); + ilog("in-memory state history has ${n} blocks", ("n", ind.size())); + if (my->irreversible_end_block != my->irreversible_begin_block) { + // todo: load if ind.empty() + EOS_ASSERT(!ind.empty(), plugin_exception, "irreversible_state.log and shared_memory don't match"); + auto& x = *ind.begin(); + EOS_ASSERT(x.id._id == my->irreversible_end_block - 1, plugin_exception, + "irreversible_state.log and shared_memory don't match"); + } - // auto& ind = my->db->get_index().indices(); // size_t total = 0, max_size = 0, max_block = 0, num = 0; // for (auto& x : ind) { // if (!(x.id._id % 10000)) @@ -834,12 +978,13 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { // dlog("num: ${s}", ("s", num)); } FC_LOG_AND_RETHROW() -} +} // namespace eosio void state_history_plugin::plugin_startup() { my->listen(); } void state_history_plugin::plugin_shutdown() { my->accepted_block_connection.reset(); + my->irreversible_block_connection.reset(); while (!my->sessions.empty()) my->sessions.begin()->second->close(); my->db.reset(); From 032498999a1703424ab3b1fc4e11245b44d6ef8d Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Thu, 4 Oct 2018 09:51:45 -0400 Subject: [PATCH 015/369] state history plugin: get_irreversible --- .../state_history_plugin.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 7ef9a09fe3f..ff10add37e7 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -624,6 +624,17 @@ struct state_history_plugin_impl : std::enable_shared_from_this { std::shared_ptr plugin; std::unique_ptr> stream; @@ -715,6 +726,11 @@ struct state_history_plugin_impl : std::enable_shared_from_thisdeltas.begin(), it->deltas.end()); // dlog(" bytes: ${b}", ("b", result.deltas)); + } else if (req.block_num >= plugin->irreversible_begin_block && + req.block_num < plugin->irreversible_end_block) { + irrev_state_log_header header; + plugin->get_irreversible(req.block_num, header, result.deltas); + result.found = true; } send(std::move(result)); } From 48f93a0db06b99967fd12205fcdb74aec6544807 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Fri, 5 Oct 2018 13:14:18 -0400 Subject: [PATCH 016/369] state history: fix abi --- .../state_history_plugin/state_history_plugin.cpp | 4 +++- .../state_history_plugin_abi.cpp | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index ff10add37e7..f3901372755 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -718,7 +718,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisdb->get_index().indices(); auto it = ind.find(req.block_num); @@ -823,6 +823,8 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain(); auto& idx = db->get_index(); + // todo: ignore blocks already in irreversible_state.log + uint64_t num_removed = 0; while (!idx.indices().empty() && (--idx.indices().end())->id._id >= block_state->block->block_num()) { db->remove(*--idx.indices().end()); diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index 881c63c3084..cca1d7aff4f 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -41,7 +41,7 @@ extern const char* const state_history_plugin_abi = R"({ { "type": "uint8", "name": "vm_type" }, { "type": "uint8", "name": "vm_version" }, { "type": "bool", "name": "privileged" }, - { "type": "time_point", "name": " last_code_update" }, + { "type": "time_point", "name": "last_code_update" }, { "type": "checksum256", "name": "code_version" }, { "type": "block_timestamp_type", "name": "creation_date" }, { "type": "bytes", "name": "code" }, @@ -184,7 +184,7 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "transaction_v0", "fields": [ { "type": "uint64", "name": "id" }, - { "type": "time_point_sec", "name": " expiration" }, + { "type": "time_point_sec", "name": "expiration" }, { "type": "checksum256", "name": "trx_id" } ] }, @@ -195,9 +195,9 @@ extern const char* const state_history_plugin_abi = R"({ { "type": "name", "name": "sender" }, { "type": "uint128", "name": "sender_id" }, { "type": "name", "name": "payer" }, - { "type": "time_point", "name": " delay_until" }, - { "type": "time_point", "name": " expiration" }, - { "type": "time_point", "name": " published" }, + { "type": "time_point", "name": "delay_until" }, + { "type": "time_point", "name": "expiration" }, + { "type": "time_point", "name": "published" }, { "type": "bytes", "name": "packed_trx" } ] }, @@ -240,14 +240,14 @@ extern const char* const state_history_plugin_abi = R"({ { "type": "uint64", "name": "parent" }, { "type": "name", "name": "owner" }, { "type": "name", "name": "name" }, - { "type": "time_point", "name": " last_updated" }, + { "type": "time_point", "name": "last_updated" }, { "type": "shared_authority", "name": "auth" } ] }, { "name": "permission_usage_v0", "fields": [ { "type": "uint64", "name": "id" }, - { "type": "time_point", "name": " last_used" } + { "type": "time_point", "name": "last_used" } ] }, { From 73fd5bff7205ce1a44342f61218a27995ef6be26 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Fri, 5 Oct 2018 15:35:36 -0400 Subject: [PATCH 017/369] state history --- .../state_history_plugin.cpp | 235 ++++++++++-------- 1 file changed, 131 insertions(+), 104 deletions(-) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index f3901372755..4865274dcb4 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -521,118 +521,155 @@ auto catch_and_log(F f) { } /* - * irreversible_state.log: + * *.log: * +---------+----------------+-----------+------------------+-----+---------+----------------+ * | Entry i | Pos of Entry i | Entry i+1 | Pos of Entry i+1 | ... | Entry z | Pos of Entry z | * +---------+----------------+-----------+------------------+-----+---------+----------------+ * - * irreversible_state.index: + * *.index: * +----------------+------------------+-----+----------------+ * | Pos of Entry i | Pos of Entry i+1 | ... | Pos of Entry z | * +----------------+------------------+-----+----------------+ * - * Each entry: + * each entry: * uint32_t block_num - * uint64_t size of remainder + * uint64_t size of payload + * uint8_t version + * payload + * + * irreversible_state payload: + * uint32_t size of deltas * char[] deltas */ -struct irrev_state_log_header { - uint32_t block_num; - uint64_t size; +struct history_log_header { + uint32_t block_num = 0; + uint64_t payload_size = 0; + uint16_t version = 0; }; -struct state_history_plugin_impl : std::enable_shared_from_this { - chain_plugin* chain_plug = nullptr; - std::unique_ptr db; - std::fstream irreversible_log; - std::fstream irreversible_index; - uint32_t irreversible_begin_block = 0; - uint32_t irreversible_end_block = 0; - fc::optional accepted_block_connection; - fc::optional irreversible_block_connection; - string endpoint_address = "0.0.0.0"; - uint16_t endpoint_port = 4321; - std::unique_ptr acceptor; - - void open_irreversible(const boost::filesystem::path& path) { - open_irreversible_log((path / "irreversible_state.log").string()); - open_irreversible_index((path / "irreversible_state.index").string()); - } - - void open_irreversible_log(const std::string& filename) { - irreversible_log.open(filename, - std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); - uint64_t size = irreversible_log.tellg(); - if (size >= sizeof(irrev_state_log_header)) { - irrev_state_log_header header; - irreversible_log.seekg(0); - irreversible_log.read((char*)&header, sizeof(header)); - EOS_ASSERT(sizeof(header) + header.size + sizeof(uint64_t) <= size, plugin_exception, - "corrupt irreversible_state.log (1)"); - irreversible_begin_block = header.block_num; +struct history_log { + const char* const name = ""; + std::fstream log; + std::fstream index; + uint32_t begin_block = 0; + uint32_t end_block = 0; + + void open_log(const std::string& filename) { + log.open(filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); + uint64_t size = log.tellg(); + if (size >= sizeof(history_log_header)) { + history_log_header header; + log.seekg(0); + log.read((char*)&header, sizeof(header)); + EOS_ASSERT(header.version == 0 && sizeof(header) + header.payload_size + sizeof(uint64_t) <= size, + plugin_exception, "corrupt ${name}.log (1)", ("name", name)); + begin_block = header.block_num; uint64_t end_pos; - irreversible_log.seekg(size - sizeof(end_pos)); - irreversible_log.read((char*)&end_pos, sizeof(end_pos)); - EOS_ASSERT(end_pos <= size && end_pos + sizeof(header) <= size, plugin_exception, - "corrupt irreversible_state.log (2)"); - irreversible_log.seekg(end_pos); - irreversible_log.read((char*)&header, sizeof(header)); - EOS_ASSERT(end_pos + sizeof(header) + header.size + sizeof(uint64_t) == size, plugin_exception, - "corrupt irreversible_state.log (3)"); - irreversible_end_block = header.block_num + 1; - - EOS_ASSERT(irreversible_begin_block < irreversible_end_block, plugin_exception, - "corrupt irreversible_state.log (4)"); - ilog("irreversible_state.log has blocks ${b}-${e}", - ("b", irreversible_begin_block)("e", irreversible_end_block - 1)); + log.seekg(size - sizeof(end_pos)); + log.read((char*)&end_pos, sizeof(end_pos)); + EOS_ASSERT(end_pos <= size && end_pos + sizeof(header) <= size, plugin_exception, "corrupt ${name}.log (2)", + ("name", name)); + log.seekg(end_pos); + log.read((char*)&header, sizeof(header)); + EOS_ASSERT(end_pos + sizeof(header) + header.payload_size + sizeof(uint64_t) == size, plugin_exception, + "corrupt ${name}.log (3)", ("name", name)); + end_block = header.block_num + 1; + + EOS_ASSERT(begin_block < end_block, plugin_exception, "corrupt ${name}.log (4)", ("name", name)); + ilog("${name}.log has blocks ${b}-${e}", ("name", name)("b", begin_block)("e", end_block - 1)); } else { - EOS_ASSERT(!size, plugin_exception, "corrupt irreversible_state.log (5)"); - ilog("irreversible_state.log is empty"); + EOS_ASSERT(!size, plugin_exception, "corrupt ${name}.log (5)", ("name", name)); + ilog("${name}.log is empty", ("name", name)); } } - void open_irreversible_index(const std::string& filename) { - irreversible_index.open(filename, - std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); - if (irreversible_index.tellg() == (irreversible_end_block - irreversible_begin_block) * sizeof(uint64_t)) + void open_index(const std::string& filename) { + index.open(filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); + if (index.tellg() == (end_block - begin_block) * sizeof(uint64_t)) return; - ilog("Regenerate irreversible_state.index"); - irreversible_index.close(); - irreversible_index.open(filename, - std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc); + ilog("Regenerate ${name}.index", ("name", name)); + index.close(); + index.open(filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc); - irreversible_log.seekg(0, std::ios_base::end); - uint64_t size = irreversible_log.tellg(); + log.seekg(0, std::ios_base::end); + uint64_t size = log.tellg(); uint64_t pos = 0; while (pos < size) { - irreversible_index.write((char*)&pos, sizeof(pos)); - irrev_state_log_header header; - EOS_ASSERT(pos + sizeof(header) <= size, plugin_exception, "corrupt irreversible_state.log (6)"); - irreversible_log.seekg(pos); - irreversible_log.read((char*)&header, sizeof(header)); - uint64_t suffix_pos = pos + sizeof(header) + header.size; + index.write((char*)&pos, sizeof(pos)); + history_log_header header; + EOS_ASSERT(pos + sizeof(header) <= size, plugin_exception, "corrupt ${name}.log (6)", ("name", name)); + log.seekg(pos); + log.read((char*)&header, sizeof(header)); + uint64_t suffix_pos = pos + sizeof(header) + header.payload_size; uint64_t suffix; - EOS_ASSERT(suffix_pos + sizeof(suffix) <= size, plugin_exception, "corrupt irreversible_state.log (7)"); - irreversible_log.seekg(suffix_pos); - irreversible_log.read((char*)&suffix, sizeof(suffix)); + EOS_ASSERT(suffix_pos + sizeof(suffix) <= size, plugin_exception, "corrupt ${name}.log (7)", ("name", name)); + log.seekg(suffix_pos); + log.read((char*)&suffix, sizeof(suffix)); // ilog("block ${b} at ${pos}-${end} suffix=${suffix} file_size=${fs}", // ("b", header.block_num)("pos", pos)("end", suffix_pos + sizeof(suffix))("suffix", suffix)("fs", size)); - EOS_ASSERT(suffix == pos, plugin_exception, "corrupt irreversible_state.log (8)"); + EOS_ASSERT(suffix == pos, plugin_exception, "corrupt ${name}.log (8)", ("name", name)); pos = suffix_pos + sizeof(suffix); } } - void get_irreversible(uint32_t block_num, irrev_state_log_header& header, bytes& deltas) { + template + void write_entry(history_log_header header, F write_payload) { + EOS_ASSERT(begin_block == end_block || header.block_num == end_block, plugin_exception, + "writing unexpected block_num to ${name}.log", ("name", name)); + log.seekg(0, std::ios_base::end); + uint64_t pos = log.tellg(); + log.write((char*)&header, sizeof(header)); + write_payload(log); + uint64_t end = log.tellg(); + EOS_ASSERT(end == pos + sizeof(header) + header.payload_size, plugin_exception, + "wrote payload with incorrect size to ${name}.log", ("name", name)); + log.write((char*)&pos, sizeof(pos)); + + index.seekg(0, std::ios_base::end); + index.write((char*)&pos, sizeof(pos)); + if (begin_block == end_block) + begin_block = header.block_num; + end_block = header.block_num + 1; + } + + // returns stream positioned at payload + std::fstream& get_entry(uint32_t block_num, history_log_header& header) { + EOS_ASSERT(block_num >= begin_block && block_num < end_block, plugin_exception, + "read non-existing block in ${name}.log", ("name", name)); uint64_t pos; - irreversible_index.seekg((block_num - irreversible_begin_block) * sizeof(pos)); - irreversible_index.read((char*)&pos, sizeof(pos)); - irreversible_log.seekg(pos); - irreversible_log.read((char*)&header, sizeof(header)); - deltas.resize(header.size); - if (header.size) - irreversible_log.read(deltas.data(), header.size); + index.seekg((block_num - begin_block) * sizeof(pos)); + index.read((char*)&pos, sizeof(pos)); + log.seekg(pos); + log.read((char*)&header, sizeof(header)); + return log; + } +}; // history_log + +struct state_history_plugin_impl : std::enable_shared_from_this { + chain_plugin* chain_plug = nullptr; + std::unique_ptr db; + history_log state_log{"irreversible_state"}; + fc::optional accepted_block_connection; + fc::optional irreversible_block_connection; + string endpoint_address = "0.0.0.0"; + uint16_t endpoint_port = 4321; + std::unique_ptr acceptor; + + void open_irreversible(const boost::filesystem::path& path) { + state_log.open_log((path / "irreversible_state.log").string()); + state_log.open_index((path / "irreversible_state.index").string()); + } + + void get_irreversible_state(uint32_t block_num, bytes& deltas) { + history_log_header header; + auto& stream = state_log.get_entry(block_num, header); + uint32_t s; + stream.read((char*)&s, sizeof(s)); + deltas.resize(s); + if (s) + stream.read(deltas.data(), s); } struct session : std::enable_shared_from_this { @@ -726,10 +763,8 @@ struct state_history_plugin_impl : std::enable_shared_from_thisdeltas.begin(), it->deltas.end()); // dlog(" bytes: ${b}", ("b", result.deltas)); - } else if (req.block_num >= plugin->irreversible_begin_block && - req.block_num < plugin->irreversible_end_block) { - irrev_state_log_header header; - plugin->get_irreversible(req.block_num, header, result.deltas); + } else if (req.block_num >= plugin->state_log.begin_block && req.block_num < plugin->state_log.end_block) { + plugin->get_irreversible_state(req.block_num, result.deltas); result.found = true; } send(std::move(result)); @@ -889,25 +924,17 @@ struct state_history_plugin_impl : std::enable_shared_from_this block_state->block->block_num()) break; - if (obj.id._id == irreversible_end_block || irreversible_begin_block == irreversible_end_block) { + if (obj.id._id == state_log.end_block || state_log.begin_block == state_log.end_block) { // ilog("write irreversible ${n}", ("n", obj.id._id)); - irreversible_log.seekg(0, std::ios_base::end); - uint64_t pos = irreversible_log.tellg(); - irrev_state_log_header header{.block_num = (uint32_t)obj.id._id, .size = obj.deltas.size()}; - irreversible_log.write((char*)&header, sizeof(header)); - if (!obj.deltas.empty()) - irreversible_log.write(obj.deltas.data(), obj.deltas.size()); - irreversible_log.write((char*)&pos, sizeof(pos)); - - // ilog("block ${b} at ${pos}-${end}", - // ("b", header.block_num)("pos", pos)("end", pos + sizeof(header) + obj.deltas.size() + sizeof(pos))); - - irreversible_index.seekg(0, std::ios_base::end); - irreversible_index.write((char*)&pos, sizeof(pos)); - - if (irreversible_begin_block == irreversible_end_block) - irreversible_begin_block = obj.id._id; - irreversible_end_block = obj.id._id + 1; + EOS_ASSERT(obj.deltas.size() == (uint32_t)obj.deltas.size(), plugin_exception, "deltas is too big"); + history_log_header header{.block_num = (uint32_t)obj.id._id, + .payload_size = sizeof(uint32_t) + obj.deltas.size()}; + state_log.write_entry(header, [&](auto& stream) { + uint32_t s = (uint32_t)obj.deltas.size(); + stream.write((char*)&s, sizeof(s)); + if (!obj.deltas.empty()) + stream.write(obj.deltas.data(), obj.deltas.size()); + }); } if (idx.size() >= 2 && (++idx.begin())->id._id <= block_state->block->block_num()) { // ilog("drop irreversible ${n}", ("n", obj.id._id)); @@ -972,11 +999,11 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { auto& ind = my->db->get_index().indices(); ilog("in-memory state history has ${n} blocks", ("n", ind.size())); - if (my->irreversible_end_block != my->irreversible_begin_block) { + if (my->state_log.end_block != my->state_log.begin_block) { // todo: load if ind.empty() EOS_ASSERT(!ind.empty(), plugin_exception, "irreversible_state.log and shared_memory don't match"); auto& x = *ind.begin(); - EOS_ASSERT(x.id._id == my->irreversible_end_block - 1, plugin_exception, + EOS_ASSERT(x.id._id == my->state_log.end_block - 1, plugin_exception, "irreversible_state.log and shared_memory don't match"); } From 908ad7caeb3b7cb724580fe03f46da344d72c036 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Fri, 5 Oct 2018 16:22:07 -0400 Subject: [PATCH 018/369] state history plugin: cleanup --- .../state_history_plugin/history_log.hpp | 143 +++++ .../state_history_serialization.hpp | 425 +++++++++++++ .../state_history_plugin.cpp | 568 +----------------- 3 files changed, 576 insertions(+), 560 deletions(-) create mode 100644 plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp create mode 100644 plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp new file mode 100644 index 00000000000..a13008344a5 --- /dev/null +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp @@ -0,0 +1,143 @@ +/** + * @file + * @copyright defined in eos/LICENSE.txt + */ +#pragma once + +#include +#include + +#include +#include + +namespace eosio { + +/* + * *.log: + * +---------+----------------+-----------+------------------+-----+---------+----------------+ + * | Entry i | Pos of Entry i | Entry i+1 | Pos of Entry i+1 | ... | Entry z | Pos of Entry z | + * +---------+----------------+-----------+------------------+-----+---------+----------------+ + * + * *.index: + * +----------------+------------------+-----+----------------+ + * | Pos of Entry i | Pos of Entry i+1 | ... | Pos of Entry z | + * +----------------+------------------+-----+----------------+ + * + * each entry: + * uint32_t block_num + * uint64_t size of payload + * uint8_t version + * payload + * + * irreversible_state payload: + * uint32_t size of deltas + * char[] deltas + */ + +struct history_log_header { + uint32_t block_num = 0; + uint64_t payload_size = 0; + uint16_t version = 0; +}; + +struct history_log { + const char* const name = ""; + std::fstream log; + std::fstream index; + uint32_t begin_block = 0; + uint32_t end_block = 0; + + void open_log(const std::string& filename) { + log.open(filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); + uint64_t size = log.tellg(); + if (size >= sizeof(history_log_header)) { + history_log_header header; + log.seekg(0); + log.read((char*)&header, sizeof(header)); + EOS_ASSERT(header.version == 0 && sizeof(header) + header.payload_size + sizeof(uint64_t) <= size, + chain::plugin_exception, "corrupt ${name}.log (1)", ("name", name)); + begin_block = header.block_num; + + uint64_t end_pos; + log.seekg(size - sizeof(end_pos)); + log.read((char*)&end_pos, sizeof(end_pos)); + EOS_ASSERT(end_pos <= size && end_pos + sizeof(header) <= size, chain::plugin_exception, + "corrupt ${name}.log (2)", ("name", name)); + log.seekg(end_pos); + log.read((char*)&header, sizeof(header)); + EOS_ASSERT(end_pos + sizeof(header) + header.payload_size + sizeof(uint64_t) == size, chain::plugin_exception, + "corrupt ${name}.log (3)", ("name", name)); + end_block = header.block_num + 1; + + EOS_ASSERT(begin_block < end_block, chain::plugin_exception, "corrupt ${name}.log (4)", ("name", name)); + ilog("${name}.log has blocks ${b}-${e}", ("name", name)("b", begin_block)("e", end_block - 1)); + } else { + EOS_ASSERT(!size, chain::plugin_exception, "corrupt ${name}.log (5)", ("name", name)); + ilog("${name}.log is empty", ("name", name)); + } + } + + void open_index(const std::string& filename) { + index.open(filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); + if (index.tellg() == (end_block - begin_block) * sizeof(uint64_t)) + return; + ilog("Regenerate ${name}.index", ("name", name)); + index.close(); + index.open(filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc); + + log.seekg(0, std::ios_base::end); + uint64_t size = log.tellg(); + uint64_t pos = 0; + while (pos < size) { + index.write((char*)&pos, sizeof(pos)); + history_log_header header; + EOS_ASSERT(pos + sizeof(header) <= size, chain::plugin_exception, "corrupt ${name}.log (6)", ("name", name)); + log.seekg(pos); + log.read((char*)&header, sizeof(header)); + uint64_t suffix_pos = pos + sizeof(header) + header.payload_size; + uint64_t suffix; + EOS_ASSERT(suffix_pos + sizeof(suffix) <= size, chain::plugin_exception, "corrupt ${name}.log (7)", + ("name", name)); + log.seekg(suffix_pos); + log.read((char*)&suffix, sizeof(suffix)); + // ilog("block ${b} at ${pos}-${end} suffix=${suffix} file_size=${fs}", + // ("b", header.block_num)("pos", pos)("end", suffix_pos + sizeof(suffix))("suffix", suffix)("fs", size)); + EOS_ASSERT(suffix == pos, chain::plugin_exception, "corrupt ${name}.log (8)", ("name", name)); + pos = suffix_pos + sizeof(suffix); + } + } + + template + void write_entry(history_log_header header, F write_payload) { + EOS_ASSERT(begin_block == end_block || header.block_num == end_block, chain::plugin_exception, + "writing unexpected block_num to ${name}.log", ("name", name)); + log.seekg(0, std::ios_base::end); + uint64_t pos = log.tellg(); + log.write((char*)&header, sizeof(header)); + write_payload(log); + uint64_t end = log.tellg(); + EOS_ASSERT(end == pos + sizeof(header) + header.payload_size, chain::plugin_exception, + "wrote payload with incorrect size to ${name}.log", ("name", name)); + log.write((char*)&pos, sizeof(pos)); + + index.seekg(0, std::ios_base::end); + index.write((char*)&pos, sizeof(pos)); + if (begin_block == end_block) + begin_block = header.block_num; + end_block = header.block_num + 1; + } + + // returns stream positioned at payload + std::fstream& get_entry(uint32_t block_num, history_log_header& header) { + EOS_ASSERT(block_num >= begin_block && block_num < end_block, chain::plugin_exception, + "read non-existing block in ${name}.log", ("name", name)); + uint64_t pos; + index.seekg((block_num - begin_block) * sizeof(pos)); + index.read((char*)&pos, sizeof(pos)); + log.seekg(pos); + log.read((char*)&header, sizeof(header)); + return log; + } +}; // history_log + +} // namespace eosio diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp new file mode 100644 index 00000000000..84e263d8900 --- /dev/null +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -0,0 +1,425 @@ +/** + * @file + * @copyright defined in eos/LICENSE.txt + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +template +struct history_serial_wrapper { + const T& obj; +}; + +template +history_serial_wrapper make_history_serial_wrapper(const T& obj) { + return {obj}; +} + +namespace fc { + +template +const T& as_type(const T& x) { + return x; +} + +template +void serialize_shared_vector(datastream& ds, const T& v) { + fc::raw::pack(ds, unsigned_int(v.size())); + for (auto& x : v) + ds << make_history_serial_wrapper(x); +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.name.value)); + fc::raw::pack(ds, as_type(obj.obj.vm_type)); + fc::raw::pack(ds, as_type(obj.obj.vm_version)); + fc::raw::pack(ds, as_type(obj.obj.privileged)); + fc::raw::pack(ds, as_type(obj.obj.last_code_update)); + fc::raw::pack(ds, as_type(obj.obj.code_version)); + fc::raw::pack(ds, as_type(obj.obj.creation_date)); + fc::raw::pack(ds, as_type(obj.obj.code)); + fc::raw::pack(ds, as_type(obj.obj.abi)); + return ds; +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.name.value)); + fc::raw::pack(ds, as_type(obj.obj.recv_sequence)); + fc::raw::pack(ds, as_type(obj.obj.auth_sequence)); + fc::raw::pack(ds, as_type(obj.obj.code_sequence)); + fc::raw::pack(ds, as_type(obj.obj.abi_sequence)); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.code.value)); + fc::raw::pack(ds, as_type(obj.obj.scope.value)); + fc::raw::pack(ds, as_type(obj.obj.table.value)); + fc::raw::pack(ds, as_type(obj.obj.payer.value)); + fc::raw::pack(ds, as_type(obj.obj.count)); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.t_id._id)); + fc::raw::pack(ds, as_type(obj.obj.primary_key)); + fc::raw::pack(ds, as_type(obj.obj.payer.value)); + fc::raw::pack(ds, as_type(obj.obj.value)); + return ds; +} + +template +void serialize_secondary_index_data(datastream& ds, const T& obj) { + fc::raw::pack(ds, obj); +} + +template +void serialize_secondary_index_data(datastream& ds, const float64_t& obj) { + uint64_t i; + memcpy(&i, &obj, sizeof(i)); + fc::raw::pack(ds, i); +} + +template +void serialize_secondary_index_data(datastream& ds, const float128_t& obj) { + __uint128_t i; + memcpy(&i, &obj, sizeof(i)); + fc::raw::pack(ds, i); +} + +template +void serialize_secondary_index_data(datastream& ds, const eosio::chain::key256_t& obj) { + fc::raw::pack(ds, obj[0]); + fc::raw::pack(ds, obj[1]); +} + +template +datastream& serialize_secondary_index(datastream& ds, const T& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.id._id)); + fc::raw::pack(ds, as_type(obj.t_id._id)); + fc::raw::pack(ds, as_type(obj.primary_key)); + fc::raw::pack(ds, as_type(obj.payer.value)); + serialize_secondary_index_data(ds, obj.secondary_key); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + return serialize_secondary_index(ds, obj.obj); +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + return serialize_secondary_index(ds, obj.obj); +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + return serialize_secondary_index(ds, obj.obj); +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + return serialize_secondary_index(ds, obj.obj); +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + return serialize_secondary_index(ds, obj.obj); +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.producer_name.value)); + fc::raw::pack(ds, as_type(obj.obj.block_signing_key)); + return ds; +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.version)); + serialize_shared_vector(ds, obj.obj.producers); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.max_block_net_usage)); + fc::raw::pack(ds, as_type(obj.obj.target_block_net_usage_pct)); + fc::raw::pack(ds, as_type(obj.obj.max_transaction_net_usage)); + fc::raw::pack(ds, as_type(obj.obj.base_per_transaction_net_usage)); + fc::raw::pack(ds, as_type(obj.obj.net_usage_leeway)); + fc::raw::pack(ds, as_type(obj.obj.context_free_discount_net_usage_num)); + fc::raw::pack(ds, as_type(obj.obj.context_free_discount_net_usage_den)); + fc::raw::pack(ds, as_type(obj.obj.max_block_cpu_usage)); + fc::raw::pack(ds, as_type(obj.obj.target_block_cpu_usage_pct)); + fc::raw::pack(ds, as_type(obj.obj.max_transaction_cpu_usage)); + fc::raw::pack(ds, as_type(obj.obj.min_transaction_cpu_usage)); + fc::raw::pack(ds, as_type(obj.obj.max_transaction_lifetime)); + fc::raw::pack(ds, as_type(obj.obj.deferred_trx_expiration_window)); + fc::raw::pack(ds, as_type(obj.obj.max_transaction_delay)); + fc::raw::pack(ds, as_type(obj.obj.max_inline_action_size)); + fc::raw::pack(ds, as_type(obj.obj.max_inline_action_depth)); + fc::raw::pack(ds, as_type(obj.obj.max_authority_depth)); + return ds; +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + + fc::raw::pack(ds, as_type>(obj.obj.proposed_schedule_block_num)); + fc::raw::pack(ds, make_history_serial_wrapper( + as_type(obj.obj.proposed_schedule))); + fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.configuration))); + + return ds; +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.global_action_sequence)); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.block_id)); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.expiration)); + fc::raw::pack(ds, as_type(obj.obj.trx_id)); + return ds; +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.trx_id)); + fc::raw::pack(ds, as_type(obj.obj.sender.value)); + fc::raw::pack(ds, as_type<__uint128_t>(obj.obj.sender_id)); + fc::raw::pack(ds, as_type(obj.obj.payer.value)); + fc::raw::pack(ds, as_type(obj.obj.delay_until)); + fc::raw::pack(ds, as_type(obj.obj.expiration)); + fc::raw::pack(ds, as_type(obj.obj.published)); + fc::raw::pack(ds, as_type(obj.obj.packed_trx)); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.key)); + fc::raw::pack(ds, as_type(obj.obj.weight)); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.actor.value)); + fc::raw::pack(ds, as_type(obj.obj.permission.value)); + return ds; +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.permission))); + fc::raw::pack(ds, as_type(obj.obj.weight)); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.wait_sec)); + fc::raw::pack(ds, as_type(obj.obj.weight)); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.threshold)); + serialize_shared_vector(ds, obj.obj.keys); + serialize_shared_vector(ds, obj.obj.accounts); + serialize_shared_vector(ds, obj.obj.waits); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.usage_id._id)); + fc::raw::pack(ds, as_type(obj.obj.parent._id)); + fc::raw::pack(ds, as_type(obj.obj.owner.value)); + fc::raw::pack(ds, as_type(obj.obj.name.value)); + fc::raw::pack(ds, as_type(obj.obj.last_updated)); + fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.auth))); + return ds; +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.last_used)); + return ds; +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.account.value)); + fc::raw::pack(ds, as_type(obj.obj.code.value)); + fc::raw::pack(ds, as_type(obj.obj.message_type.value)); + fc::raw::pack(ds, as_type(obj.obj.required_permission.value)); + return ds; +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.owner.value)); + fc::raw::pack(ds, as_type(obj.obj.pending)); + fc::raw::pack(ds, as_type(obj.obj.net_weight)); + fc::raw::pack(ds, as_type(obj.obj.cpu_weight)); + fc::raw::pack(ds, as_type(obj.obj.ram_bytes)); + return ds; +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.last_ordinal)); + fc::raw::pack(ds, as_type(obj.obj.value_ex)); + fc::raw::pack(ds, as_type(obj.obj.consumed)); + return ds; +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, as_type(obj.obj.owner.value)); + fc::raw::pack( + ds, make_history_serial_wrapper(as_type(obj.obj.net_usage))); + fc::raw::pack( + ds, make_history_serial_wrapper(as_type(obj.obj.cpu_usage))); + fc::raw::pack(ds, as_type(obj.obj.ram_usage)); + return ds; +} + +template +datastream& +operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, make_history_serial_wrapper( + as_type(obj.obj.average_block_net_usage))); + fc::raw::pack(ds, make_history_serial_wrapper( + as_type(obj.obj.average_block_cpu_usage))); + fc::raw::pack(ds, as_type(obj.obj.pending_net_usage)); + fc::raw::pack(ds, as_type(obj.obj.pending_cpu_usage)); + fc::raw::pack(ds, as_type(obj.obj.total_net_weight)); + fc::raw::pack(ds, as_type(obj.obj.total_cpu_weight)); + fc::raw::pack(ds, as_type(obj.obj.total_ram_bytes)); + fc::raw::pack(ds, as_type(obj.obj.virtual_net_limit)); + fc::raw::pack(ds, as_type(obj.obj.virtual_cpu_limit)); + return ds; +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.numerator)); + fc::raw::pack(ds, as_type(obj.obj.denominator)); + return ds; +} + +template +datastream& operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.target)); + fc::raw::pack(ds, as_type(obj.obj.max)); + fc::raw::pack(ds, as_type(obj.obj.periods)); + fc::raw::pack(ds, as_type(obj.obj.max_multiplier)); + fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.contract_rate))); + fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.expand_rate))); + return ds; +} + +template +datastream& +operator<<(datastream& ds, + const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id._id)); + fc::raw::pack(ds, make_history_serial_wrapper(as_type( + obj.obj.cpu_limit_parameters))); + fc::raw::pack(ds, make_history_serial_wrapper(as_type( + obj.obj.net_limit_parameters))); + fc::raw::pack(ds, as_type(obj.obj.account_cpu_usage_average_window)); + fc::raw::pack(ds, as_type(obj.obj.account_net_usage_average_window)); + return ds; +}; + +} // namespace fc diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 4865274dcb4..50da245f96b 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -1,16 +1,11 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +/** + * @file + * @copyright defined in eos/LICENSE.txt + */ + +#include #include +#include #include #include @@ -25,426 +20,6 @@ namespace ws = boost::beast::websocket; extern const char* const state_history_plugin_abi; -template -struct history_serial_wrapper { - const T& obj; -}; - -template -static history_serial_wrapper make_history_serial_wrapper(const T& obj) { - return {obj}; -} - -namespace fc { - -template -static const T& as_type(const T& x) { - return x; -} - -template -static void serialize_shared_vector(datastream& ds, const T& v) { - fc::raw::pack(ds, unsigned_int(v.size())); - for (auto& x : v) - ds << make_history_serial_wrapper(x); -} - -template -static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.name.value)); - fc::raw::pack(ds, as_type(obj.obj.vm_type)); - fc::raw::pack(ds, as_type(obj.obj.vm_version)); - fc::raw::pack(ds, as_type(obj.obj.privileged)); - fc::raw::pack(ds, as_type(obj.obj.last_code_update)); - fc::raw::pack(ds, as_type(obj.obj.code_version)); - fc::raw::pack(ds, as_type(obj.obj.creation_date)); - fc::raw::pack(ds, as_type(obj.obj.code)); - fc::raw::pack(ds, as_type(obj.obj.abi)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.name.value)); - fc::raw::pack(ds, as_type(obj.obj.recv_sequence)); - fc::raw::pack(ds, as_type(obj.obj.auth_sequence)); - fc::raw::pack(ds, as_type(obj.obj.code_sequence)); - fc::raw::pack(ds, as_type(obj.obj.abi_sequence)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.code.value)); - fc::raw::pack(ds, as_type(obj.obj.scope.value)); - fc::raw::pack(ds, as_type(obj.obj.table.value)); - fc::raw::pack(ds, as_type(obj.obj.payer.value)); - fc::raw::pack(ds, as_type(obj.obj.count)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.t_id._id)); - fc::raw::pack(ds, as_type(obj.obj.primary_key)); - fc::raw::pack(ds, as_type(obj.obj.payer.value)); - fc::raw::pack(ds, as_type(obj.obj.value)); - return ds; -} - -template -static void serialize_secondary_index_data(datastream& ds, const T& obj) { - fc::raw::pack(ds, obj); -} - -template -static void serialize_secondary_index_data(datastream& ds, const float64_t& obj) { - uint64_t i; - memcpy(&i, &obj, sizeof(i)); - fc::raw::pack(ds, i); -} - -template -static void serialize_secondary_index_data(datastream& ds, const float128_t& obj) { - __uint128_t i; - memcpy(&i, &obj, sizeof(i)); - fc::raw::pack(ds, i); -} - -template -static void serialize_secondary_index_data(datastream& ds, const eosio::chain::key256_t& obj) { - fc::raw::pack(ds, obj[0]); - fc::raw::pack(ds, obj[1]); -} - -template -static datastream& serialize_secondary_index(datastream& ds, const T& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.id._id)); - fc::raw::pack(ds, as_type(obj.t_id._id)); - fc::raw::pack(ds, as_type(obj.primary_key)); - fc::raw::pack(ds, as_type(obj.payer.value)); - serialize_secondary_index_data(ds, obj.secondary_key); - return ds; -} - -template -static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - return serialize_secondary_index(ds, obj.obj); -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - return serialize_secondary_index(ds, obj.obj); -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - return serialize_secondary_index(ds, obj.obj); -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - return serialize_secondary_index(ds, obj.obj); -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - return serialize_secondary_index(ds, obj.obj); -} - -template -static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.producer_name.value)); - fc::raw::pack(ds, as_type(obj.obj.block_signing_key)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.version)); - serialize_shared_vector(ds, obj.obj.producers); - return ds; -} - -template -static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.max_block_net_usage)); - fc::raw::pack(ds, as_type(obj.obj.target_block_net_usage_pct)); - fc::raw::pack(ds, as_type(obj.obj.max_transaction_net_usage)); - fc::raw::pack(ds, as_type(obj.obj.base_per_transaction_net_usage)); - fc::raw::pack(ds, as_type(obj.obj.net_usage_leeway)); - fc::raw::pack(ds, as_type(obj.obj.context_free_discount_net_usage_num)); - fc::raw::pack(ds, as_type(obj.obj.context_free_discount_net_usage_den)); - fc::raw::pack(ds, as_type(obj.obj.max_block_cpu_usage)); - fc::raw::pack(ds, as_type(obj.obj.target_block_cpu_usage_pct)); - fc::raw::pack(ds, as_type(obj.obj.max_transaction_cpu_usage)); - fc::raw::pack(ds, as_type(obj.obj.min_transaction_cpu_usage)); - fc::raw::pack(ds, as_type(obj.obj.max_transaction_lifetime)); - fc::raw::pack(ds, as_type(obj.obj.deferred_trx_expiration_window)); - fc::raw::pack(ds, as_type(obj.obj.max_transaction_delay)); - fc::raw::pack(ds, as_type(obj.obj.max_inline_action_size)); - fc::raw::pack(ds, as_type(obj.obj.max_inline_action_depth)); - fc::raw::pack(ds, as_type(obj.obj.max_authority_depth)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - - fc::raw::pack(ds, as_type>(obj.obj.proposed_schedule_block_num)); - fc::raw::pack(ds, make_history_serial_wrapper( - as_type(obj.obj.proposed_schedule))); - fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.configuration))); - - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.global_action_sequence)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.block_id)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.expiration)); - fc::raw::pack(ds, as_type(obj.obj.trx_id)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.trx_id)); - fc::raw::pack(ds, as_type(obj.obj.sender.value)); - fc::raw::pack(ds, as_type<__uint128_t>(obj.obj.sender_id)); - fc::raw::pack(ds, as_type(obj.obj.payer.value)); - fc::raw::pack(ds, as_type(obj.obj.delay_until)); - fc::raw::pack(ds, as_type(obj.obj.expiration)); - fc::raw::pack(ds, as_type(obj.obj.published)); - fc::raw::pack(ds, as_type(obj.obj.packed_trx)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.key)); - fc::raw::pack(ds, as_type(obj.obj.weight)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.actor.value)); - fc::raw::pack(ds, as_type(obj.obj.permission.value)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.permission))); - fc::raw::pack(ds, as_type(obj.obj.weight)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.wait_sec)); - fc::raw::pack(ds, as_type(obj.obj.weight)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.threshold)); - serialize_shared_vector(ds, obj.obj.keys); - serialize_shared_vector(ds, obj.obj.accounts); - serialize_shared_vector(ds, obj.obj.waits); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.usage_id._id)); - fc::raw::pack(ds, as_type(obj.obj.parent._id)); - fc::raw::pack(ds, as_type(obj.obj.owner.value)); - fc::raw::pack(ds, as_type(obj.obj.name.value)); - fc::raw::pack(ds, as_type(obj.obj.last_updated)); - fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.auth))); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.last_used)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.account.value)); - fc::raw::pack(ds, as_type(obj.obj.code.value)); - fc::raw::pack(ds, as_type(obj.obj.message_type.value)); - fc::raw::pack(ds, as_type(obj.obj.required_permission.value)); - return ds; -} - -template -static datastream& -operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.owner.value)); - fc::raw::pack(ds, as_type(obj.obj.pending)); - fc::raw::pack(ds, as_type(obj.obj.net_weight)); - fc::raw::pack(ds, as_type(obj.obj.cpu_weight)); - fc::raw::pack(ds, as_type(obj.obj.ram_bytes)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.last_ordinal)); - fc::raw::pack(ds, as_type(obj.obj.value_ex)); - fc::raw::pack(ds, as_type(obj.obj.consumed)); - return ds; -} - -template -static datastream& -operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.owner.value)); - fc::raw::pack( - ds, make_history_serial_wrapper(as_type(obj.obj.net_usage))); - fc::raw::pack( - ds, make_history_serial_wrapper(as_type(obj.obj.cpu_usage))); - fc::raw::pack(ds, as_type(obj.obj.ram_usage)); - return ds; -} - -template -static datastream& -operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, make_history_serial_wrapper( - as_type(obj.obj.average_block_net_usage))); - fc::raw::pack(ds, make_history_serial_wrapper( - as_type(obj.obj.average_block_cpu_usage))); - fc::raw::pack(ds, as_type(obj.obj.pending_net_usage)); - fc::raw::pack(ds, as_type(obj.obj.pending_cpu_usage)); - fc::raw::pack(ds, as_type(obj.obj.total_net_weight)); - fc::raw::pack(ds, as_type(obj.obj.total_cpu_weight)); - fc::raw::pack(ds, as_type(obj.obj.total_ram_bytes)); - fc::raw::pack(ds, as_type(obj.obj.virtual_net_limit)); - fc::raw::pack(ds, as_type(obj.obj.virtual_cpu_limit)); - return ds; -} - -template -static datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.numerator)); - fc::raw::pack(ds, as_type(obj.obj.denominator)); - return ds; -} - -template -static datastream& -operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.target)); - fc::raw::pack(ds, as_type(obj.obj.max)); - fc::raw::pack(ds, as_type(obj.obj.periods)); - fc::raw::pack(ds, as_type(obj.obj.max_multiplier)); - fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.contract_rate))); - fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.expand_rate))); - return ds; -} - -template -static datastream& -operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, make_history_serial_wrapper(as_type( - obj.obj.cpu_limit_parameters))); - fc::raw::pack(ds, make_history_serial_wrapper(as_type( - obj.obj.net_limit_parameters))); - fc::raw::pack(ds, as_type(obj.obj.account_cpu_usage_average_window)); - fc::raw::pack(ds, as_type(obj.obj.account_net_usage_average_window)); - return ds; -}; - -} // namespace fc - namespace eosio { using namespace chain; using boost::signals2::scoped_connection; @@ -520,133 +95,6 @@ auto catch_and_log(F f) { } } -/* - * *.log: - * +---------+----------------+-----------+------------------+-----+---------+----------------+ - * | Entry i | Pos of Entry i | Entry i+1 | Pos of Entry i+1 | ... | Entry z | Pos of Entry z | - * +---------+----------------+-----------+------------------+-----+---------+----------------+ - * - * *.index: - * +----------------+------------------+-----+----------------+ - * | Pos of Entry i | Pos of Entry i+1 | ... | Pos of Entry z | - * +----------------+------------------+-----+----------------+ - * - * each entry: - * uint32_t block_num - * uint64_t size of payload - * uint8_t version - * payload - * - * irreversible_state payload: - * uint32_t size of deltas - * char[] deltas - */ - -struct history_log_header { - uint32_t block_num = 0; - uint64_t payload_size = 0; - uint16_t version = 0; -}; - -struct history_log { - const char* const name = ""; - std::fstream log; - std::fstream index; - uint32_t begin_block = 0; - uint32_t end_block = 0; - - void open_log(const std::string& filename) { - log.open(filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); - uint64_t size = log.tellg(); - if (size >= sizeof(history_log_header)) { - history_log_header header; - log.seekg(0); - log.read((char*)&header, sizeof(header)); - EOS_ASSERT(header.version == 0 && sizeof(header) + header.payload_size + sizeof(uint64_t) <= size, - plugin_exception, "corrupt ${name}.log (1)", ("name", name)); - begin_block = header.block_num; - - uint64_t end_pos; - log.seekg(size - sizeof(end_pos)); - log.read((char*)&end_pos, sizeof(end_pos)); - EOS_ASSERT(end_pos <= size && end_pos + sizeof(header) <= size, plugin_exception, "corrupt ${name}.log (2)", - ("name", name)); - log.seekg(end_pos); - log.read((char*)&header, sizeof(header)); - EOS_ASSERT(end_pos + sizeof(header) + header.payload_size + sizeof(uint64_t) == size, plugin_exception, - "corrupt ${name}.log (3)", ("name", name)); - end_block = header.block_num + 1; - - EOS_ASSERT(begin_block < end_block, plugin_exception, "corrupt ${name}.log (4)", ("name", name)); - ilog("${name}.log has blocks ${b}-${e}", ("name", name)("b", begin_block)("e", end_block - 1)); - } else { - EOS_ASSERT(!size, plugin_exception, "corrupt ${name}.log (5)", ("name", name)); - ilog("${name}.log is empty", ("name", name)); - } - } - - void open_index(const std::string& filename) { - index.open(filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); - if (index.tellg() == (end_block - begin_block) * sizeof(uint64_t)) - return; - ilog("Regenerate ${name}.index", ("name", name)); - index.close(); - index.open(filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc); - - log.seekg(0, std::ios_base::end); - uint64_t size = log.tellg(); - uint64_t pos = 0; - while (pos < size) { - index.write((char*)&pos, sizeof(pos)); - history_log_header header; - EOS_ASSERT(pos + sizeof(header) <= size, plugin_exception, "corrupt ${name}.log (6)", ("name", name)); - log.seekg(pos); - log.read((char*)&header, sizeof(header)); - uint64_t suffix_pos = pos + sizeof(header) + header.payload_size; - uint64_t suffix; - EOS_ASSERT(suffix_pos + sizeof(suffix) <= size, plugin_exception, "corrupt ${name}.log (7)", ("name", name)); - log.seekg(suffix_pos); - log.read((char*)&suffix, sizeof(suffix)); - // ilog("block ${b} at ${pos}-${end} suffix=${suffix} file_size=${fs}", - // ("b", header.block_num)("pos", pos)("end", suffix_pos + sizeof(suffix))("suffix", suffix)("fs", size)); - EOS_ASSERT(suffix == pos, plugin_exception, "corrupt ${name}.log (8)", ("name", name)); - pos = suffix_pos + sizeof(suffix); - } - } - - template - void write_entry(history_log_header header, F write_payload) { - EOS_ASSERT(begin_block == end_block || header.block_num == end_block, plugin_exception, - "writing unexpected block_num to ${name}.log", ("name", name)); - log.seekg(0, std::ios_base::end); - uint64_t pos = log.tellg(); - log.write((char*)&header, sizeof(header)); - write_payload(log); - uint64_t end = log.tellg(); - EOS_ASSERT(end == pos + sizeof(header) + header.payload_size, plugin_exception, - "wrote payload with incorrect size to ${name}.log", ("name", name)); - log.write((char*)&pos, sizeof(pos)); - - index.seekg(0, std::ios_base::end); - index.write((char*)&pos, sizeof(pos)); - if (begin_block == end_block) - begin_block = header.block_num; - end_block = header.block_num + 1; - } - - // returns stream positioned at payload - std::fstream& get_entry(uint32_t block_num, history_log_header& header) { - EOS_ASSERT(block_num >= begin_block && block_num < end_block, plugin_exception, - "read non-existing block in ${name}.log", ("name", name)); - uint64_t pos; - index.seekg((block_num - begin_block) * sizeof(pos)); - index.read((char*)&pos, sizeof(pos)); - log.seekg(pos); - log.read((char*)&header, sizeof(header)); - return log; - } -}; // history_log - struct state_history_plugin_impl : std::enable_shared_from_this { chain_plugin* chain_plug = nullptr; std::unique_ptr db; @@ -1023,7 +471,7 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { // dlog("num: ${s}", ("s", num)); } FC_LOG_AND_RETHROW() -} // namespace eosio +} // state_history_plugin::plugin_initialize void state_history_plugin::plugin_startup() { my->listen(); } From 0cfc0b3dc258b697f69ef3584eb03068a025f4f5 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Mon, 8 Oct 2018 15:20:53 -0400 Subject: [PATCH 019/369] state history: simplify storage --- .../state_history_plugin/history_log.hpp | 145 ++++++++--- .../state_history_plugin.hpp | 22 +- .../state_history_plugin.cpp | 246 +++++------------- .../state_history_plugin_abi.cpp | 14 +- 4 files changed, 192 insertions(+), 235 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp index a13008344a5..3d48c45033e 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp @@ -4,6 +4,7 @@ */ #pragma once +#include #include #include @@ -40,15 +41,75 @@ struct history_log_header { uint16_t version = 0; }; -struct history_log { - const char* const name = ""; +class history_log { + private: + const char* const name = ""; + bool _is_open = false; + std::string log_filename; + std::string index_filename; std::fstream log; std::fstream index; - uint32_t begin_block = 0; - uint32_t end_block = 0; + uint32_t _begin_block = 0; + uint32_t _end_block = 0; - void open_log(const std::string& filename) { - log.open(filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); + public: + history_log(const char* const name) + : name(name) {} + + bool is_open() const { return _is_open; } + uint32_t begin_block() const { return _begin_block; } + uint32_t end_block() const { return _end_block; } + + void open(std::string log_filename, std::string index_filename) { + this->log_filename = std::move(log_filename); + this->index_filename = std::move(index_filename); + open_log(); + open_index(); + _is_open = true; + } + + void close() { + log.close(); + index.close(); + _is_open = false; + } + + template + void write_entry(history_log_header header, F write_payload) { + EOS_ASSERT(_is_open, chain::plugin_exception, "writing to closed ${name}.log", ("name", name)); + EOS_ASSERT(_begin_block == _end_block || header.block_num <= _end_block, chain::plugin_exception, + "missed a block writing to ${name}.log", ("name", name)); + if (header.block_num < _end_block) + truncate(header.block_num); + log.seekg(0, std::ios_base::end); + uint64_t pos = log.tellg(); + log.write((char*)&header, sizeof(header)); + write_payload(log); + uint64_t end = log.tellg(); + EOS_ASSERT(end == pos + sizeof(header) + header.payload_size, chain::plugin_exception, + "wrote payload with incorrect size to ${name}.log", ("name", name)); + log.write((char*)&pos, sizeof(pos)); + + index.seekg(0, std::ios_base::end); + index.write((char*)&pos, sizeof(pos)); + if (_begin_block == _end_block) + _begin_block = header.block_num; + _end_block = header.block_num + 1; + } + + // returns stream positioned at payload + std::fstream& get_entry(uint32_t block_num, history_log_header& header) { + EOS_ASSERT(_is_open, chain::plugin_exception, "reading from closed ${name}.log", ("name", name)); + EOS_ASSERT(block_num >= _begin_block && block_num < _end_block, chain::plugin_exception, + "read non-existing block in ${name}.log", ("name", name)); + log.seekg(get_pos(block_num)); + log.read((char*)&header, sizeof(header)); + return log; + } + + private: + void open_log() { + log.open(log_filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); uint64_t size = log.tellg(); if (size >= sizeof(history_log_header)) { history_log_header header; @@ -56,7 +117,7 @@ struct history_log { log.read((char*)&header, sizeof(header)); EOS_ASSERT(header.version == 0 && sizeof(header) + header.payload_size + sizeof(uint64_t) <= size, chain::plugin_exception, "corrupt ${name}.log (1)", ("name", name)); - begin_block = header.block_num; + _begin_block = header.block_num; uint64_t end_pos; log.seekg(size - sizeof(end_pos)); @@ -67,23 +128,23 @@ struct history_log { log.read((char*)&header, sizeof(header)); EOS_ASSERT(end_pos + sizeof(header) + header.payload_size + sizeof(uint64_t) == size, chain::plugin_exception, "corrupt ${name}.log (3)", ("name", name)); - end_block = header.block_num + 1; + _end_block = header.block_num + 1; - EOS_ASSERT(begin_block < end_block, chain::plugin_exception, "corrupt ${name}.log (4)", ("name", name)); - ilog("${name}.log has blocks ${b}-${e}", ("name", name)("b", begin_block)("e", end_block - 1)); + EOS_ASSERT(_begin_block < _end_block, chain::plugin_exception, "corrupt ${name}.log (4)", ("name", name)); + ilog("${name}.log has blocks ${b}-${e}", ("name", name)("b", _begin_block)("e", _end_block - 1)); } else { EOS_ASSERT(!size, chain::plugin_exception, "corrupt ${name}.log (5)", ("name", name)); ilog("${name}.log is empty", ("name", name)); } } - void open_index(const std::string& filename) { - index.open(filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); - if (index.tellg() == (end_block - begin_block) * sizeof(uint64_t)) + void open_index() { + index.open(index_filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); + if (index.tellg() == (_end_block - _begin_block) * sizeof(uint64_t)) return; ilog("Regenerate ${name}.index", ("name", name)); index.close(); - index.open(filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc); + index.open(index_filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc); log.seekg(0, std::ios_base::end); uint64_t size = log.tellg(); @@ -107,36 +168,36 @@ struct history_log { } } - template - void write_entry(history_log_header header, F write_payload) { - EOS_ASSERT(begin_block == end_block || header.block_num == end_block, chain::plugin_exception, - "writing unexpected block_num to ${name}.log", ("name", name)); - log.seekg(0, std::ios_base::end); - uint64_t pos = log.tellg(); - log.write((char*)&header, sizeof(header)); - write_payload(log); - uint64_t end = log.tellg(); - EOS_ASSERT(end == pos + sizeof(header) + header.payload_size, chain::plugin_exception, - "wrote payload with incorrect size to ${name}.log", ("name", name)); - log.write((char*)&pos, sizeof(pos)); - - index.seekg(0, std::ios_base::end); - index.write((char*)&pos, sizeof(pos)); - if (begin_block == end_block) - begin_block = header.block_num; - end_block = header.block_num + 1; - } - - // returns stream positioned at payload - std::fstream& get_entry(uint32_t block_num, history_log_header& header) { - EOS_ASSERT(block_num >= begin_block && block_num < end_block, chain::plugin_exception, - "read non-existing block in ${name}.log", ("name", name)); + uint64_t get_pos(uint32_t block_num) { uint64_t pos; - index.seekg((block_num - begin_block) * sizeof(pos)); + index.seekg((block_num - _begin_block) * sizeof(pos)); index.read((char*)&pos, sizeof(pos)); - log.seekg(pos); - log.read((char*)&header, sizeof(header)); - return log; + return pos; + } + + void truncate(uint32_t block_num) { + log.flush(); + index.flush(); + uint64_t num_removed = 0; + if (block_num <= _begin_block) { + num_removed = _end_block - _begin_block; + log.seekg(0); + index.seekg(0); + boost::filesystem::resize_file(log_filename, 0); + boost::filesystem::resize_file(index_filename, 0); + _begin_block = _end_block = 0; + } else { + num_removed = _end_block - block_num; + uint64_t pos = get_pos(block_num); + log.seekg(0); + index.seekg(0); + boost::filesystem::resize_file(log_filename, pos); + boost::filesystem::resize_file(index_filename, (block_num - _begin_block) * sizeof(pos)); + _end_block = block_num; + } + log.sync(); + index.sync(); + ilog("fork or replay: removed ${n} blocks from ${name}.log", ("n", num_removed)("name", name)); } }; // history_log diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index 14cd1947b7d..8d1afb361f5 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -24,10 +24,11 @@ struct table_delta { std::vector removed{}; }; -struct get_state_request_v0 {}; +struct get_status_request_v0 {}; -struct get_state_result_v0 { - uint32_t last_block_num = 0; +struct get_status_result_v0 { + uint32_t state_begin_block_num = 0; + uint32_t state_end_block_num = 0; }; struct get_block_request_v0 { @@ -35,13 +36,12 @@ struct get_block_request_v0 { }; struct get_block_result_v0 { - uint32_t block_num = 0; - bool found = false; - bytes deltas; + uint32_t block_num = 0; + fc::optional deltas; }; -using state_request = fc::static_variant; -using state_result = fc::static_variant; +using state_request = fc::static_variant; +using state_result = fc::static_variant; class state_history_plugin : public plugin { public: @@ -63,7 +63,7 @@ class state_history_plugin : public plugin { } // namespace eosio FC_REFLECT(eosio::table_delta, (struct_version)(name)(rows)(removed)); -FC_REFLECT_EMPTY(eosio::get_state_request_v0); -FC_REFLECT(eosio::get_state_result_v0, (last_block_num)); +FC_REFLECT_EMPTY(eosio::get_status_request_v0); +FC_REFLECT(eosio::get_status_result_v0, (state_begin_block_num)(state_end_block_num)); FC_REFLECT(eosio::get_block_request_v0, (block_num)); -FC_REFLECT(eosio::get_block_result_v0, (block_num)(found)(deltas)); +FC_REFLECT(eosio::get_block_result_v0, (block_num)(deltas)); diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 50da245f96b..d62f4c53b5a 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -26,31 +26,6 @@ using boost::signals2::scoped_connection; static appbase::abstract_plugin& _state_history_plugin = app().register_plugin(); -enum class state_history_db_object_type { - null_object_type = 0, - state_history_object_type, -}; - -struct state_history_object - : public chainbase::object<(uint16_t)state_history_db_object_type::state_history_object_type, - state_history_object> { - OBJECT_CTOR(state_history_object, (deltas)) - - id_type id; - shared_string deltas; -}; - -using state_history_index = chainbase::shared_multi_index_container< - state_history_object, - indexed_by, - member>>>; - -} // namespace eosio - -CHAINBASE_SET_INDEX_TYPE(eosio::state_history_object, eosio::state_history_index) - -namespace eosio { - template static void for_each_table(const chainbase::database& db, F f) { f("account", db.get_index(), [](auto&) { return true; }); @@ -96,21 +71,16 @@ auto catch_and_log(F f) { } struct state_history_plugin_impl : std::enable_shared_from_this { - chain_plugin* chain_plug = nullptr; - std::unique_ptr db; - history_log state_log{"irreversible_state"}; - fc::optional accepted_block_connection; - fc::optional irreversible_block_connection; - string endpoint_address = "0.0.0.0"; - uint16_t endpoint_port = 4321; - std::unique_ptr acceptor; - - void open_irreversible(const boost::filesystem::path& path) { - state_log.open_log((path / "irreversible_state.log").string()); - state_log.open_index((path / "irreversible_state.index").string()); - } - - void get_irreversible_state(uint32_t block_num, bytes& deltas) { + chain_plugin* chain_plug = nullptr; + history_log state_log{"state_history"}; + bool stopping = false; + fc::optional accepted_block_connection; + fc::optional irreversible_block_connection; + string endpoint_address = "0.0.0.0"; + uint16_t endpoint_port = 4321; + std::unique_ptr acceptor; + + void get_state(uint32_t block_num, bytes& deltas) { history_log_header header; auto& stream = state_log.get_entry(block_num, header); uint32_t s; @@ -137,7 +107,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisnext_layer().set_option(boost::asio::socket_base::send_buffer_size(1024 * 1024)); stream->next_layer().set_option(boost::asio::socket_base::receive_buffer_size(1024 * 1024)); stream->async_accept([self = shared_from_this(), this](boost::system::error_code ec) { - if (!plugin->db) + if (plugin->stopping) return; callback(ec, "async_accept", [&] { start_read(); @@ -150,7 +120,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this(); stream->async_read( *in_buffer, [self = shared_from_this(), this, in_buffer](boost::system::error_code ec, size_t) { - if (!plugin->db) + if (plugin->stopping) return; callback(ec, "async_read", [&] { auto d = boost::asio::buffer_cast(boost::beast::buffers_front(in_buffer->data())); @@ -183,7 +153,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisasync_write( // boost::asio::buffer(send_queue[0]), [self = shared_from_this(), this](boost::system::error_code ec, size_t) { - if (!plugin->db) + if (plugin->stopping) return; callback(ec, "async_write", [&] { send_queue.erase(send_queue.begin()); @@ -194,26 +164,19 @@ struct state_history_plugin_impl : std::enable_shared_from_thisdb->get_index().indices(); - if (!ind.empty()) - result.last_block_num = (--ind.end())->id._id; + void operator()(get_status_request_v0&) { + get_status_result_v0 result; + result.state_begin_block_num = plugin->state_log.begin_block(); + result.state_end_block_num = plugin->state_log.end_block(); send(std::move(result)); } void operator()(get_block_request_v0& req) { // ilog("${b} get_block_request_v0", ("b", req.block_num)); get_block_result_v0 result{req.block_num}; - auto& ind = plugin->db->get_index().indices(); - auto it = ind.find(req.block_num); - if (it != ind.end()) { - result.found = true; - result.deltas.assign(it->deltas.begin(), it->deltas.end()); - // dlog(" bytes: ${b}", ("b", result.deltas)); - } else if (req.block_num >= plugin->state_log.begin_block && req.block_num < plugin->state_log.end_block) { - plugin->get_irreversible_state(req.block_num, result.deltas); - result.found = true; + if (req.block_num >= plugin->state_log.begin_block() && req.block_num < plugin->state_log.end_block()) { + result.deltas.emplace(); + plugin->get_state(req.block_num, *result.deltas); } send(std::move(result)); } @@ -283,7 +246,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this(app().get_io_service()); acceptor->async_accept(*socket, [self = shared_from_this(), socket, this](auto ec) { - if (!db) + if (stopping) return; if (ec) { if (ec == boost::system::errc::too_many_files_open) @@ -299,99 +262,60 @@ struct state_history_plugin_impl : std::enable_shared_from_thisblock->block_num())); - auto& chain = chain_plug->chain(); - auto& idx = db->get_index(); - - // todo: ignore blocks already in irreversible_state.log - - uint64_t num_removed = 0; - while (!idx.indices().empty() && (--idx.indices().end())->id._id >= block_state->block->block_num()) { - db->remove(*--idx.indices().end()); - ++num_removed; - } - if (num_removed) - ilog("fork: removed ${n} blocks", ("n", num_removed)); - EOS_ASSERT(idx.indices().empty() || (--idx.indices().end())->id._id + 1 == block_state->block->block_num(), - plugin_exception, "missed a fork switch"); - - bool fresh = idx.indices().empty(); + bool fresh = state_log.begin_block() == state_log.end_block(); if (fresh) ilog("Placing initial state in block ${n}", ("n", block_state->block->block_num())); - db->create([&](state_history_object& hist) { - hist.id = block_state->block->block_num(); - std::vector deltas; - for_each_table(chain.db(), [&, this](auto* name, auto& index, auto filter) { - if (fresh) { - if (index.indices().empty()) - return; - deltas.push_back({}); - auto& delta = deltas.back(); - delta.name = name; - for (auto& row : index.indices()) - if (filter(row)) - delta.rows.emplace_back(row.id._id, fc::raw::pack(make_history_serial_wrapper(row))); - } else { - if (index.stack().empty()) - return; - auto& undo = index.stack().back(); - if (undo.old_values.empty() && undo.new_ids.empty() && undo.removed_values.empty()) - return; - deltas.push_back({}); - auto& delta = deltas.back(); - delta.name = name; - for (auto& old : undo.old_values) { - auto& row = index.get(old.first); - if (filter(row)) - delta.rows.emplace_back(old.first._id, fc::raw::pack(make_history_serial_wrapper(row))); - } - for (auto id : undo.new_ids) { - auto& row = index.get(id); - if (filter(row)) - delta.rows.emplace_back(id._id, fc::raw::pack(make_history_serial_wrapper(row))); - } - for (auto& old : undo.removed_values) - if (filter(old.second)) - delta.removed.push_back(old.first._id); + std::vector deltas; + for_each_table(chain_plug->chain().db(), [&, this](auto* name, auto& index, auto filter) { + if (fresh) { + if (index.indices().empty()) + return; + deltas.push_back({}); + auto& delta = deltas.back(); + delta.name = name; + for (auto& row : index.indices()) + if (filter(row)) + delta.rows.emplace_back(row.id._id, fc::raw::pack(make_history_serial_wrapper(row))); + } else { + if (index.stack().empty()) + return; + auto& undo = index.stack().back(); + if (undo.old_values.empty() && undo.new_ids.empty() && undo.removed_values.empty()) + return; + deltas.push_back({}); + auto& delta = deltas.back(); + delta.name = name; + for (auto& old : undo.old_values) { + auto& row = index.get(old.first); + if (filter(row)) + delta.rows.emplace_back(old.first._id, fc::raw::pack(make_history_serial_wrapper(row))); } - }); - auto bin = fc::raw::pack(deltas); - // dlog(" ${s} bytes", ("s", bin.size())); - hist.deltas.insert(hist.deltas.end(), bin.begin(), bin.end()); + for (auto id : undo.new_ids) { + auto& row = index.get(id); + if (filter(row)) + delta.rows.emplace_back(id._id, fc::raw::pack(make_history_serial_wrapper(row))); + } + for (auto& old : undo.removed_values) + if (filter(old.second)) + delta.removed.push_back(old.first._id); + } + }); + auto deltas_bin = fc::raw::pack(deltas); + EOS_ASSERT(deltas_bin.size() == (uint32_t)deltas_bin.size(), plugin_exception, "deltas is too big"); + history_log_header header{.block_num = block_state->block->block_num(), + .payload_size = sizeof(uint32_t) + deltas_bin.size()}; + state_log.write_entry(header, [&](auto& stream) { + uint32_t s = (uint32_t)deltas_bin.size(); + stream.write((char*)&s, sizeof(s)); + if (!deltas_bin.empty()) + stream.write(deltas_bin.data(), deltas_bin.size()); }); } // on_accepted_block - void on_irreversible_block(const block_state_ptr& block_state) { - // ilog("irreversible ${n}", ("n", block_state->block->block_num())); - auto& idx = db->get_index().indices(); - while (!idx.empty()) { - auto& obj = *idx.begin(); - if (obj.id._id > block_state->block->block_num()) - break; - if (obj.id._id == state_log.end_block || state_log.begin_block == state_log.end_block) { - // ilog("write irreversible ${n}", ("n", obj.id._id)); - EOS_ASSERT(obj.deltas.size() == (uint32_t)obj.deltas.size(), plugin_exception, "deltas is too big"); - history_log_header header{.block_num = (uint32_t)obj.id._id, - .payload_size = sizeof(uint32_t) + obj.deltas.size()}; - state_log.write_entry(header, [&](auto& stream) { - uint32_t s = (uint32_t)obj.deltas.size(); - stream.write((char*)&s, sizeof(s)); - if (!obj.deltas.empty()) - stream.write(obj.deltas.data(), obj.deltas.size()); - }); - } - if (idx.size() >= 2 && (++idx.begin())->id._id <= block_state->block->block_num()) { - // ilog("drop irreversible ${n}", ("n", obj.id._id)); - db->remove(obj); - } else - break; - } - } // on_irreversible_block -}; // state_history_plugin_impl + void on_irreversible_block(const block_state_ptr& block_state) {} +}; // state_history_plugin_impl state_history_plugin::state_history_plugin() : my(std::make_shared()) {} @@ -402,8 +326,6 @@ void state_history_plugin::set_program_options(options_description& cli, options auto options = cfg.add_options(); options("state-history-dir", bpo::value()->default_value("state-history"), "the location of the state-history directory (absolute path or relative to application data dir)"); - options("state-history-db-size-mb", bpo::value()->default_value(1024), - "Maximum size (in MiB) of the state history database"); options("delete-state-history", bpo::bool_switch()->default_value(false), "clear state history database"); options("state-history-endpoint", bpo::value()->default_value("0.0.0.0:8080"), "the endpoint upon which to listen for incoming connections"); @@ -427,7 +349,6 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { state_history_dir = app().data_dir() / dir_option; else state_history_dir = dir_option; - auto state_history_size = options.at("state-history-db-size-mb").as() * 1024 * 1024; auto ip_port = options.at("state-history-endpoint").as(); auto port = ip_port.substr(ip_port.find(':') + 1, ip_port.size()); @@ -441,34 +362,9 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { boost::filesystem::remove_all(state_history_dir); } - my->db = std::make_unique(state_history_dir, database::read_write, state_history_size); - my->db->add_index(); - my->open_irreversible(state_history_dir); - - auto& ind = my->db->get_index().indices(); - ilog("in-memory state history has ${n} blocks", ("n", ind.size())); - if (my->state_log.end_block != my->state_log.begin_block) { - // todo: load if ind.empty() - EOS_ASSERT(!ind.empty(), plugin_exception, "irreversible_state.log and shared_memory don't match"); - auto& x = *ind.begin(); - EOS_ASSERT(x.id._id == my->state_log.end_block - 1, plugin_exception, - "irreversible_state.log and shared_memory don't match"); - } - - // size_t total = 0, max_size = 0, max_block = 0, num = 0; - // for (auto& x : ind) { - // if (!(x.id._id % 10000)) - // dlog(" ${i}: ${s} bytes", ("i", x.id._id)("s", x.deltas.size())); - // total += x.deltas.size(); - // if (x.deltas.size() > max_size) { - // max_block = x.id._id; - // max_size = x.deltas.size(); - // } - // ++num; - // } - // dlog("total: ${s} bytes", ("s", total)); - // dlog("max: ${s} bytes in ${b}", ("s", max_size)("b", max_block)); - // dlog("num: ${s}", ("s", num)); + boost::filesystem::create_directories(state_history_dir); + my->state_log.open((state_history_dir / "state_history.log").string(), + (state_history_dir / "state_history.index").string()); } FC_LOG_AND_RETHROW() } // state_history_plugin::plugin_initialize @@ -480,7 +376,7 @@ void state_history_plugin::plugin_shutdown() { my->irreversible_block_connection.reset(); while (!my->sessions.empty()) my->sessions.begin()->second->close(); - my->db.reset(); + my->stopping = true; } } // namespace eosio diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index cca1d7aff4f..26ce726ae88 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -2,11 +2,12 @@ extern const char* const state_history_plugin_abi = R"({ "version": "eosio::abi/1.1", "structs": [ { - "name": "get_state_request_v0", "fields": [] + "name": "get_status_request_v0", "fields": [] }, { - "name": "get_state_result_v0", "fields": [ - { "name": "last_block_num", "type": "uint32" } + "name": "get_status_result_v0", "fields": [ + { "name": "state_begin_block_num", "type": "uint32" }, + { "name": "state_end_block_num", "type": "uint32" } ] }, { @@ -17,8 +18,7 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "get_block_result_v0", "fields": [ { "name": "block_num", "type": "uint32" }, - { "name": "found", "type": "bool" }, - { "name": "deltas", "type": "bytes" } + { "name": "deltas", "type": "bytes?" } ] }, { @@ -326,8 +326,8 @@ extern const char* const state_history_plugin_abi = R"({ } ], "variants": [ - { "name": "request", "types": ["get_state_request_v0", "get_block_request_v0"] }, - { "name": "result", "types": ["get_state_result_v0", "get_block_result_v0"] }, + { "name": "request", "types": ["get_status_request_v0", "get_block_request_v0"] }, + { "name": "result", "types": ["get_status_result_v0", "get_block_result_v0"] }, { "name": "table_delta", "types": ["table_delta_v0"] }, { "name": "account", "types": ["account_v0"] }, From bfa79b4188fb4a6c52a2709432c7ac442015e399 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Mon, 8 Oct 2018 17:27:22 -0400 Subject: [PATCH 020/369] state history --- .../state_history_plugin/history_log.hpp | 93 ++++++++++++------- .../state_history_plugin.cpp | 3 +- 2 files changed, 64 insertions(+), 32 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp index 3d48c45033e..ac5eaa0d113 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp @@ -9,6 +9,7 @@ #include #include +#include #include namespace eosio { @@ -20,37 +21,49 @@ namespace eosio { * +---------+----------------+-----------+------------------+-----+---------+----------------+ * * *.index: - * +----------------+------------------+-----+----------------+ - * | Pos of Entry i | Pos of Entry i+1 | ... | Pos of Entry z | - * +----------------+------------------+-----+----------------+ + * +-----------+-------------+-----+-----------+ + * | Summary i | Summary i+1 | ... | Summary z | + * +-----------+-------------+-----+-----------+ * * each entry: - * uint32_t block_num - * uint64_t size of payload - * uint8_t version - * payload + * uint32_t block_num + * block_id_type block_id + * uint64_t size of payload + * uint8_t version + * payload * - * irreversible_state payload: + * each summary: + * uint64_t position of entry in *.log + * block_id_type block_id + * + * state payload: * uint32_t size of deltas * char[] deltas */ struct history_log_header { - uint32_t block_num = 0; - uint64_t payload_size = 0; - uint16_t version = 0; + uint32_t block_num = 0; + chain::block_id_type block_id; + uint64_t payload_size = 0; + uint8_t version = 0; +}; + +struct history_summary { + uint64_t pos = 0; + chain::block_id_type block_id; }; class history_log { private: - const char* const name = ""; - bool _is_open = false; - std::string log_filename; - std::string index_filename; - std::fstream log; - std::fstream index; - uint32_t _begin_block = 0; - uint32_t _end_block = 0; + const char* const name = ""; + bool _is_open = false; + std::string log_filename; + std::string index_filename; + std::fstream log; + std::fstream index; + uint32_t _begin_block = 0; + uint32_t _end_block = 0; + chain::block_id_type last_block_id; public: history_log(const char* const name) @@ -75,10 +88,23 @@ class history_log { } template - void write_entry(history_log_header header, F write_payload) { + void write_entry(history_log_header header, const chain::block_id_type& prev_id, F write_payload) { EOS_ASSERT(_is_open, chain::plugin_exception, "writing to closed ${name}.log", ("name", name)); EOS_ASSERT(_begin_block == _end_block || header.block_num <= _end_block, chain::plugin_exception, - "missed a block writing to ${name}.log", ("name", name)); + "missed a block in ${name}.log", ("name", name)); + + if (_begin_block != _end_block && header.block_num > _begin_block) { + if (header.block_num == _end_block) { + EOS_ASSERT(prev_id == last_block_id, chain::plugin_exception, "missed a fork change in ${name}.log", + ("name", name)); + } else { + history_log_header prev; + get_entry(header.block_num - 1, prev); + EOS_ASSERT(prev_id == prev.block_id, chain::plugin_exception, "missed a fork change in ${name}.log", + ("name", name)); + } + } + if (header.block_num < _end_block) truncate(header.block_num); log.seekg(0, std::ios_base::end); @@ -91,10 +117,12 @@ class history_log { log.write((char*)&pos, sizeof(pos)); index.seekg(0, std::ios_base::end); - index.write((char*)&pos, sizeof(pos)); + history_summary summary{.pos = pos, .block_id = header.block_id}; + index.write((char*)&summary, sizeof(summary)); if (_begin_block == _end_block) _begin_block = header.block_num; - _end_block = header.block_num + 1; + _end_block = header.block_num + 1; + last_block_id = header.block_id; } // returns stream positioned at payload @@ -128,7 +156,8 @@ class history_log { log.read((char*)&header, sizeof(header)); EOS_ASSERT(end_pos + sizeof(header) + header.payload_size + sizeof(uint64_t) == size, chain::plugin_exception, "corrupt ${name}.log (3)", ("name", name)); - _end_block = header.block_num + 1; + _end_block = header.block_num + 1; + last_block_id = header.block_id; EOS_ASSERT(_begin_block < _end_block, chain::plugin_exception, "corrupt ${name}.log (4)", ("name", name)); ilog("${name}.log has blocks ${b}-${e}", ("name", name)("b", _begin_block)("e", _end_block - 1)); @@ -140,7 +169,7 @@ class history_log { void open_index() { index.open(index_filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); - if (index.tellg() == (_end_block - _begin_block) * sizeof(uint64_t)) + if (index.tellg() == (_end_block - _begin_block) * sizeof(history_summary)) return; ilog("Regenerate ${name}.index", ("name", name)); index.close(); @@ -150,7 +179,6 @@ class history_log { uint64_t size = log.tellg(); uint64_t pos = 0; while (pos < size) { - index.write((char*)&pos, sizeof(pos)); history_log_header header; EOS_ASSERT(pos + sizeof(header) <= size, chain::plugin_exception, "corrupt ${name}.log (6)", ("name", name)); log.seekg(pos); @@ -164,15 +192,18 @@ class history_log { // ilog("block ${b} at ${pos}-${end} suffix=${suffix} file_size=${fs}", // ("b", header.block_num)("pos", pos)("end", suffix_pos + sizeof(suffix))("suffix", suffix)("fs", size)); EOS_ASSERT(suffix == pos, chain::plugin_exception, "corrupt ${name}.log (8)", ("name", name)); + + history_summary summary{.pos = pos, .block_id = header.block_id}; + index.write((char*)&summary, sizeof(summary)); pos = suffix_pos + sizeof(suffix); } } uint64_t get_pos(uint32_t block_num) { - uint64_t pos; - index.seekg((block_num - _begin_block) * sizeof(pos)); - index.read((char*)&pos, sizeof(pos)); - return pos; + history_summary summary; + index.seekg((block_num - _begin_block) * sizeof(summary)); + index.read((char*)&summary, sizeof(summary)); + return summary.pos; } void truncate(uint32_t block_num) { @@ -192,7 +223,7 @@ class history_log { log.seekg(0); index.seekg(0); boost::filesystem::resize_file(log_filename, pos); - boost::filesystem::resize_file(index_filename, (block_num - _begin_block) * sizeof(pos)); + boost::filesystem::resize_file(index_filename, (block_num - _begin_block) * sizeof(history_summary)); _end_block = block_num; } log.sync(); diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index d62f4c53b5a..c2af8fc94a0 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -305,8 +305,9 @@ struct state_history_plugin_impl : std::enable_shared_from_thisblock->block_num(), + .block_id = block_state->block->id(), .payload_size = sizeof(uint32_t) + deltas_bin.size()}; - state_log.write_entry(header, [&](auto& stream) { + state_log.write_entry(header, block_state->block->previous, [&](auto& stream) { uint32_t s = (uint32_t)deltas_bin.size(); stream.write((char*)&s, sizeof(s)); if (!deltas_bin.empty()) From 1654060fc8b80c1b57e45bb18c1f2563916291d6 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Tue, 9 Oct 2018 10:35:52 -0400 Subject: [PATCH 021/369] state history --- .../state_history_plugin.hpp | 8 +++++--- .../state_history_serialization.hpp | 9 --------- .../state_history_plugin.cpp | 17 ++++++----------- .../state_history_plugin_abi.cpp | 10 ++-------- 4 files changed, 13 insertions(+), 31 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index 8d1afb361f5..0d958325e8f 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -27,8 +27,10 @@ struct table_delta { struct get_status_request_v0 {}; struct get_status_result_v0 { - uint32_t state_begin_block_num = 0; - uint32_t state_end_block_num = 0; + uint32_t last_irreversible_block_num = 0; + chain::block_id_type last_irreversible_block_id = {}; + uint32_t state_begin_block_num = 0; + uint32_t state_end_block_num = 0; }; struct get_block_request_v0 { @@ -64,6 +66,6 @@ class state_history_plugin : public plugin { FC_REFLECT(eosio::table_delta, (struct_version)(name)(rows)(removed)); FC_REFLECT_EMPTY(eosio::get_status_request_v0); -FC_REFLECT(eosio::get_status_result_v0, (state_begin_block_num)(state_end_block_num)); +FC_REFLECT(eosio::get_status_result_v0, (last_irreversible_block_num)(last_irreversible_block_id)(state_begin_block_num)(state_end_block_num)); FC_REFLECT(eosio::get_block_request_v0, (block_num)); FC_REFLECT(eosio::get_block_result_v0, (block_num)(deltas)); diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp index 84e263d8900..933b2cd35a6 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -5,7 +5,6 @@ #pragma once #include -#include #include #include #include @@ -218,14 +217,6 @@ datastream& operator<<(datastream& return ds; } -template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.block_id)); - return ds; -} - template datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index c2af8fc94a0..108f31a53da 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -41,8 +41,6 @@ static void for_each_table(const chainbase::database& db, F f) { f("global_property", db.get_index(), [](auto&) { return true; }); f("dynamic_global_property", db.get_index(), [](auto&) { return true; }); - f("block_summary", db.get_index(), - [](auto& row) { return row.block_id != block_id_type(); }); f("transaction", db.get_index(), [](auto&) { return true; }); f("generated_transaction", db.get_index(), [](auto&) { return true; }); @@ -75,7 +73,6 @@ struct state_history_plugin_impl : std::enable_shared_from_this accepted_block_connection; - fc::optional irreversible_block_connection; string endpoint_address = "0.0.0.0"; uint16_t endpoint_port = 4321; std::unique_ptr acceptor; @@ -165,9 +162,12 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain_plug->chain(); get_status_result_v0 result; - result.state_begin_block_num = plugin->state_log.begin_block(); - result.state_end_block_num = plugin->state_log.end_block(); + result.last_irreversible_block_num = chain.last_irreversible_block_num(); + result.last_irreversible_block_id = chain.last_irreversible_block_id(); + result.state_begin_block_num = plugin->state_log.begin_block(); + result.state_end_block_num = plugin->state_log.end_block(); send(std::move(result)); } @@ -314,9 +314,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this()) {} @@ -341,8 +339,6 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { auto& chain = my->chain_plug->chain(); my->accepted_block_connection.emplace( chain.accepted_block.connect([&](const block_state_ptr& p) { my->on_accepted_block(p); })); - my->irreversible_block_connection.emplace( - chain.irreversible_block.connect([&](const block_state_ptr& p) { my->on_irreversible_block(p); })); auto dir_option = options.at("state-history-dir").as(); boost::filesystem::path state_history_dir; @@ -374,7 +370,6 @@ void state_history_plugin::plugin_startup() { my->listen(); } void state_history_plugin::plugin_shutdown() { my->accepted_block_connection.reset(); - my->irreversible_block_connection.reset(); while (!my->sessions.empty()) my->sessions.begin()->second->close(); my->stopping = true; diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index 26ce726ae88..948b96b528c 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -6,6 +6,8 @@ extern const char* const state_history_plugin_abi = R"({ }, { "name": "get_status_result_v0", "fields": [ + { "name": "last_irreversible_block_num", "type": "uint32" }, + { "name": "last_irreversible_block_id", "type": "checksum256" }, { "name": "state_begin_block_num", "type": "uint32" }, { "name": "state_end_block_num", "type": "uint32" } ] @@ -175,12 +177,6 @@ extern const char* const state_history_plugin_abi = R"({ { "type": "uint64", "name": "global_action_sequence" } ] }, - { - "name": "block_summary_v0", "fields": [ - { "type": "uint64", "name": "id" }, - { "type": "checksum256", "name": "block_id" } - ] - }, { "name": "transaction_v0", "fields": [ { "type": "uint64", "name": "id" }, @@ -344,7 +340,6 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "chain_config", "types": ["chain_config_v0"] }, { "name": "global_property", "types": ["global_property_v0"] }, { "name": "dynamic_global_property", "types": ["dynamic_global_property_v0"] }, - { "name": "block_summary", "types": ["block_summary_v0"] }, { "name": "transaction", "types": ["transaction_v0"] }, { "name": "generated_transaction", "types": ["generated_transaction_v0"] }, { "name": "key_weight", "types": ["key_weight_v0"] }, @@ -375,7 +370,6 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "index_long_double", "type": "index_long_double" }, { "name": "global_property", "type": "global_property" }, { "name": "dynamic_global_property", "type": "dynamic_global_property" }, - { "name": "block_summary", "type": "block_summary" }, { "name": "transaction", "type": "transaction" }, { "name": "generated_transaction", "type": "generated_transaction" }, { "name": "permission", "type": "permission" }, From 190dc807d56fca038f7a468e0f4c0301e0d8deb9 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Tue, 9 Oct 2018 11:44:28 -0400 Subject: [PATCH 022/369] state history plugin: recover_blocks --- .../state_history_plugin/history_log.hpp | 79 +++++++++++++++---- 1 file changed, 63 insertions(+), 16 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp index ac5eaa0d113..cb1b5ceb6d8 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp @@ -136,6 +136,59 @@ class history_log { } private: + bool get_last_block(uint64_t size) { + history_log_header header; + uint64_t suffix; + log.seekg(size - sizeof(suffix)); + log.read((char*)&suffix, sizeof(suffix)); + if (suffix > size || suffix + sizeof(header) > size) { + elog("corrupt ${name}.log (2)", ("name", name)); + return false; + } + log.seekg(suffix); + log.read((char*)&header, sizeof(header)); + if (suffix + sizeof(header) + header.payload_size + sizeof(suffix) != size) { + elog("corrupt ${name}.log (3)", ("name", name)); + return false; + } + _end_block = header.block_num + 1; + last_block_id = header.block_id; + if (_begin_block >= _end_block) { + elog("corrupt ${name}.log (4)", ("name", name)); + return false; + } + return true; + } + + void recover_blocks(uint64_t size) { + ilog("recover ${name}.log", ("name", name)); + uint64_t pos = 0; + uint32_t num_found = 0; + while (true) { + history_log_header header; + if (pos + sizeof(header) > size) + break; + log.seekg(pos); + log.read((char*)&header, sizeof(header)); + uint64_t suffix; + if (header.payload_size > size || pos + sizeof(header) + header.payload_size + sizeof(suffix) > size) + break; + log.seekg(pos + sizeof(header) + header.payload_size); + log.read((char*)&suffix, sizeof(suffix)); + if (suffix != pos) + break; + pos = pos + sizeof(header) + header.payload_size + sizeof(suffix); + if (!(++num_found % 10000)) { + printf("%10u blocks found, log pos=%12llu\r", (unsigned)num_found, (unsigned long long)pos); + fflush(stdout); + } + } + log.flush(); + boost::filesystem::resize_file(log_filename, pos); + log.sync(); + EOS_ASSERT(get_last_block(pos), chain::plugin_exception, "recover ${name}.log failed", ("name", name)); + } + void open_log() { log.open(log_filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app); uint64_t size = log.tellg(); @@ -145,21 +198,10 @@ class history_log { log.read((char*)&header, sizeof(header)); EOS_ASSERT(header.version == 0 && sizeof(header) + header.payload_size + sizeof(uint64_t) <= size, chain::plugin_exception, "corrupt ${name}.log (1)", ("name", name)); - _begin_block = header.block_num; - - uint64_t end_pos; - log.seekg(size - sizeof(end_pos)); - log.read((char*)&end_pos, sizeof(end_pos)); - EOS_ASSERT(end_pos <= size && end_pos + sizeof(header) <= size, chain::plugin_exception, - "corrupt ${name}.log (2)", ("name", name)); - log.seekg(end_pos); - log.read((char*)&header, sizeof(header)); - EOS_ASSERT(end_pos + sizeof(header) + header.payload_size + sizeof(uint64_t) == size, chain::plugin_exception, - "corrupt ${name}.log (3)", ("name", name)); - _end_block = header.block_num + 1; + _begin_block = header.block_num; last_block_id = header.block_id; - - EOS_ASSERT(_begin_block < _end_block, chain::plugin_exception, "corrupt ${name}.log (4)", ("name", name)); + if (!get_last_block(size)) + recover_blocks(size); ilog("${name}.log has blocks ${b}-${e}", ("name", name)("b", _begin_block)("e", _end_block - 1)); } else { EOS_ASSERT(!size, chain::plugin_exception, "corrupt ${name}.log (5)", ("name", name)); @@ -176,8 +218,9 @@ class history_log { index.open(index_filename, std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc); log.seekg(0, std::ios_base::end); - uint64_t size = log.tellg(); - uint64_t pos = 0; + uint64_t size = log.tellg(); + uint64_t pos = 0; + uint32_t num_found = 0; while (pos < size) { history_log_header header; EOS_ASSERT(pos + sizeof(header) <= size, chain::plugin_exception, "corrupt ${name}.log (6)", ("name", name)); @@ -196,6 +239,10 @@ class history_log { history_summary summary{.pos = pos, .block_id = header.block_id}; index.write((char*)&summary, sizeof(summary)); pos = suffix_pos + sizeof(suffix); + if (!(++num_found % 10000)) { + printf("%10u blocks found, log pos=%12llu\r", (unsigned)num_found, (unsigned long long)pos); + fflush(stdout); + } } } From fd022005315bb6ad3b05f69df74b855e21179282 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Wed, 10 Oct 2018 11:53:52 -0400 Subject: [PATCH 023/369] state history plugin: transaction traces --- .../state_history_plugin.hpp | 6 +- .../state_history_serialization.hpp | 93 ++++++++++++++++++- .../state_history_plugin.cpp | 72 ++++++++++++-- .../state_history_plugin_abi.cpp | 65 ++++++++++++- 4 files changed, 220 insertions(+), 16 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index 0d958325e8f..465a4692f45 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -40,6 +40,7 @@ struct get_block_request_v0 { struct get_block_result_v0 { uint32_t block_num = 0; fc::optional deltas; + fc::optional traces; }; using state_request = fc::static_variant; @@ -66,6 +67,7 @@ class state_history_plugin : public plugin { FC_REFLECT(eosio::table_delta, (struct_version)(name)(rows)(removed)); FC_REFLECT_EMPTY(eosio::get_status_request_v0); -FC_REFLECT(eosio::get_status_result_v0, (last_irreversible_block_num)(last_irreversible_block_id)(state_begin_block_num)(state_end_block_num)); +FC_REFLECT(eosio::get_status_result_v0, + (last_irreversible_block_num)(last_irreversible_block_id)(state_begin_block_num)(state_end_block_num)); FC_REFLECT(eosio::get_block_request_v0, (block_num)); -FC_REFLECT(eosio::get_block_result_v0, (block_num)(deltas)); +FC_REFLECT(eosio::get_block_result_v0, (block_num)(deltas)(traces)); diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp index 933b2cd35a6..64412494070 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -34,10 +35,33 @@ const T& as_type(const T& x) { } template -void serialize_shared_vector(datastream& ds, const T& v) { +datastream& history_serialize_container(datastream& ds, const T& v) { fc::raw::pack(ds, unsigned_int(v.size())); for (auto& x : v) ds << make_history_serial_wrapper(x); + return ds; +} + +template +datastream& history_serialize_container(datastream& ds, const std::vector>& v) { + fc::raw::pack(ds, unsigned_int(v.size())); + for (auto& x : v) { + EOS_ASSERT(!!x, eosio::chain::plugin_exception, "null inside container"); + ds << make_history_serial_wrapper(*x); + } + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper>& obj) { + return history_serialize_container(ds, obj.obj); +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper>& obj) { + fc::raw::pack(ds, obj.obj.first); + fc::raw::pack(ds, obj.obj.second); + return ds; } template @@ -167,7 +191,7 @@ datastream& operator<<(datastream& const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.version)); - serialize_shared_vector(ds, obj.obj.producers); + history_serialize_container(ds, obj.obj.producers); return ds; } @@ -279,9 +303,9 @@ template datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.threshold)); - serialize_shared_vector(ds, obj.obj.keys); - serialize_shared_vector(ds, obj.obj.accounts); - serialize_shared_vector(ds, obj.obj.waits); + history_serialize_container(ds, obj.obj.keys); + history_serialize_container(ds, obj.obj.accounts); + history_serialize_container(ds, obj.obj.waits); return ds; } @@ -413,4 +437,63 @@ operator<<(datastream& return ds; }; +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.account.value)); + fc::raw::pack(ds, as_type(obj.obj.name.value)); + history_serialize_container(ds, as_type>(obj.obj.authorization)); + fc::raw::pack(ds, as_type(obj.obj.data)); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.receiver.value)); + fc::raw::pack(ds, as_type(obj.obj.act_digest)); + fc::raw::pack(ds, as_type(obj.obj.global_sequence)); + fc::raw::pack(ds, as_type(obj.obj.recv_sequence)); + history_serialize_container(ds, as_type>(obj.obj.auth_sequence)); + fc::raw::pack(ds, as_type(obj.obj.code_sequence)); + fc::raw::pack(ds, as_type(obj.obj.abi_sequence)); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, as_type(obj.obj.account.value)); + fc::raw::pack(ds, as_type(obj.obj.delta)); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.receipt))); + fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.act))); + fc::raw::pack(ds, as_type(obj.obj.context_free)); + fc::raw::pack(ds, as_type(obj.obj.elapsed.count())); + fc::raw::pack(ds, as_type(obj.obj.cpu_usage)); + fc::raw::pack(ds, as_type(obj.obj.console)); + fc::raw::pack(ds, as_type(obj.obj.total_cpu_usage)); + history_serialize_container(ds, as_type>(obj.obj.account_ram_deltas)); + history_serialize_container(ds, as_type>(obj.obj.inline_traces)); + return ds; +} + +template +datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.id)); + fc::raw::pack(ds, as_type(obj.obj.receipt->status.value)); + fc::raw::pack(ds, as_type(obj.obj.receipt->cpu_usage_us)); + fc::raw::pack(ds, as_type(obj.obj.receipt->net_usage_words)); + fc::raw::pack(ds, as_type(obj.obj.elapsed.count())); + fc::raw::pack(ds, as_type(obj.obj.net_usage)); + fc::raw::pack(ds, as_type(obj.obj.scheduled)); + history_serialize_container(ds, as_type>(obj.obj.action_traces)); + return ds; +} + } // namespace fc diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 108f31a53da..a23db544bde 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -69,13 +69,16 @@ auto catch_and_log(F f) { } struct state_history_plugin_impl : std::enable_shared_from_this { - chain_plugin* chain_plug = nullptr; - history_log state_log{"state_history"}; - bool stopping = false; - fc::optional accepted_block_connection; - string endpoint_address = "0.0.0.0"; - uint16_t endpoint_port = 4321; - std::unique_ptr acceptor; + chain_plugin* chain_plug = nullptr; + history_log state_log{"state_history"}; + history_log trace_log{"trace_history"}; + bool stopping = false; + fc::optional applied_transaction_connection; + fc::optional accepted_block_connection; + string endpoint_address = "0.0.0.0"; + uint16_t endpoint_port = 4321; + std::unique_ptr acceptor; + std::map traces; void get_state(uint32_t block_num, bytes& deltas) { history_log_header header; @@ -87,6 +90,16 @@ struct state_history_plugin_impl : std::enable_shared_from_this { std::shared_ptr plugin; std::unique_ptr> stream; @@ -178,6 +191,10 @@ struct state_history_plugin_impl : std::enable_shared_from_thisget_state(req.block_num, *result.deltas); } + if (req.block_num >= plugin->trace_log.begin_block() && req.block_num < plugin->trace_log.end_block()) { + result.traces.emplace(); + plugin->get_trace(req.block_num, *result.traces); + } send(std::move(result)); } @@ -262,7 +279,41 @@ struct state_history_plugin_impl : std::enable_shared_from_thisid] = p; } + void on_accepted_block(const block_state_ptr& block_state) { + // todo: config options + store_traces(block_state); + store_state(block_state); + } + + void store_traces(const block_state_ptr& block_state) { + std::vector traces; + for (auto& p : block_state->trxs) { + auto it = this->traces.find(p->id); + if (it == this->traces.end() || !it->second->receipt) { + ilog("missing trace for transaction {id}", ("id", p->id)); + continue; + } + traces.push_back(it->second); + } + this->traces.clear(); + + auto traces_bin = fc::raw::pack(make_history_serial_wrapper(traces)); + EOS_ASSERT(traces_bin.size() == (uint32_t)traces_bin.size(), plugin_exception, "traces is too big"); + + history_log_header header{.block_num = block_state->block->block_num(), + .block_id = block_state->block->id(), + .payload_size = sizeof(uint32_t) + traces_bin.size()}; + trace_log.write_entry(header, block_state->block->previous, [&](auto& stream) { + uint32_t s = (uint32_t)traces_bin.size(); + stream.write((char*)&s, sizeof(s)); + if (!traces_bin.empty()) + stream.write(traces_bin.data(), traces_bin.size()); + }); + } + + void store_state(const block_state_ptr& block_state) { bool fresh = state_log.begin_block() == state_log.end_block(); if (fresh) ilog("Placing initial state in block ${n}", ("n", block_state->block->block_num())); @@ -313,7 +364,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain_plug = app().find_plugin(); EOS_ASSERT(my->chain_plug, chain::missing_chain_plugin_exception, ""); auto& chain = my->chain_plug->chain(); + my->applied_transaction_connection.emplace( + chain.applied_transaction.connect([&](const transaction_trace_ptr& p) { my->on_applied_transaction(p); })); my->accepted_block_connection.emplace( chain.accepted_block.connect([&](const block_state_ptr& p) { my->on_accepted_block(p); })); @@ -362,6 +415,8 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { boost::filesystem::create_directories(state_history_dir); my->state_log.open((state_history_dir / "state_history.log").string(), (state_history_dir / "state_history.index").string()); + my->trace_log.open((state_history_dir / "trace_history.log").string(), + (state_history_dir / "trace_history.index").string()); } FC_LOG_AND_RETHROW() } // state_history_plugin::plugin_initialize @@ -369,6 +424,7 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { void state_history_plugin::plugin_startup() { my->listen(); } void state_history_plugin::plugin_shutdown() { + my->applied_transaction_connection.reset(); my->accepted_block_connection.reset(); while (!my->sessions.empty()) my->sessions.begin()->second->close(); diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index 948b96b528c..0c170ebceb0 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -20,7 +20,8 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "get_block_result_v0", "fields": [ { "name": "block_num", "type": "uint32" }, - { "name": "deltas", "type": "bytes?" } + { "name": "deltas", "type": "bytes?" }, + { "name": "traces", "type": "bytes?" } ] }, { @@ -36,6 +37,63 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "removed", "type": "uint64[]" } ] }, + { + "name": "action_v0", "fields": [ + { "name": "account", "type": "name" }, + { "name": "name", "type": "name" }, + { "name": "authorization", "type": "permission_level[]" }, + { "name": "data", "type": "bytes" } + ] + }, + { + "name": "account_auth_sequence", "fields": [ + { "name": "account", "type": "name" }, + { "name": "sequence", "type": "uint64" } + ] + }, + { + "name": "action_receipt_v0", "fields": [ + { "name": "receiver", "type": "name" }, + { "name": "act_digest", "type": "checksum256" }, + { "name": "global_sequence", "type": "uint64" }, + { "name": "recv_sequence", "type": "uint64" }, + { "name": "auth_sequence", "type": "account_auth_sequence[]" }, + { "name": "code_sequence", "type": "varuint32" }, + { "name": "abi_sequence", "type": "varuint32" } + ] + }, + { + "name": "account_delta", "fields": [ + { "name": "account", "type": "name" }, + { "name": "delta", "type": "int64" } + ] + }, + { + "name": "action_trace_v0", "fields": [ + { "name": "receipt", "type": "action_receipt" }, + { "name": "act", "type": "action" }, + + { "name": "context_free", "type": "bool" }, + { "name": "elapsed", "type": "int64" }, + { "name": "cpu_usage", "type": "uint64" }, + { "name": "console", "type": "string" }, + { "name": "total_cpu_usage", "type": "uint64" }, + { "name": "account_ram_deltas", "type": "account_delta[]" }, + { "name": "inline_traces", "type": "action_trace[]" } + ] + }, + { + "name": "transaction_trace_v0", "fields": [ + { "name": "id", "type": "checksum256" }, + { "name": "status", "type": "uint8" }, + { "name": "cpu_usage_us", "type": "uint32" }, + { "name": "net_usage_words", "type": "varuint32" }, + { "name": "elapsed", "type": "int64" }, + { "name": "net_usage", "type": "uint64" }, + { "name": "scheduled", "type": "bool" }, + { "name": "action_traces", "type": "action_trace[]" } + ] + }, { "name": "account_v0", "fields": [ { "type": "uint64", "name": "id" }, @@ -325,6 +383,11 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "request", "types": ["get_status_request_v0", "get_block_request_v0"] }, { "name": "result", "types": ["get_status_result_v0", "get_block_result_v0"] }, + { "name": "action", "types": ["action_v0"] }, + { "name": "action_receipt", "types": ["action_receipt_v0"] }, + { "name": "action_trace", "types": ["action_trace_v0"] }, + { "name": "transaction_trace", "types": ["transaction_trace_v0"] }, + { "name": "table_delta", "types": ["table_delta_v0"] }, { "name": "account", "types": ["account_v0"] }, { "name": "account_sequence", "types": ["account_sequence_v0"] }, From 6de4046d535f52ad74895cdf56e2d87b51903eb0 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Wed, 10 Oct 2018 16:54:22 -0400 Subject: [PATCH 024/369] state history plugin --- .../state_history_plugin.hpp | 4 +- .../state_history_plugin.cpp | 61 ++++++++----------- .../state_history_plugin_abi.cpp | 4 +- 3 files changed, 31 insertions(+), 38 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index 465a4692f45..605eca96767 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -39,8 +39,8 @@ struct get_block_request_v0 { struct get_block_result_v0 { uint32_t block_num = 0; - fc::optional deltas; fc::optional traces; + fc::optional deltas; }; using state_request = fc::static_variant; @@ -70,4 +70,4 @@ FC_REFLECT_EMPTY(eosio::get_status_request_v0); FC_REFLECT(eosio::get_status_result_v0, (last_irreversible_block_num)(last_irreversible_block_id)(state_begin_block_num)(state_end_block_num)); FC_REFLECT(eosio::get_block_request_v0, (block_num)); -FC_REFLECT(eosio::get_block_result_v0, (block_num)(deltas)(traces)); +FC_REFLECT(eosio::get_block_result_v0, (block_num)(traces)(deltas)); diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index a23db544bde..b780bf63649 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -70,8 +70,8 @@ auto catch_and_log(F f) { struct state_history_plugin_impl : std::enable_shared_from_this { chain_plugin* chain_plug = nullptr; - history_log state_log{"state_history"}; history_log trace_log{"trace_history"}; + history_log chain_state_log{"chain_state_history"}; bool stopping = false; fc::optional applied_transaction_connection; fc::optional accepted_block_connection; @@ -80,30 +80,24 @@ struct state_history_plugin_impl : std::enable_shared_from_this acceptor; std::map traces; - void get_state(uint32_t block_num, bytes& deltas) { + void get_data(history_log& log, uint32_t block_num, fc::optional& result) { + if (block_num < log.begin_block() || block_num >= log.end_block()) + return; history_log_header header; - auto& stream = state_log.get_entry(block_num, header); + auto& stream = log.get_entry(block_num, header); uint32_t s; stream.read((char*)&s, sizeof(s)); - deltas.resize(s); + result.emplace(); + result->resize(s); if (s) - stream.read(deltas.data(), s); - } - - void get_trace(uint32_t block_num, bytes& traces) { - history_log_header header; - auto& stream = trace_log.get_entry(block_num, header); - uint32_t s; - stream.read((char*)&s, sizeof(s)); - traces.resize(s); - if (s) - stream.read(traces.data(), s); + stream.read(result->data(), s); } struct session : std::enable_shared_from_this { std::shared_ptr plugin; std::unique_ptr> stream; bool sending = false; + bool sentAbi = false; std::vector> send_queue; session(std::shared_ptr plugin) @@ -145,7 +139,6 @@ struct state_history_plugin_impl : std::enable_shared_from_thisbinary(sentAbi); + sentAbi = true; stream->async_write( // boost::asio::buffer(send_queue[0]), [self = shared_from_this(), this](boost::system::error_code ec, size_t) { @@ -179,22 +174,17 @@ struct state_history_plugin_impl : std::enable_shared_from_thisstate_log.begin_block(); - result.state_end_block_num = plugin->state_log.end_block(); + result.state_begin_block_num = plugin->chain_state_log.begin_block(); + result.state_end_block_num = plugin->chain_state_log.end_block(); send(std::move(result)); } void operator()(get_block_request_v0& req) { // ilog("${b} get_block_request_v0", ("b", req.block_num)); get_block_result_v0 result{req.block_num}; - if (req.block_num >= plugin->state_log.begin_block() && req.block_num < plugin->state_log.end_block()) { - result.deltas.emplace(); - plugin->get_state(req.block_num, *result.deltas); - } - if (req.block_num >= plugin->trace_log.begin_block() && req.block_num < plugin->trace_log.end_block()) { - result.traces.emplace(); - plugin->get_trace(req.block_num, *result.traces); - } + // todo: client select which datasets to receive + plugin->get_data(plugin->trace_log, req.block_num, result.traces); + plugin->get_data(plugin->chain_state_log, req.block_num, result.deltas); send(std::move(result)); } @@ -279,12 +269,15 @@ struct state_history_plugin_impl : std::enable_shared_from_thisid] = p; } + void on_applied_transaction(const transaction_trace_ptr& p) { + if (p->receipt) + traces[p->id] = p; + } void on_accepted_block(const block_state_ptr& block_state) { // todo: config options store_traces(block_state); - store_state(block_state); + store_chain_state(block_state); } void store_traces(const block_state_ptr& block_state) { @@ -313,8 +306,8 @@ struct state_history_plugin_impl : std::enable_shared_from_thisblock->block_num())); @@ -358,13 +351,13 @@ struct state_history_plugin_impl : std::enable_shared_from_thisblock->block_num(), .block_id = block_state->block->id(), .payload_size = sizeof(uint32_t) + deltas_bin.size()}; - state_log.write_entry(header, block_state->block->previous, [&](auto& stream) { + chain_state_log.write_entry(header, block_state->block->previous, [&](auto& stream) { uint32_t s = (uint32_t)deltas_bin.size(); stream.write((char*)&s, sizeof(s)); if (!deltas_bin.empty()) stream.write(deltas_bin.data(), deltas_bin.size()); }); - } // store_state + } // store_chain_state }; // state_history_plugin_impl state_history_plugin::state_history_plugin() @@ -413,10 +406,10 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { } boost::filesystem::create_directories(state_history_dir); - my->state_log.open((state_history_dir / "state_history.log").string(), - (state_history_dir / "state_history.index").string()); my->trace_log.open((state_history_dir / "trace_history.log").string(), (state_history_dir / "trace_history.index").string()); + my->chain_state_log.open((state_history_dir / "chain_state_history.log").string(), + (state_history_dir / "chain_state_history.index").string()); } FC_LOG_AND_RETHROW() } // state_history_plugin::plugin_initialize diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index 0c170ebceb0..ca7da05f8d0 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -20,8 +20,8 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "get_block_result_v0", "fields": [ { "name": "block_num", "type": "uint32" }, - { "name": "deltas", "type": "bytes?" }, - { "name": "traces", "type": "bytes?" } + { "name": "traces", "type": "bytes?" }, + { "name": "deltas", "type": "bytes?" } ] }, { From 36bc2a7365898114d366baec1ff854bf91a62efe Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Thu, 11 Oct 2018 09:49:42 -0400 Subject: [PATCH 025/369] state history: fetch blocks --- .../state_history_plugin.hpp | 4 +- .../state_history_serialization.hpp | 4 +- .../state_history_plugin.cpp | 15 +++++ .../state_history_plugin_abi.cpp | 64 +++++++++++++++++-- 4 files changed, 78 insertions(+), 9 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index 605eca96767..54472feade7 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -39,6 +39,8 @@ struct get_block_request_v0 { struct get_block_result_v0 { uint32_t block_num = 0; + fc::optional block; + fc::optional block_state; fc::optional traces; fc::optional deltas; }; @@ -70,4 +72,4 @@ FC_REFLECT_EMPTY(eosio::get_status_request_v0); FC_REFLECT(eosio::get_status_result_v0, (last_irreversible_block_num)(last_irreversible_block_id)(state_begin_block_num)(state_end_block_num)); FC_REFLECT(eosio::get_block_request_v0, (block_num)); -FC_REFLECT(eosio::get_block_result_v0, (block_num)(traces)(deltas)); +FC_REFLECT(eosio::get_block_result_v0, (block_num)(block)(block_state)(traces)(deltas)); diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp index 64412494070..185c4d64052 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -180,7 +180,6 @@ datastream& operator<<(datastream& template datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.producer_name.value)); fc::raw::pack(ds, as_type(obj.obj.block_signing_key)); return ds; @@ -189,9 +188,8 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.version)); - history_serialize_container(ds, obj.obj.producers); + history_serialize_container(ds, as_type>(obj.obj.producers)); return ds; } diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index b780bf63649..95f0736e7aa 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -70,6 +70,7 @@ auto catch_and_log(F f) { struct state_history_plugin_impl : std::enable_shared_from_this { chain_plugin* chain_plug = nullptr; + history_log block_state_log{"block_state_history"}; history_log trace_log{"trace_history"}; history_log chain_state_log{"chain_state_history"}; bool stopping = false; @@ -93,6 +94,11 @@ struct state_history_plugin_impl : std::enable_shared_from_thisdata(), s); } + void get_block(uint32_t block_num, fc::optional& result) { + auto p = chain_plug->chain().fetch_block_by_number(block_num); + result = fc::raw::pack(*p); + } + struct session : std::enable_shared_from_this { std::shared_ptr plugin; std::unique_ptr> stream; @@ -183,6 +189,8 @@ struct state_history_plugin_impl : std::enable_shared_from_thisget_block(req.block_num, result.block); + plugin->get_data(plugin->block_state_log, req.block_num, result.block_state); plugin->get_data(plugin->trace_log, req.block_num, result.traces); plugin->get_data(plugin->chain_state_log, req.block_num, result.deltas); send(std::move(result)); @@ -276,10 +284,15 @@ struct state_history_plugin_impl : std::enable_shared_from_this traces; for (auto& p : block_state->trxs) { @@ -406,6 +419,8 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { } boost::filesystem::create_directories(state_history_dir); + my->block_state_log.open((state_history_dir / "block_state_history.log").string(), + (state_history_dir / "block_state_history.index").string()); my->trace_log.open((state_history_dir / "trace_history.log").string(), (state_history_dir / "trace_history.index").string()); my->chain_state_log.open((state_history_dir / "chain_state_history.log").string(), diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index ca7da05f8d0..b3d421c0c32 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -20,6 +20,8 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "get_block_result_v0", "fields": [ { "name": "block_num", "type": "uint32" }, + { "name": "block", "type": "bytes?" }, + { "name": "block_state", "type": "bytes?" }, { "name": "traces", "type": "bytes?" }, { "name": "deltas", "type": "bytes?" } ] @@ -94,6 +96,56 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "action_traces", "type": "action_trace[]" } ] }, + { + "name": "packed_transaction", "fields": [ + { "name": "signatures", "type": "signature[]" }, + { "name": "compression", "type": "uint8" }, + { "name": "packed_context_free_data", "type": "bytes" }, + { "name": "packed_trx", "type": "bytes" } + ] + }, + { + "name": "transaction_receipt_header", "fields": [ + { "name": "status", "type": "uint8" }, + { "name": "cpu_usage_us", "type": "uint32" }, + { "name": "net_usage_words", "type": "varuint32" } + ] + }, + { + "name": "transaction_receipt", "base": "transaction_receipt_header", "fields": [ + { "name": "trx", "type": "transaction_variant" } + ] + }, + { + "name": "extension", "fields": [ + { "name": "type", "type": "uint16" }, + { "name": "data", "type": "bytes" } + ] + }, + { + "name": "block_header", "fields": [ + { "name": "timestamp", "type": "block_timestamp_type" }, + { "name": "producer", "type": "name" }, + { "name": "confirmed; ", "type": "uint16" }, + { "name": "previous", "type": "checksum256" }, + { "name": "transaction_mroot", "type": "checksum256" }, + { "name": "action_mroot", "type": "checksum256" }, + { "name": "schedule_version", "type": "uint32" }, + { "name": "new_producers", "type": "producer_schedule?" }, + { "name": "header_extensions", "type": "extension[]" } + ] + }, + { + "name": "signed_block_header", "base": "block_header", "fields": [ + { "name": "producer_signature", "type": "signature" } + ] + }, + { + "name": "signed_block", "base": "signed_block_header", "fields": [ + { "name": "transactions", "type": "transaction_receipt[]" }, + { "name": "block_extensions", "type": "extension[]" } + ] + }, { "name": "account_v0", "fields": [ { "type": "uint64", "name": "id" }, @@ -189,13 +241,13 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "producer_key_v0", "fields": [ + "name": "producer_key", "fields": [ { "type": "name", "name": "producer_name" }, { "type": "public_key", "name": "block_signing_key" } ] }, { - "name": "shared_producer_schedule_type_v0", "fields": [ + "name": "producer_schedule", "fields": [ { "type": "uint32", "name": "version" }, { "type": "producer_key[]", "name": "producers" } ] @@ -225,7 +277,7 @@ extern const char* const state_history_plugin_abi = R"({ "name": "global_property_v0", "fields": [ { "type": "uint64", "name": "id" }, { "type": "uint32?", "name": "proposed_schedule_block_num" }, - { "type": "shared_producer_schedule_type", "name": "proposed_schedule" }, + { "type": "producer_schedule", "name": "proposed_schedule" }, { "type": "chain_config", "name": "configuration" } ] }, @@ -379,6 +431,9 @@ extern const char* const state_history_plugin_abi = R"({ ] } ], + "types": [ + { "new_type_name": "transaction_id", "type": "checksum256" } + ], "variants": [ { "name": "request", "types": ["get_status_request_v0", "get_block_request_v0"] }, { "name": "result", "types": ["get_status_result_v0", "get_block_result_v0"] }, @@ -387,6 +442,7 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "action_receipt", "types": ["action_receipt_v0"] }, { "name": "action_trace", "types": ["action_trace_v0"] }, { "name": "transaction_trace", "types": ["transaction_trace_v0"] }, + { "name": "transaction_variant", "types": ["transaction_id", "packed_transaction"] }, { "name": "table_delta", "types": ["table_delta_v0"] }, { "name": "account", "types": ["account_v0"] }, @@ -398,8 +454,6 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "index256", "types": ["index256_v0"] }, { "name": "index_double", "types": ["index_double_v0"] }, { "name": "index_long_double", "types": ["index_long_double_v0"] }, - { "name": "producer_key", "types": ["producer_key_v0"] }, - { "name": "shared_producer_schedule_type", "types": ["shared_producer_schedule_type_v0"] }, { "name": "chain_config", "types": ["chain_config_v0"] }, { "name": "global_property", "types": ["global_property_v0"] }, { "name": "dynamic_global_property", "types": ["dynamic_global_property_v0"] }, From 7ea4135e3aa69772b7b85937616fcb163baf809b Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Thu, 11 Oct 2018 10:57:17 -0400 Subject: [PATCH 026/369] state history: remove probably-not-externally-useful tables --- .../state_history_serialization.hpp | 41 ------------------- .../state_history_plugin.cpp | 4 -- .../state_history_plugin_abi.cpp | 36 ---------------- 3 files changed, 81 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp index 185c4d64052..7c3eb8e31e4 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -14,7 +14,6 @@ #include #include #include -#include #include template @@ -80,19 +79,6 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.name.value)); - fc::raw::pack(ds, as_type(obj.obj.recv_sequence)); - fc::raw::pack(ds, as_type(obj.obj.auth_sequence)); - fc::raw::pack(ds, as_type(obj.obj.code_sequence)); - fc::raw::pack(ds, as_type(obj.obj.abi_sequence)); - return ds; -} - template datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); @@ -230,24 +216,6 @@ datastream& operator<<(datastream& return ds; } -template -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.global_action_sequence)); - return ds; -} - -template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.expiration)); - fc::raw::pack(ds, as_type(obj.obj.trx_id)); - return ds; -} - template datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { @@ -320,15 +288,6 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.last_used)); - return ds; -} - template datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 95f0736e7aa..5dd6c15866c 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -29,7 +29,6 @@ static appbase::abstract_plugin& _state_history_plugin = app().register_plugin static void for_each_table(const chainbase::database& db, F f) { f("account", db.get_index(), [](auto&) { return true; }); - f("account_sequence", db.get_index(), [](auto&) { return true; }); f("table_id", db.get_index(), [](auto&) { return true; }); f("key_value", db.get_index(), [](auto&) { return true; }); @@ -40,12 +39,9 @@ static void for_each_table(const chainbase::database& db, F f) { f("index_long_double", db.get_index(), [](auto&) { return true; }); f("global_property", db.get_index(), [](auto&) { return true; }); - f("dynamic_global_property", db.get_index(), [](auto&) { return true; }); - f("transaction", db.get_index(), [](auto&) { return true; }); f("generated_transaction", db.get_index(), [](auto&) { return true; }); f("permission", db.get_index(), [](auto&) { return true; }); - f("permission_usage", db.get_index(), [](auto&) { return true; }); f("permission_link", db.get_index(), [](auto&) { return true; }); f("resource_limits", db.get_index(), [](auto&) { return true; }); diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index b3d421c0c32..25c4af7de25 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -160,16 +160,6 @@ extern const char* const state_history_plugin_abi = R"({ { "type": "bytes", "name": "abi" } ] }, - { - "name": "account_sequence_v0", "fields": [ - { "type": "uint64", "name": "id" }, - { "type": "name", "name": "name" }, - { "type": "uint64", "name": "recv_sequence" }, - { "type": "uint64", "name": "auth_sequence" }, - { "type": "uint64", "name": "code_sequence" }, - { "type": "uint64", "name": "abi_sequence" } - ] - }, { "name": "table_id_v0", "fields": [ { "type": "uint64", "name": "id" }, @@ -281,19 +271,6 @@ extern const char* const state_history_plugin_abi = R"({ { "type": "chain_config", "name": "configuration" } ] }, - { - "name": "dynamic_global_property_v0", "fields": [ - { "type": "uint64", "name": "id" }, - { "type": "uint64", "name": "global_action_sequence" } - ] - }, - { - "name": "transaction_v0", "fields": [ - { "type": "uint64", "name": "id" }, - { "type": "time_point_sec", "name": "expiration" }, - { "type": "checksum256", "name": "trx_id" } - ] - }, { "name": "generated_transaction_v0", "fields": [ { "type": "uint64", "name": "id" }, @@ -350,12 +327,6 @@ extern const char* const state_history_plugin_abi = R"({ { "type": "shared_authority", "name": "auth" } ] }, - { - "name": "permission_usage_v0", "fields": [ - { "type": "uint64", "name": "id" }, - { "type": "time_point", "name": "last_used" } - ] - }, { "name": "permission_link_v0", "fields": [ { "type": "uint64", "name": "id" }, @@ -446,7 +417,6 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "table_delta", "types": ["table_delta_v0"] }, { "name": "account", "types": ["account_v0"] }, - { "name": "account_sequence", "types": ["account_sequence_v0"] }, { "name": "table_id", "types": ["table_id_v0"] }, { "name": "key_value", "types": ["key_value_v0"] }, { "name": "index64", "types": ["index64_v0"] }, @@ -456,8 +426,6 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "index_long_double", "types": ["index_long_double_v0"] }, { "name": "chain_config", "types": ["chain_config_v0"] }, { "name": "global_property", "types": ["global_property_v0"] }, - { "name": "dynamic_global_property", "types": ["dynamic_global_property_v0"] }, - { "name": "transaction", "types": ["transaction_v0"] }, { "name": "generated_transaction", "types": ["generated_transaction_v0"] }, { "name": "key_weight", "types": ["key_weight_v0"] }, { "name": "permission_level", "types": ["permission_level_v0"] }, @@ -465,7 +433,6 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "wait_weight", "types": ["wait_weight_v0"] }, { "name": "shared_authority", "types": ["shared_authority_v0"] }, { "name": "permission", "types": ["permission_v0"] }, - { "name": "permission_usage", "types": ["permission_usage_v0"] }, { "name": "permission_link", "types": ["permission_link_v0"] }, { "name": "resource_limits", "types": ["resource_limits_v0"] }, { "name": "usage_accumulator", "types": ["usage_accumulator_v0"] }, @@ -477,7 +444,6 @@ extern const char* const state_history_plugin_abi = R"({ ], "tables": [ { "name": "account", "type": "account" }, - { "name": "account_sequence", "type": "account_sequence" }, { "name": "table_id", "type": "table_id" }, { "name": "key_value", "type": "key_value" }, { "name": "index64", "type": "index64" }, @@ -486,11 +452,9 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "index_double", "type": "index_double" }, { "name": "index_long_double", "type": "index_long_double" }, { "name": "global_property", "type": "global_property" }, - { "name": "dynamic_global_property", "type": "dynamic_global_property" }, { "name": "transaction", "type": "transaction" }, { "name": "generated_transaction", "type": "generated_transaction" }, { "name": "permission", "type": "permission" }, - { "name": "permission_usage", "type": "permission_usage" }, { "name": "permission_link", "type": "permission_link" }, { "name": "resource_limits", "type": "resource_limits" }, { "name": "resource_usage", "type": "resource_usage" }, From 25b021025b8fb6d85baef0f32cd69a774efeacce Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Thu, 11 Oct 2018 16:56:25 -0400 Subject: [PATCH 027/369] history plugin: rework contract tables --- .../state_history_plugin.hpp | 9 +- .../state_history_serialization.hpp | 51 +++++---- .../state_history_plugin.cpp | 85 ++++++++------ .../state_history_plugin_abi.cpp | 105 +++++++++--------- tutorials/bios-boot-tutorial/.gitignore | 1 + 5 files changed, 139 insertions(+), 112 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index 54472feade7..fcb54c362cd 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -18,10 +18,9 @@ using std::shared_ptr; typedef shared_ptr state_history_ptr; struct table_delta { - fc::unsigned_int struct_version = 0; - std::string name{}; - std::vector> rows{}; - std::vector removed{}; + fc::unsigned_int struct_version = 0; + std::string name{}; + std::vector> rows{}; }; struct get_status_request_v0 {}; @@ -67,7 +66,7 @@ class state_history_plugin : public plugin { } // namespace eosio -FC_REFLECT(eosio::table_delta, (struct_version)(name)(rows)(removed)); +FC_REFLECT(eosio::table_delta, (struct_version)(name)(rows)); FC_REFLECT_EMPTY(eosio::get_status_request_v0); FC_REFLECT(eosio::get_status_result_v0, (last_irreversible_block_num)(last_irreversible_block_id)(state_begin_block_num)(state_end_block_num)); diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp index 7c3eb8e31e4..e0ec9bd916a 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -26,6 +26,17 @@ history_serial_wrapper make_history_serial_wrapper(const T& obj) { return {obj}; } +template +struct history_table_wrapper { + const eosio::chain::table_id_object& table_id; + const T& obj; +}; + +template +history_table_wrapper make_history_table_wrapper(const eosio::chain::table_id_object& table_id, const T& obj) { + return {table_id, obj}; +} + namespace fc { template @@ -66,7 +77,6 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.name.value)); fc::raw::pack(ds, as_type(obj.obj.vm_type)); fc::raw::pack(ds, as_type(obj.obj.vm_version)); @@ -82,20 +92,19 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.code.value)); fc::raw::pack(ds, as_type(obj.obj.scope.value)); fc::raw::pack(ds, as_type(obj.obj.table.value)); fc::raw::pack(ds, as_type(obj.obj.payer.value)); - fc::raw::pack(ds, as_type(obj.obj.count)); return ds; } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +datastream& operator<<(datastream& ds, const history_table_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type(obj.obj.t_id._id)); + fc::raw::pack(ds, as_type(obj.table_id.code.value)); + fc::raw::pack(ds, as_type(obj.table_id.scope.value)); + fc::raw::pack(ds, as_type(obj.table_id.table.value)); fc::raw::pack(ds, as_type(obj.obj.primary_key)); fc::raw::pack(ds, as_type(obj.obj.payer.value)); fc::raw::pack(ds, as_type(obj.obj.value)); @@ -128,10 +137,12 @@ void serialize_secondary_index_data(datastream& ds, const eosio::chain::key2 } template -datastream& serialize_secondary_index(datastream& ds, const T& obj) { +datastream& serialize_secondary_index(datastream& ds, const eosio::chain::table_id_object& table_id, + const T& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.id._id)); - fc::raw::pack(ds, as_type(obj.t_id._id)); + fc::raw::pack(ds, as_type(table_id.code.value)); + fc::raw::pack(ds, as_type(table_id.scope.value)); + fc::raw::pack(ds, as_type(table_id.table.value)); fc::raw::pack(ds, as_type(obj.primary_key)); fc::raw::pack(ds, as_type(obj.payer.value)); serialize_secondary_index_data(ds, obj.secondary_key); @@ -139,29 +150,29 @@ datastream& serialize_secondary_index(datastream& ds, const T& obj) { } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - return serialize_secondary_index(ds, obj.obj); +datastream& operator<<(datastream& ds, const history_table_wrapper& obj) { + return serialize_secondary_index(ds, obj.table_id, obj.obj); } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - return serialize_secondary_index(ds, obj.obj); +datastream& operator<<(datastream& ds, const history_table_wrapper& obj) { + return serialize_secondary_index(ds, obj.table_id, obj.obj); } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - return serialize_secondary_index(ds, obj.obj); +datastream& operator<<(datastream& ds, const history_table_wrapper& obj) { + return serialize_secondary_index(ds, obj.table_id, obj.obj); } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - return serialize_secondary_index(ds, obj.obj); +datastream& operator<<(datastream& ds, const history_table_wrapper& obj) { + return serialize_secondary_index(ds, obj.table_id, obj.obj); } template -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { - return serialize_secondary_index(ds, obj.obj); +datastream& operator<<(datastream& ds, + const history_table_wrapper& obj) { + return serialize_secondary_index(ds, obj.table_id, obj.obj); } template diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 5dd6c15866c..f53afb7e08b 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -26,31 +26,6 @@ using boost::signals2::scoped_connection; static appbase::abstract_plugin& _state_history_plugin = app().register_plugin(); -template -static void for_each_table(const chainbase::database& db, F f) { - f("account", db.get_index(), [](auto&) { return true; }); - - f("table_id", db.get_index(), [](auto&) { return true; }); - f("key_value", db.get_index(), [](auto&) { return true; }); - f("index64", db.get_index(), [](auto&) { return true; }); - f("index128", db.get_index(), [](auto&) { return true; }); - f("index256", db.get_index(), [](auto&) { return true; }); - f("index_double", db.get_index(), [](auto&) { return true; }); - f("index_long_double", db.get_index(), [](auto&) { return true; }); - - f("global_property", db.get_index(), [](auto&) { return true; }); - f("generated_transaction", db.get_index(), [](auto&) { return true; }); - - f("permission", db.get_index(), [](auto&) { return true; }); - f("permission_link", db.get_index(), [](auto&) { return true; }); - - f("resource_limits", db.get_index(), [](auto&) { return true; }); - f("resource_usage", db.get_index(), [](auto&) { return true; }); - f("resource_limits_state", db.get_index(), [](auto&) { return true; }); - f("resource_limits_config", db.get_index(), - [](auto&) { return true; }); -} - template auto catch_and_log(F f) { try { @@ -321,7 +296,29 @@ struct state_history_plugin_impl : std::enable_shared_from_thisblock->block_num())); std::vector deltas; - for_each_table(chain_plug->chain().db(), [&, this](auto* name, auto& index, auto filter) { + auto& db = chain_plug->chain().db(); + + const auto& table_id_index = db.get_index(); + std::map removed_table_id; + for (auto& rem : table_id_index.stack().back().removed_values) + removed_table_id[rem.first._id] = &rem.second; + + auto get_table_id = [&](uint64_t tid) -> const table_id_object& { + auto obj = table_id_index.find(tid); + if (obj) + return *obj; + auto it = removed_table_id.find(tid); + EOS_ASSERT(it != removed_table_id.end(), chain::plugin_exception, "can not found table id ${tid}", + ("tid", tid)); + return *it->second; + }; + + auto pack_row = [](auto& row) { return fc::raw::pack(make_history_serial_wrapper(row)); }; + auto pack_contract_row = [&](auto& row) { + return fc::raw::pack(make_history_table_wrapper(get_table_id(row.t_id._id), row)); + }; + + auto process_table = [&](auto* name, auto& index, auto& pack_row) { if (fresh) { if (index.indices().empty()) return; @@ -329,8 +326,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this(), pack_row); + + process_table("contract_table", db.get_index(), pack_row); + process_table("contract_row", db.get_index(), pack_contract_row); + process_table("contract_index64", db.get_index(), pack_contract_row); + process_table("contract_index128", db.get_index(), pack_contract_row); + process_table("contract_index256", db.get_index(), pack_contract_row); + process_table("contract_index_double", db.get_index(), pack_contract_row); + process_table("contract_index_long_double", db.get_index(), pack_contract_row); + + process_table("global_property", db.get_index(), pack_row); + process_table("generated_transaction", db.get_index(), pack_row); + + process_table("permission", db.get_index(), pack_row); + process_table("permission_link", db.get_index(), pack_row); + + process_table("resource_limits", db.get_index(), pack_row); + process_table("resource_usage", db.get_index(), pack_row); + process_table("resource_limits_state", db.get_index(), pack_row); + process_table("resource_limits_config", db.get_index(), pack_row); + auto deltas_bin = fc::raw::pack(deltas); EOS_ASSERT(deltas_bin.size() == (uint32_t)deltas_bin.size(), plugin_exception, "deltas is too big"); history_log_header header{.block_num = block_state->block->block_num(), diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index 25c4af7de25..db3a4292797 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -28,15 +28,14 @@ extern const char* const state_history_plugin_abi = R"({ }, { "name": "row", "fields": [ - { "name": "id", "type": "uint64" }, + { "name": "present", "type": "bool" }, { "name": "data", "type": "bytes" } ] }, { "name": "table_delta_v0", "fields": [ { "name": "name", "type": "string" }, - { "name": "rows", "type": "row[]" }, - { "name": "removed", "type": "uint64[]" } + { "name": "rows", "type": "row[]" } ] }, { @@ -148,7 +147,6 @@ extern const char* const state_history_plugin_abi = R"({ }, { "name": "account_v0", "fields": [ - { "type": "uint64", "name": "id" }, { "type": "name", "name": "name" }, { "type": "uint8", "name": "vm_type" }, { "type": "uint8", "name": "vm_version" }, @@ -161,70 +159,74 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "table_id_v0", "fields": [ - { "type": "uint64", "name": "id" }, + "name": "contract_table_v0", "fields": [ { "type": "name", "name": "code" }, { "type": "name", "name": "scope" }, { "type": "name", "name": "table" }, - { "type": "name", "name": "payer" }, - { "type": "uint32", "name": "count" } + { "type": "name", "name": "payer" } ] }, { - "name": "key_value_v0", "fields": [ - { "type": "uint64", "name": "id" }, - { "type": "uint64", "name": "t_id" }, + "name": "contract_row_v0", "fields": [ + { "type": "name", "name": "code" }, + { "type": "name", "name": "scope" }, + { "type": "name", "name": "table" }, { "type": "uint64", "name": "primary_key" }, { "type": "name", "name": "payer" }, { "type": "bytes", "name": "value" } ] }, { - "name": "index64_v0", "fields": [ - { "type": "uint64", "name": "id" }, - { "type": "uint64", "name": "t_id" }, + "name": "contract_index64_v0", "fields": [ + { "type": "name", "name": "code" }, + { "type": "name", "name": "scope" }, + { "type": "name", "name": "table" }, { "type": "uint64", "name": "primary_key" }, { "type": "name", "name": "payer" }, { "type": "uint64", "name": "secondary_key" } ] }, { - "name": "index128_v0", "fields": [ - { "type": "uint64", "name": "id" }, - { "type": "uint64", "name": "t_id" }, + "name": "contract_index128_v0", "fields": [ + { "type": "name", "name": "code" }, + { "type": "name", "name": "scope" }, + { "type": "name", "name": "table" }, { "type": "uint64", "name": "primary_key" }, { "type": "name", "name": "payer" }, { "type": "uint128", "name": "secondary_key" } ] }, { - "name": "index256_key", "fields": [ + "name": "contract_index256_key", "fields": [ { "type": "uint128", "name": "a0" }, { "type": "uint128", "name": "a1" } ] }, { - "name": "index256_v0", "fields": [ - { "type": "uint64", "name": "id" }, - { "type": "uint64", "name": "t_id" }, + "name": "contract_index256_v0", "fields": [ + { "type": "name", "name": "code" }, + { "type": "name", "name": "scope" }, + { "type": "name", "name": "table" }, { "type": "uint64", "name": "primary_key" }, { "type": "name", "name": "payer" }, - { "type": "index256_key", "name": "secondary_key" } + { "type": "contract_index256_key", "name": "secondary_key" } ] }, { - "name": "index_double_v0", "fields": [ - { "type": "uint64", "name": "id" }, - { "type": "uint64", "name": "t_id" }, + "name": "contract_index_double_v0", "fields": [ + { "type": "name", "name": "code" }, + { "type": "name", "name": "scope" }, + { "type": "name", "name": "table" }, { "type": "uint64", "name": "primary_key" }, { "type": "name", "name": "payer" }, { "type": "float64", "name": "secondary_key" } ] }, { - "name": "index_long_double_v0", "fields": [ - { "type": "uint64", "name": "id" }, - { "type": "uint64", "name": "t_id" }, + "name": "contract_index_long_double_v0", "fields": [ + { "type": "name", "name": "code" }, + { "type": "name", "name": "scope" }, + { "type": "name", "name": "table" }, { "type": "uint64", "name": "primary_key" }, { "type": "name", "name": "payer" }, { "type": "float128", "name": "secondary_key" } @@ -417,13 +419,13 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "table_delta", "types": ["table_delta_v0"] }, { "name": "account", "types": ["account_v0"] }, - { "name": "table_id", "types": ["table_id_v0"] }, - { "name": "key_value", "types": ["key_value_v0"] }, - { "name": "index64", "types": ["index64_v0"] }, - { "name": "index128", "types": ["index128_v0"] }, - { "name": "index256", "types": ["index256_v0"] }, - { "name": "index_double", "types": ["index_double_v0"] }, - { "name": "index_long_double", "types": ["index_long_double_v0"] }, + { "name": "contract_table", "types": ["contract_table_v0"] }, + { "name": "contract_row", "types": ["contract_row_v0"] }, + { "name": "contract_index64", "types": ["contract_index64_v0"] }, + { "name": "contract_index128", "types": ["contract_index128_v0"] }, + { "name": "contract_index256", "types": ["contract_index256_v0"] }, + { "name": "contract_index_double", "types": ["contract_index_double_v0"] }, + { "name": "contract_index_long_double", "types": ["contract_index_long_double_v0"] }, { "name": "chain_config", "types": ["chain_config_v0"] }, { "name": "global_property", "types": ["global_property_v0"] }, { "name": "generated_transaction", "types": ["generated_transaction_v0"] }, @@ -443,22 +445,21 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "resource_limits_config", "types": ["resource_limits_config_v0"] } ], "tables": [ - { "name": "account", "type": "account" }, - { "name": "table_id", "type": "table_id" }, - { "name": "key_value", "type": "key_value" }, - { "name": "index64", "type": "index64" }, - { "name": "index128", "type": "index128" }, - { "name": "index256", "type": "index256" }, - { "name": "index_double", "type": "index_double" }, - { "name": "index_long_double", "type": "index_long_double" }, - { "name": "global_property", "type": "global_property" }, - { "name": "transaction", "type": "transaction" }, - { "name": "generated_transaction", "type": "generated_transaction" }, - { "name": "permission", "type": "permission" }, - { "name": "permission_link", "type": "permission_link" }, - { "name": "resource_limits", "type": "resource_limits" }, - { "name": "resource_usage", "type": "resource_usage" }, - { "name": "resource_limits_state", "type": "resource_limits_state" }, - { "name": "resource_limits_config", "type": "resource_limits_config" } + { "name": "account", "type": "account", "key_names": ["name"] }, + { "name": "contract_table", "type": "contract_table", "key_names": ["code", "scope", "table"] }, + { "name": "contract_row", "type": "contract_row", "key_names": ["code", "scope", "table", "primary_key"] }, + { "name": "contract_index64", "type": "contract_index64", "key_names": ["code", "scope", "table", "primary_key"] }, + { "name": "contract_index128", "type": "contract_index128", "key_names": ["code", "scope", "table", "primary_key"] }, + { "name": "contract_index256", "type": "contract_index256", "key_names": ["code", "scope", "table", "primary_key"] }, + { "name": "contract_index_double", "type": "contract_index_double", "key_names": ["code", "scope", "table", "primary_key"] }, + { "name": "contract_index_long_double", "type": "contract_index_long_double", "key_names": ["code", "scope", "table", "primary_key"] }, + { "name": "global_property", "type": "global_property", "key_names": ["id"] }, + { "name": "generated_transaction", "type": "generated_transaction", "key_names": ["id"] }, + { "name": "permission", "type": "permission", "key_names": ["id"] }, + { "name": "permission_link", "type": "permission_link", "key_names": ["id"] }, + { "name": "resource_limits", "type": "resource_limits", "key_names": ["id"] }, + { "name": "resource_usage", "type": "resource_usage", "key_names": ["id"] }, + { "name": "resource_limits_state", "type": "resource_limits_state", "key_names": ["id"] }, + { "name": "resource_limits_config", "type": "resource_limits_config", "key_names": ["id"] } ] })"; diff --git a/tutorials/bios-boot-tutorial/.gitignore b/tutorials/bios-boot-tutorial/.gitignore index 87fe0a9c229..558c77bdf4f 100644 --- a/tutorials/bios-boot-tutorial/.gitignore +++ b/tutorials/bios-boot-tutorial/.gitignore @@ -1,2 +1,3 @@ nodes/ +wallet/ output.log From a1d829d054027b475db3b63d6204eb851e8e5df6 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Thu, 11 Oct 2018 17:14:16 -0400 Subject: [PATCH 028/369] state history: action_trace no longer has cpu_usage --- .../eosio/state_history_plugin/state_history_serialization.hpp | 2 -- plugins/state_history_plugin/state_history_plugin_abi.cpp | 2 -- 2 files changed, 4 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp index e0ec9bd916a..d2d6bb3d089 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -442,9 +442,7 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper(obj.obj.act))); fc::raw::pack(ds, as_type(obj.obj.context_free)); fc::raw::pack(ds, as_type(obj.obj.elapsed.count())); - fc::raw::pack(ds, as_type(obj.obj.cpu_usage)); fc::raw::pack(ds, as_type(obj.obj.console)); - fc::raw::pack(ds, as_type(obj.obj.total_cpu_usage)); history_serialize_container(ds, as_type>(obj.obj.account_ram_deltas)); history_serialize_container(ds, as_type>(obj.obj.inline_traces)); return ds; diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index db3a4292797..84e8801cf74 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -76,9 +76,7 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "context_free", "type": "bool" }, { "name": "elapsed", "type": "int64" }, - { "name": "cpu_usage", "type": "uint64" }, { "name": "console", "type": "string" }, - { "name": "total_cpu_usage", "type": "uint64" }, { "name": "account_ram_deltas", "type": "account_delta[]" }, { "name": "inline_traces", "type": "action_trace[]" } ] From 036c4563809dc291af9207936451bef1dcf61d35 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Fri, 12 Oct 2018 15:14:51 -0400 Subject: [PATCH 029/369] state history plugin --- .../state_history_plugin.hpp | 16 ++++++-- .../state_history_serialization.hpp | 3 -- .../state_history_plugin.cpp | 10 ++++- .../state_history_plugin_abi.cpp | 37 ++++++++++++++----- 4 files changed, 47 insertions(+), 19 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index fcb54c362cd..e9737975cde 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -26,10 +26,16 @@ struct table_delta { struct get_status_request_v0 {}; struct get_status_result_v0 { + uint32_t head_block_num = 0; + chain::block_id_type head_block_id = {}; uint32_t last_irreversible_block_num = 0; chain::block_id_type last_irreversible_block_id = {}; - uint32_t state_begin_block_num = 0; - uint32_t state_end_block_num = 0; + uint32_t block_state_begin_block = 0; + uint32_t block_state_end_block = 0; + uint32_t trace_begin_block = 0; + uint32_t trace_end_block = 0; + uint32_t chain_state_begin_block = 0; + uint32_t chain_state_end_block = 0; }; struct get_block_request_v0 { @@ -68,7 +74,9 @@ class state_history_plugin : public plugin { FC_REFLECT(eosio::table_delta, (struct_version)(name)(rows)); FC_REFLECT_EMPTY(eosio::get_status_request_v0); -FC_REFLECT(eosio::get_status_result_v0, - (last_irreversible_block_num)(last_irreversible_block_id)(state_begin_block_num)(state_end_block_num)); +FC_REFLECT( + eosio::get_status_result_v0, + (head_block_num)(head_block_id)(last_irreversible_block_num)(last_irreversible_block_id)(block_state_begin_block)( + block_state_end_block)(trace_begin_block)(trace_end_block)(chain_state_begin_block)(chain_state_end_block)); FC_REFLECT(eosio::get_block_request_v0, (block_num)); FC_REFLECT(eosio::get_block_result_v0, (block_num)(block)(block_state)(traces)(deltas)); diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp index d2d6bb3d089..756aa22a30a 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -253,7 +253,6 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.actor.value)); fc::raw::pack(ds, as_type(obj.obj.permission.value)); return ds; @@ -262,7 +261,6 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, make_history_serial_wrapper(as_type(obj.obj.permission))); fc::raw::pack(ds, as_type(obj.obj.weight)); return ds; @@ -407,7 +405,6 @@ operator<<(datastream& template datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.account.value)); fc::raw::pack(ds, as_type(obj.obj.name.value)); history_serialize_container(ds, as_type>(obj.obj.authorization)); diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index f53afb7e08b..70f1934d3da 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -149,10 +149,16 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain_plug->chain(); get_status_result_v0 result; + result.head_block_num = chain.head_block_num(); + result.head_block_id = chain.head_block_id(); result.last_irreversible_block_num = chain.last_irreversible_block_num(); result.last_irreversible_block_id = chain.last_irreversible_block_id(); - result.state_begin_block_num = plugin->chain_state_log.begin_block(); - result.state_end_block_num = plugin->chain_state_log.end_block(); + result.block_state_begin_block = plugin->block_state_log.begin_block(); + result.block_state_end_block = plugin->block_state_log.end_block(); + result.trace_begin_block = plugin->trace_log.begin_block(); + result.trace_end_block = plugin->trace_log.end_block(); + result.chain_state_begin_block = plugin->chain_state_log.begin_block(); + result.chain_state_end_block = plugin->chain_state_log.end_block(); send(std::move(result)); } diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index 84e8801cf74..5ae8bfd7123 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -6,10 +6,16 @@ extern const char* const state_history_plugin_abi = R"({ }, { "name": "get_status_result_v0", "fields": [ + { "name": "head_block_num", "type": "uint32" }, + { "name": "head_block_id", "type": "checksum256" }, { "name": "last_irreversible_block_num", "type": "uint32" }, { "name": "last_irreversible_block_id", "type": "checksum256" }, - { "name": "state_begin_block_num", "type": "uint32" }, - { "name": "state_end_block_num", "type": "uint32" } + { "name": "block_state_begin_block", "type": "uint32" }, + { "name": "block_state_end_block", "type": "uint32" }, + { "name": "trace_begin_block", "type": "uint32" }, + { "name": "trace_end_block", "type": "uint32" }, + { "name": "chain_state_begin_block", "type": "uint32" }, + { "name": "chain_state_end_block", "type": "uint32" } ] }, { @@ -39,7 +45,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "action_v0", "fields": [ + "name": "action", "fields": [ { "name": "account", "type": "name" }, { "name": "name", "type": "name" }, { "name": "authorization", "type": "permission_level[]" }, @@ -73,7 +79,6 @@ extern const char* const state_history_plugin_abi = R"({ "name": "action_trace_v0", "fields": [ { "name": "receipt", "type": "action_receipt" }, { "name": "act", "type": "action" }, - { "name": "context_free", "type": "bool" }, { "name": "elapsed", "type": "int64" }, { "name": "console", "type": "string" }, @@ -123,7 +128,7 @@ extern const char* const state_history_plugin_abi = R"({ "name": "block_header", "fields": [ { "name": "timestamp", "type": "block_timestamp_type" }, { "name": "producer", "type": "name" }, - { "name": "confirmed; ", "type": "uint16" }, + { "name": "confirmed", "type": "uint16" }, { "name": "previous", "type": "checksum256" }, { "name": "transaction_mroot", "type": "checksum256" }, { "name": "action_mroot", "type": "checksum256" }, @@ -143,6 +148,21 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "block_extensions", "type": "extension[]" } ] }, + { "name": "transaction_header", "fields": [ + { "name": "expiration", "type": "time_point_sec" }, + { "name": "ref_block_num", "type": "uint16" }, + { "name": "ref_block_prefix", "type": "uint32" }, + { "name": "max_net_usage_words", "type": "varuint32" }, + { "name": "max_cpu_usage_ms", "type": "uint8" }, + { "name": "delay_sec", "type": "varuint32" } + ] + }, + { "name": "transaction", "base": "transaction_header", "fields": [ + { "name": "context_free_actions", "type": "action[]" }, + { "name": "actions", "type": "action[]" }, + { "name": "transaction_extensions", "type": "extension[]" } + ] + }, { "name": "account_v0", "fields": [ { "type": "name", "name": "name" }, @@ -291,13 +311,13 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "permission_level_v0", "fields": [ + "name": "permission_level", "fields": [ { "type": "name", "name": "actor" }, { "type": "name", "name": "permission" } ] }, { - "name": "permission_level_weight_v0", "fields": [ + "name": "permission_level_weight", "fields": [ { "type": "permission_level", "name": "permission" }, { "type": "uint16", "name": "weight" } ] @@ -409,7 +429,6 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "request", "types": ["get_status_request_v0", "get_block_request_v0"] }, { "name": "result", "types": ["get_status_result_v0", "get_block_result_v0"] }, - { "name": "action", "types": ["action_v0"] }, { "name": "action_receipt", "types": ["action_receipt_v0"] }, { "name": "action_trace", "types": ["action_trace_v0"] }, { "name": "transaction_trace", "types": ["transaction_trace_v0"] }, @@ -428,8 +447,6 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "global_property", "types": ["global_property_v0"] }, { "name": "generated_transaction", "types": ["generated_transaction_v0"] }, { "name": "key_weight", "types": ["key_weight_v0"] }, - { "name": "permission_level", "types": ["permission_level_v0"] }, - { "name": "permission_level_weight", "types": ["permission_level_weight_v0"] }, { "name": "wait_weight", "types": ["wait_weight_v0"] }, { "name": "shared_authority", "types": ["shared_authority_v0"] }, { "name": "permission", "types": ["permission_v0"] }, From c0f8e4d994de4cceb088250492d13c514c2b848d Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Fri, 12 Oct 2018 17:19:41 -0400 Subject: [PATCH 030/369] state history: deferred and onblock --- .../state_history_serialization.hpp | 4 ++ .../state_history_plugin.cpp | 42 +++++++++++++++---- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp index 756aa22a30a..0f0af68662e 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -456,6 +456,10 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper(obj.obj.net_usage)); fc::raw::pack(ds, as_type(obj.obj.scheduled)); history_serialize_container(ds, as_type>(obj.obj.action_traces)); + // todo + // failed_dtrx_trace + // except + // except_ptr return ds; } diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 70f1934d3da..69783969732 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -3,6 +3,7 @@ * @copyright defined in eos/LICENSE.txt */ +#include #include #include #include @@ -50,7 +51,8 @@ struct state_history_plugin_impl : std::enable_shared_from_this acceptor; - std::map traces; + std::map cached_traces; + transaction_trace_ptr onblock_trace; void get_data(history_log& log, uint32_t block_num, fc::optional& result) { if (block_num < log.begin_block() || block_num >= log.end_block()) @@ -254,9 +256,25 @@ struct state_history_plugin_impl : std::enable_shared_from_thisaction_traces.size() != 1) + return false; + auto& act = p->action_traces[0].act; + if (act.account != eosio::chain::config::system_account_name || act.name != N(onblock) || + act.authorization.size() != 1) + return false; + auto& auth = act.authorization[0]; + return auth.actor == eosio::chain::config::system_account_name && + auth.permission == eosio::chain::config::active_name; + } + void on_applied_transaction(const transaction_trace_ptr& p) { - if (p->receipt) - traces[p->id] = p; + if (p->receipt) { + if (is_onblock(p)) + onblock_trace = p; + else + cached_traces[p->id] = p; + } } void on_accepted_block(const block_state_ptr& block_state) { @@ -271,16 +289,22 @@ struct state_history_plugin_impl : std::enable_shared_from_this traces; - for (auto& p : block_state->trxs) { - auto it = this->traces.find(p->id); - if (it == this->traces.end() || !it->second->receipt) { - ilog("missing trace for transaction {id}", ("id", p->id)); + std::vector traces{onblock_trace}; + for (auto& r : block_state->block->transactions) { + transaction_id_type id; + if (r.trx.contains()) + id = r.trx.get(); + else + id = r.trx.get().id(); + auto it = cached_traces.find(id); + if (it == cached_traces.end() || !it->second->receipt) { + ilog("missing trace for transaction {id}", ("id", id)); continue; } traces.push_back(it->second); } - this->traces.clear(); + cached_traces.clear(); + onblock_trace.reset(); auto traces_bin = fc::raw::pack(make_history_serial_wrapper(traces)); EOS_ASSERT(traces_bin.size() == (uint32_t)traces_bin.size(), plugin_exception, "traces is too big"); From 367d6b2b31d79ad35c289e213b66e9c9d8141d13 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Mon, 15 Oct 2018 16:47:04 -0400 Subject: [PATCH 031/369] state history: failed transactions --- .../state_history_serialization.hpp | 25 ++++++++++++++++--- .../state_history_plugin.cpp | 8 ++++-- .../state_history_plugin_abi.cpp | 5 +++- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp index 0f0af68662e..1fd2f8586c2 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -44,6 +44,14 @@ const T& as_type(const T& x) { return x; } +template +datastream& history_serialize_ptr(datastream& ds, const T* p) { + fc::raw::pack(ds, bool(p)); + if (p) + fc::raw::pack(ds, make_history_serial_wrapper(*p)); + return ds; +} + template datastream& history_serialize_container(datastream& ds, const T& v) { fc::raw::pack(ds, unsigned_int(v.size())); @@ -441,6 +449,12 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper(obj.obj.elapsed.count())); fc::raw::pack(ds, as_type(obj.obj.console)); history_serialize_container(ds, as_type>(obj.obj.account_ram_deltas)); + + fc::optional e; + if (obj.obj.except) + e = obj.obj.except->to_string(); + fc::raw::pack(ds, as_type>(e)); + history_serialize_container(ds, as_type>(obj.obj.inline_traces)); return ds; } @@ -456,10 +470,13 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper(obj.obj.net_usage)); fc::raw::pack(ds, as_type(obj.obj.scheduled)); history_serialize_container(ds, as_type>(obj.obj.action_traces)); - // todo - // failed_dtrx_trace - // except - // except_ptr + + fc::optional e; + if (obj.obj.except) + e = obj.obj.except->to_string(); + fc::raw::pack(ds, as_type>(e)); + + history_serialize_ptr(ds, as_type(obj.obj.failed_dtrx_trace.get())); return ds; } diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 69783969732..2746d80e330 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -272,6 +272,8 @@ struct state_history_plugin_impl : std::enable_shared_from_thisreceipt) { if (is_onblock(p)) onblock_trace = p; + else if (p->failed_dtrx_trace) + cached_traces[p->failed_dtrx_trace->id] = p; else cached_traces[p->id] = p; } @@ -289,7 +291,9 @@ struct state_history_plugin_impl : std::enable_shared_from_this traces{onblock_trace}; + std::vector traces; + if (onblock_trace) + traces.push_back(onblock_trace); for (auto& r : block_state->block->transactions) { transaction_id_type id; if (r.trx.contains()) @@ -298,7 +302,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this().id(); auto it = cached_traces.find(id); if (it == cached_traces.end() || !it->second->receipt) { - ilog("missing trace for transaction {id}", ("id", id)); + ilog("missing trace for transaction ${id}", ("id", id)); continue; } traces.push_back(it->second); diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index 5ae8bfd7123..29200838fc3 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -83,6 +83,7 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "elapsed", "type": "int64" }, { "name": "console", "type": "string" }, { "name": "account_ram_deltas", "type": "account_delta[]" }, + { "name": "except", "type": "string?" }, { "name": "inline_traces", "type": "action_trace[]" } ] }, @@ -95,7 +96,9 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "elapsed", "type": "int64" }, { "name": "net_usage", "type": "uint64" }, { "name": "scheduled", "type": "bool" }, - { "name": "action_traces", "type": "action_trace[]" } + { "name": "action_traces", "type": "action_trace[]" }, + { "name": "except", "type": "string?" }, + { "name": "failed_dtrx_trace", "type": "transaction_trace?" } ] }, { From 35cc2436e3e218c29e08193c9c4e26507a7138b6 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Mon, 15 Oct 2018 18:10:33 -0400 Subject: [PATCH 032/369] state history --- .../state_history_serialization.hpp | 7 ------- .../state_history_plugin_abi.cpp | 18 ++++++------------ 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp index 1fd2f8586c2..692618206a3 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -225,8 +225,6 @@ template datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); - fc::raw::pack(ds, as_type>(obj.obj.proposed_schedule_block_num)); fc::raw::pack(ds, make_history_serial_wrapper( as_type(obj.obj.proposed_schedule))); @@ -239,7 +237,6 @@ template datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, as_type(obj.obj.trx_id)); fc::raw::pack(ds, as_type(obj.obj.sender.value)); fc::raw::pack(ds, as_type<__uint128_t>(obj.obj.sender_id)); @@ -253,7 +250,6 @@ datastream& operator<<(datastream& template datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.key)); fc::raw::pack(ds, as_type(obj.obj.weight)); return ds; @@ -276,7 +272,6 @@ datastream& operator<<(datastream& template datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.wait_sec)); fc::raw::pack(ds, as_type(obj.obj.weight)); return ds; @@ -359,7 +354,6 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, make_history_serial_wrapper( as_type(obj.obj.average_block_net_usage))); fc::raw::pack(ds, make_history_serial_wrapper( @@ -401,7 +395,6 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.obj.id._id)); fc::raw::pack(ds, make_history_serial_wrapper(as_type( obj.obj.cpu_limit_parameters))); fc::raw::pack(ds, make_history_serial_wrapper(as_type( diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index 29200838fc3..8a5f0c2fb25 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -288,7 +288,6 @@ extern const char* const state_history_plugin_abi = R"({ }, { "name": "global_property_v0", "fields": [ - { "type": "uint64", "name": "id" }, { "type": "uint32?", "name": "proposed_schedule_block_num" }, { "type": "producer_schedule", "name": "proposed_schedule" }, { "type": "chain_config", "name": "configuration" } @@ -296,7 +295,6 @@ extern const char* const state_history_plugin_abi = R"({ }, { "name": "generated_transaction_v0", "fields": [ - { "type": "uint64", "name": "id" }, { "type": "checksum256", "name": "trx_id" }, { "type": "name", "name": "sender" }, { "type": "uint128", "name": "sender_id" }, @@ -308,7 +306,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "key_weight_v0", "fields": [ + "name": "key_weight", "fields": [ { "type": "public_key", "name": "key" }, { "type": "uint16", "name": "weight" } ] @@ -326,7 +324,7 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "wait_weight_v0", "fields": [ + "name": "wait_weight", "fields": [ { "type": "uint32", "name": "wait_sec" }, { "type": "uint16", "name": "weight" } ] @@ -387,7 +385,6 @@ extern const char* const state_history_plugin_abi = R"({ }, { "name": "resource_limits_state_v0", "fields": [ - { "type": "uint64", "name": "id" }, { "type": "usage_accumulator", "name": "average_block_net_usage" }, { "type": "usage_accumulator", "name": "average_block_cpu_usage" }, { "type": "uint64", "name": "pending_net_usage" }, @@ -417,7 +414,6 @@ extern const char* const state_history_plugin_abi = R"({ }, { "name": "resource_limits_config_v0", "fields": [ - { "type": "uint64", "name": "id" }, { "type": "elastic_limit_parameters", "name": "cpu_limit_parameters" }, { "type": "elastic_limit_parameters", "name": "net_limit_parameters" }, { "type": "uint32", "name": "account_cpu_usage_average_window" }, @@ -449,8 +445,6 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "chain_config", "types": ["chain_config_v0"] }, { "name": "global_property", "types": ["global_property_v0"] }, { "name": "generated_transaction", "types": ["generated_transaction_v0"] }, - { "name": "key_weight", "types": ["key_weight_v0"] }, - { "name": "wait_weight", "types": ["wait_weight_v0"] }, { "name": "shared_authority", "types": ["shared_authority_v0"] }, { "name": "permission", "types": ["permission_v0"] }, { "name": "permission_link", "types": ["permission_link_v0"] }, @@ -471,13 +465,13 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "contract_index256", "type": "contract_index256", "key_names": ["code", "scope", "table", "primary_key"] }, { "name": "contract_index_double", "type": "contract_index_double", "key_names": ["code", "scope", "table", "primary_key"] }, { "name": "contract_index_long_double", "type": "contract_index_long_double", "key_names": ["code", "scope", "table", "primary_key"] }, - { "name": "global_property", "type": "global_property", "key_names": ["id"] }, - { "name": "generated_transaction", "type": "generated_transaction", "key_names": ["id"] }, + { "name": "global_property", "type": "global_property", "key_names": [] }, + { "name": "generated_transaction", "type": "generated_transaction", "key_names": ["trx_id"] }, { "name": "permission", "type": "permission", "key_names": ["id"] }, { "name": "permission_link", "type": "permission_link", "key_names": ["id"] }, { "name": "resource_limits", "type": "resource_limits", "key_names": ["id"] }, { "name": "resource_usage", "type": "resource_usage", "key_names": ["id"] }, - { "name": "resource_limits_state", "type": "resource_limits_state", "key_names": ["id"] }, - { "name": "resource_limits_config", "type": "resource_limits_config", "key_names": ["id"] } + { "name": "resource_limits_state", "type": "resource_limits_state", "key_names": [] }, + { "name": "resource_limits_config", "type": "resource_limits_config", "key_names": [] } ] })"; From 29e266f23eb3a88b1d05e6e331eec481bd0608d4 Mon Sep 17 00:00:00 2001 From: Zane Reynolds Date: Tue, 16 Oct 2018 10:26:02 -0400 Subject: [PATCH 033/369] added the address sanitizer (without the leak sanitizer) --- .buildkite/sanitizers.yml | 65 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/.buildkite/sanitizers.yml b/.buildkite/sanitizers.yml index db479d25697..1c0b28660a9 100644 --- a/.buildkite/sanitizers.yml +++ b/.buildkite/sanitizers.yml @@ -41,6 +41,49 @@ steps: - UBSAN_OPTIONS=print_stacktrace=1 timeout: 60 + - command: | + echo "--- :hammer: Building with Address Sanitizer" && \ + /usr/bin/cmake -GNinja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_COMPILER=clang++-4.0 \ + -DCMAKE_C_COMPILER=clang-4.0 \ + -DBOOST_ROOT="${BOOST_ROOT}" \ + -DWASM_ROOT="${WASM_ROOT}" \ + -DOPENSSL_ROOT_DIR="${OPENSSL_ROOT_DIR}" \ + -DBUILD_MONGO_DB_PLUGIN=true \ + -DENABLE_COVERAGE_TESTING=true \ + -DBUILD_DOXYGEN=false \ + -DCMAKE_CXX_FLAGS="-fsanitize=address -fsanitize-recover=all -O1 -g -fno-omit-frame-pointer" \ + -DCMAKE_C_FLAGS="-fsanitize=address -fsanitize-recover=all -O1 -g -fno-omit-frame-pointer" \ + -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address -fsanitize-recover=all -rtlib=compiler-rt -lgcc_s" \ + -DCMAKE_MODULE_LINKER_FLAGS="-fsanitize=address -fsanitize-recover=all -rtlib=compiler-rt -lgcc_s" + echo "--- :shinto_shrine: Running ninja" && \ + /usr/bin/ninja | tee ninja.log && \ + echo "--- :compression: Compressing build directory" && \ + tar -pczf build.tar.gz * + echo "--- :beers: Done" + label: ":_: Address Sanitizer" + agents: + - "role=automation-builder-large" + artifact_paths: + - "build.tar.gz" + - "ninja.log" + plugins: + docker#v1.4.0: + image: "eosio/ci:ubuntu18" + command: ["--privileged"] + workdir: /data/job + mounts: + - /etc/buildkite-agent/config:/config + environment: + - BOOST_ROOT=/root/opt/boost + - OPENSSL_ROOT_DIR=/usr/include/openssl + - WASM_ROOT=/root/opt/wasm + - PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/opt/wasm/bin + - CI=true + - ASAN_OPTIONS=fast_unwind_on_malloc=0:halt_on_error=0:detect_odr_violation=0:detect_leaks=0:symbolize=1:verbosity=1 + timeout: 60 + - wait - command: | @@ -52,6 +95,28 @@ steps: echo "+++ :microscope: Running tests" && \ ctest -j8 -LE _tests -V -O sanitizer.log label: ":_: Undefined Sanitizer Tests" + agents: + - "role=automation-builder-large" + artifact_paths: + - "mongod.log" + - "sanitizer.log" + plugins: + docker#v1.4.0: + image: "eosio/ci:ubuntu18" + workdir: /data/job + mounts: + - /etc/buildkite-agent/config:/config + timeout: 120 + + - command: | + echo "--- :arrow_down: Downloading build directory" && \ + buildkite-agent artifact download "build.tar.gz" . --step ":_: Address Sanitizer" && \ + tar -zxf build.tar.gz --no-same-owner && \ + echo "--- :m: Starting MongoDB" && \ + $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ + echo "+++ :microscope: Running tests" && \ + ctest -j8 -LE _tests -V -O sanitizer.log + label: ":_: Address Sanitizer Tests" agents: - "role=automation-builder-large" artifact_paths: From 66d2618619b5801d84394225b13a05cb5456f370 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Tue, 16 Oct 2018 17:27:50 -0400 Subject: [PATCH 034/369] state history plugin: options --- .../state_history_plugin/history_log.hpp | 27 ++----- .../state_history_plugin.hpp | 8 ++- .../state_history_plugin.cpp | 70 ++++++++++++------- .../state_history_plugin_abi.cpp | 6 +- 4 files changed, 62 insertions(+), 49 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp index cb1b5ceb6d8..a2b1e44555d 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/history_log.hpp @@ -55,8 +55,7 @@ struct history_summary { class history_log { private: - const char* const name = ""; - bool _is_open = false; + const char* const name = ""; std::string log_filename; std::string index_filename; std::fstream log; @@ -66,30 +65,19 @@ class history_log { chain::block_id_type last_block_id; public: - history_log(const char* const name) - : name(name) {} - - bool is_open() const { return _is_open; } - uint32_t begin_block() const { return _begin_block; } - uint32_t end_block() const { return _end_block; } - - void open(std::string log_filename, std::string index_filename) { - this->log_filename = std::move(log_filename); - this->index_filename = std::move(index_filename); + history_log(const char* const name, std::string log_filename, std::string index_filename) + : name(name) + , log_filename(std::move(log_filename)) + , index_filename(std::move(index_filename)) { open_log(); open_index(); - _is_open = true; } - void close() { - log.close(); - index.close(); - _is_open = false; - } + uint32_t begin_block() const { return _begin_block; } + uint32_t end_block() const { return _end_block; } template void write_entry(history_log_header header, const chain::block_id_type& prev_id, F write_payload) { - EOS_ASSERT(_is_open, chain::plugin_exception, "writing to closed ${name}.log", ("name", name)); EOS_ASSERT(_begin_block == _end_block || header.block_num <= _end_block, chain::plugin_exception, "missed a block in ${name}.log", ("name", name)); @@ -127,7 +115,6 @@ class history_log { // returns stream positioned at payload std::fstream& get_entry(uint32_t block_num, history_log_header& header) { - EOS_ASSERT(_is_open, chain::plugin_exception, "reading from closed ${name}.log", ("name", name)); EOS_ASSERT(block_num >= _begin_block && block_num < _end_block, chain::plugin_exception, "read non-existing block in ${name}.log", ("name", name)); log.seekg(get_pos(block_num)); diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index e9737975cde..4d9a3663ca8 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -39,7 +39,11 @@ struct get_status_result_v0 { }; struct get_block_request_v0 { - uint32_t block_num = 0; + uint32_t block_num = 0; + bool fetch_block = false; + bool fetch_block_state = false; + bool fetch_traces = false; + bool fetch_deltas = false; }; struct get_block_result_v0 { @@ -78,5 +82,5 @@ FC_REFLECT( eosio::get_status_result_v0, (head_block_num)(head_block_id)(last_irreversible_block_num)(last_irreversible_block_id)(block_state_begin_block)( block_state_end_block)(trace_begin_block)(trace_end_block)(chain_state_begin_block)(chain_state_end_block)); -FC_REFLECT(eosio::get_block_request_v0, (block_num)); +FC_REFLECT(eosio::get_block_request_v0, (block_num)(fetch_block)(fetch_block_state)(fetch_traces)(fetch_deltas)); FC_REFLECT(eosio::get_block_result_v0, (block_num)(block)(block_state)(traces)(deltas)); diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 2746d80e330..0657659745c 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -42,9 +42,9 @@ auto catch_and_log(F f) { struct state_history_plugin_impl : std::enable_shared_from_this { chain_plugin* chain_plug = nullptr; - history_log block_state_log{"block_state_history"}; - history_log trace_log{"trace_history"}; - history_log chain_state_log{"chain_state_history"}; + fc::optional block_state_log; + fc::optional trace_log; + fc::optional chain_state_log; bool stopping = false; fc::optional applied_transaction_connection; fc::optional accepted_block_connection; @@ -155,23 +155,32 @@ struct state_history_plugin_impl : std::enable_shared_from_thisblock_state_log.begin_block(); - result.block_state_end_block = plugin->block_state_log.end_block(); - result.trace_begin_block = plugin->trace_log.begin_block(); - result.trace_end_block = plugin->trace_log.end_block(); - result.chain_state_begin_block = plugin->chain_state_log.begin_block(); - result.chain_state_end_block = plugin->chain_state_log.end_block(); + if (plugin->block_state_log) { + result.block_state_begin_block = plugin->block_state_log->begin_block(); + result.block_state_end_block = plugin->block_state_log->end_block(); + } + if (plugin->trace_log) { + result.trace_begin_block = plugin->trace_log->begin_block(); + result.trace_end_block = plugin->trace_log->end_block(); + } + if (plugin->chain_state_log) { + result.chain_state_begin_block = plugin->chain_state_log->begin_block(); + result.chain_state_end_block = plugin->chain_state_log->end_block(); + } send(std::move(result)); } void operator()(get_block_request_v0& req) { // ilog("${b} get_block_request_v0", ("b", req.block_num)); get_block_result_v0 result{req.block_num}; - // todo: client select which datasets to receive - plugin->get_block(req.block_num, result.block); - plugin->get_data(plugin->block_state_log, req.block_num, result.block_state); - plugin->get_data(plugin->trace_log, req.block_num, result.traces); - plugin->get_data(plugin->chain_state_log, req.block_num, result.deltas); + if (req.fetch_block) + plugin->get_block(req.block_num, result.block); + if (req.fetch_block_state && plugin->block_state_log) + plugin->get_data(*plugin->block_state_log, req.block_num, result.block_state); + if (req.fetch_traces && plugin->trace_log) + plugin->get_data(*plugin->trace_log, req.block_num, result.traces); + if (req.fetch_deltas && plugin->chain_state_log) + plugin->get_data(*plugin->chain_state_log, req.block_num, result.deltas); send(std::move(result)); } @@ -280,7 +289,6 @@ struct state_history_plugin_impl : std::enable_shared_from_this traces; if (onblock_trace) traces.push_back(onblock_trace); @@ -316,7 +326,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisblock->block_num(), .block_id = block_state->block->id(), .payload_size = sizeof(uint32_t) + traces_bin.size()}; - trace_log.write_entry(header, block_state->block->previous, [&](auto& stream) { + trace_log->write_entry(header, block_state->block->previous, [&](auto& stream) { uint32_t s = (uint32_t)traces_bin.size(); stream.write((char*)&s, sizeof(s)); if (!traces_bin.empty()) @@ -325,7 +335,9 @@ struct state_history_plugin_impl : std::enable_shared_from_thisbegin_block() == chain_state_log->end_block(); if (fresh) ilog("Placing initial state in block ${n}", ("n", block_state->block->block_num())); @@ -409,7 +421,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisblock->block_num(), .block_id = block_state->block->id(), .payload_size = sizeof(uint32_t) + deltas_bin.size()}; - chain_state_log.write_entry(header, block_state->block->previous, [&](auto& stream) { + chain_state_log->write_entry(header, block_state->block->previous, [&](auto& stream) { uint32_t s = (uint32_t)deltas_bin.size(); stream.write((char*)&s, sizeof(s)); if (!deltas_bin.empty()) @@ -427,7 +439,10 @@ void state_history_plugin::set_program_options(options_description& cli, options auto options = cfg.add_options(); options("state-history-dir", bpo::value()->default_value("state-history"), "the location of the state-history directory (absolute path or relative to application data dir)"); - options("delete-state-history", bpo::bool_switch()->default_value(false), "clear state history database"); + options("delete-state-history", bpo::bool_switch()->default_value(false), "clear state history files"); + options("block-state-history", bpo::bool_switch()->default_value(false), "enable block state history"); + options("trace-history", bpo::bool_switch()->default_value(false), "enable trace history"); + options("chain-state-history", bpo::bool_switch()->default_value(false), "enable chain state history"); options("state-history-endpoint", bpo::value()->default_value("0.0.0.0:8080"), "the endpoint upon which to listen for incoming connections"); } @@ -462,14 +477,17 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { ilog("Deleting state history"); boost::filesystem::remove_all(state_history_dir); } - boost::filesystem::create_directories(state_history_dir); - my->block_state_log.open((state_history_dir / "block_state_history.log").string(), - (state_history_dir / "block_state_history.index").string()); - my->trace_log.open((state_history_dir / "trace_history.log").string(), - (state_history_dir / "trace_history.index").string()); - my->chain_state_log.open((state_history_dir / "chain_state_history.log").string(), - (state_history_dir / "chain_state_history.index").string()); + + if (options.at("block-state-history").as()) + my->block_state_log.emplace("block_state_history", (state_history_dir / "block_state_history.log").string(), + (state_history_dir / "block_state_history.index").string()); + if (options.at("trace-history").as()) + my->trace_log.emplace("trace_history", (state_history_dir / "trace_history.log").string(), + (state_history_dir / "trace_history.index").string()); + if (options.at("chain-state-history").as()) + my->chain_state_log.emplace("chain_state_history", (state_history_dir / "chain_state_history.log").string(), + (state_history_dir / "chain_state_history.index").string()); } FC_LOG_AND_RETHROW() } // state_history_plugin::plugin_initialize diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index 8a5f0c2fb25..024208082b5 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -20,7 +20,11 @@ extern const char* const state_history_plugin_abi = R"({ }, { "name": "get_block_request_v0", "fields": [ - { "name": "block_num", "type": "uint32" } + { "name": "block_num", "type": "uint32" }, + { "name": "fetch_block", "type": "bool" }, + { "name": "fetch_block_state", "type": "bool" }, + { "name": "fetch_traces", "type": "bool" }, + { "name": "fetch_deltas", "type": "bool" } ] }, { From 560cf6df89a47e6a10f5f0983a08ec8f4b3464e9 Mon Sep 17 00:00:00 2001 From: liuyujun Date: Wed, 17 Oct 2018 15:23:06 +0800 Subject: [PATCH 035/369] cleos parse last_bid_time support time_point --- programs/cleos/main.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/programs/cleos/main.cpp b/programs/cleos/main.cpp index 30334246fd0..43ad89f91a3 100644 --- a/programs/cleos/main.cpp +++ b/programs/cleos/main.cpp @@ -1301,12 +1301,16 @@ struct bidname_info_subcommand { return; } for ( auto& row : result.rows ) { - fc::time_point time(fc::microseconds(row["last_bid_time"].as_uint64())); + string time = row["last_bid_time"].as_string(); + try { + time = (string)fc::time_point(fc::microseconds(to_uint64(time))); + } catch (fc::parse_error_exception&) { + } int64_t bid = row["high_bid"].as_int64(); std::cout << std::left << std::setw(18) << "bidname:" << std::right << std::setw(24) << row["newname"].as_string() << "\n" << std::left << std::setw(18) << "highest bidder:" << std::right << std::setw(24) << row["high_bidder"].as_string() << "\n" << std::left << std::setw(18) << "highest bid:" << std::right << std::setw(24) << (bid > 0 ? bid : -bid) << "\n" - << std::left << std::setw(18) << "last bid time:" << std::right << std::setw(24) << ((std::string)time).c_str() << std::endl; + << std::left << std::setw(18) << "last bid time:" << std::right << std::setw(24) << time << std::endl; if (bid < 0) std::cout << "This auction has already closed" << std::endl; } }); From 26cd10e3e126209e32fb49e5c167074cf76262b5 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 17 Oct 2018 09:42:07 -0400 Subject: [PATCH 036/369] Update log message --- plugins/net_plugin/net_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 9ee380b666f..7015ad2afc7 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2379,7 +2379,7 @@ namespace eosio { request_message req; bool send_req = false; if (msg.known_trx.mode != none) { - fc_dlog(logger,"this is a ${m} notice with ${n} blocks", ("m",modes_str(msg.known_trx.mode))("n",msg.known_trx.pending)); + fc_dlog(logger,"this is a ${m} notice with ${n} transactions", ("m",modes_str(msg.known_trx.mode))("n",msg.known_trx.pending)); } switch (msg.known_trx.mode) { case none: From 369cf22123cc307358c81215e523d47f732e1ae5 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 17 Oct 2018 09:42:56 -0400 Subject: [PATCH 037/369] catch_up skipped --- plugins/net_plugin/net_plugin.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 7015ad2afc7..627ea760874 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2413,9 +2413,6 @@ namespace eosio { } switch (msg.known_blocks.mode) { case none : { - if (msg.known_trx.mode != normal) { - return; - } break; } case last_irr_catch_up: From db98061ba63272eb155643e272c409e8deb2f615 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 17 Oct 2018 15:21:48 -0500 Subject: [PATCH 038/369] Removed unused files, dependencies and includes. GH #6041 --- contracts/CMakeLists.txt | 4 - contracts/bancor/CMakeLists.txt | 8 - contracts/bancor/bancor.cpp | 18 -- contracts/bancor/bancor.hpp | 22 -- contracts/bancor/converter.hpp | 166 ------------ contracts/bancor/eosio.system.abi | 37 --- contracts/eosiolib/currency.hpp | 254 ------------------ contracts/infinite/CMakeLists.txt | 5 - contracts/infinite/infinite.cpp | 16 -- contracts/simple.token/CMakeLists.txt | 9 - contracts/simple.token/simple.token.abi | 2 - contracts/simple.token/simple.token.cpp | 49 ---- contracts/social/CMakeLists.txt | 5 - contracts/social/social.cpp | 107 -------- tests/CMakeLists.txt | 1 - tests/chain_plugin_tests.cpp | 9 - tests/chain_tests/proof_tests.cpp | 340 ------------------------ tests/get_table_tests.cpp | 6 - tests/p2p_tests/fuzz_test_generic.sh | 11 - tests/p2p_tests/pump/run_test.pl | 285 -------------------- unittests/CMakeLists.txt | 2 +- unittests/eosio.sudo_tests.cpp | 2 - unittests/multisig_tests.cpp | 3 - unittests/ram_tests.cpp | 3 - unittests/tic_tac_toe_tests.cpp | 3 - unittests/wasm_tests.cpp | 70 ----- 26 files changed, 1 insertion(+), 1436 deletions(-) delete mode 100644 contracts/bancor/CMakeLists.txt delete mode 100644 contracts/bancor/bancor.cpp delete mode 100644 contracts/bancor/bancor.hpp delete mode 100644 contracts/bancor/converter.hpp delete mode 100644 contracts/bancor/eosio.system.abi delete mode 100644 contracts/eosiolib/currency.hpp delete mode 100644 contracts/infinite/CMakeLists.txt delete mode 100644 contracts/infinite/infinite.cpp delete mode 100644 contracts/simple.token/CMakeLists.txt delete mode 100644 contracts/simple.token/simple.token.abi delete mode 100644 contracts/simple.token/simple.token.cpp delete mode 100644 contracts/social/CMakeLists.txt delete mode 100644 contracts/social/social.cpp delete mode 100644 tests/chain_tests/proof_tests.cpp delete mode 100644 tests/p2p_tests/fuzz_test_generic.sh delete mode 100755 tests/p2p_tests/pump/run_test.pl diff --git a/contracts/CMakeLists.txt b/contracts/CMakeLists.txt index bd84d155e85..443330c886a 100644 --- a/contracts/CMakeLists.txt +++ b/contracts/CMakeLists.txt @@ -7,7 +7,6 @@ set(STANDARD_INCLUDE_FOLDERS ${CMAKE_SOURCE_DIR}/contracts ${CMAKE_BINARY_DIR}/c add_subdirectory(eosiolib) add_subdirectory(musl) add_subdirectory(libc++) -add_subdirectory(simple.token) add_subdirectory(eosio.token) add_subdirectory(eosio.msig) add_subdirectory(eosio.sudo) @@ -18,17 +17,14 @@ add_subdirectory(identity) add_subdirectory(stltest) add_subdirectory(test.inline) -#add_subdirectory(bancor) add_subdirectory(hello) add_subdirectory(asserter) -add_subdirectory(infinite) add_subdirectory(proxy) add_subdirectory(test_api) add_subdirectory(test_api_mem) add_subdirectory(test_api_db) add_subdirectory(test_api_multi_index) add_subdirectory(test_ram_limit) -#add_subdirectory(social) add_subdirectory(eosio.bios) add_subdirectory(noop) add_subdirectory(tic_tac_toe) diff --git a/contracts/bancor/CMakeLists.txt b/contracts/bancor/CMakeLists.txt deleted file mode 100644 index b845361e8f0..00000000000 --- a/contracts/bancor/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -file(GLOB ABI_FILES "*.abi") -configure_file("${ABI_FILES}" "${CMAKE_CURRENT_BINARY_DIR}" COPYONLY) - -add_wast_executable(TARGET bancor - INCLUDE_FOLDERS "${STANDARD_INCLUDE_FOLDERS}" - LIBRARIES libc libc++ eosiolib - DESTINATION_FOLDER ${CMAKE_CURRENT_BINARY_DIR} -) diff --git a/contracts/bancor/bancor.cpp b/contracts/bancor/bancor.cpp deleted file mode 100644 index 15bc9e802c8..00000000000 --- a/contracts/bancor/bancor.cpp +++ /dev/null @@ -1,18 +0,0 @@ -/** - * @file - * @copyright defined in eos/LICENSE.txt - */ - -#include - -namespace bancor { - -extern "C" { - - /// The apply method implements the dispatch of events to this contract - void apply( uint64_t r, uint64_t c, uint64_t a ) { - bancor::example_converter::apply( c, a ); - } -} - -} diff --git a/contracts/bancor/bancor.hpp b/contracts/bancor/bancor.hpp deleted file mode 100644 index b6030908b81..00000000000 --- a/contracts/bancor/bancor.hpp +++ /dev/null @@ -1,22 +0,0 @@ -/** - * @file - * @copyright defined in eos/LICENSE.txt - */ -#pragma once - -#include -#include -#include -#include - -#include -#include - -namespace bancor { - typedef eosio::generic_currency< eosio::token > other_currency; - typedef eosio::generic_currency< eosio::token > relay_currency; - typedef eosio::generic_currency< eosio::token > cur_currency; - - typedef converter example_converter; -} /// bancor - diff --git a/contracts/bancor/converter.hpp b/contracts/bancor/converter.hpp deleted file mode 100644 index 774fc910e32..00000000000 --- a/contracts/bancor/converter.hpp +++ /dev/null @@ -1,166 +0,0 @@ -#pragma once -namespace bancor { - template - class converter_contract { - public: - typedef ConverterCurrency converter_currency; - typedef FirstCurrency first_currency; - typedef SecondCurrency second_currency; - - - static const account_name converter_account = converter_currency::code; - - struct converter_state { - typename converter_currency::token_type supply; /// total supply held by all users - typename converter_currency::token_type balance; /// supply held by converter in its own balance - }; - - struct converter_args { - eosio::account_name to_currency_account; - eosio::symbol_name to_currency_symbol; - uint64_t min_return_currency; - }; - - - template - struct connector { - typedef CurrencyType currency_type; - typedef typename converter_currency::token_type converter_token_type; - typedef typename currency_type::token_type in_token_type; - - converter_token_type convert_to_converter( in_token_type in, converter_state& state ) { - in_token_type balance = currency_type::get_balance( converter_account ); - - /// balance already changed when transfer executed, get pre-transfer balance - in_token_type previous_balance = balance - in; - - auto init_price = (previous_balance * Base) / (Weight * state.supply); - auto init_out = init_price * in; - - auto out_price = (balance*Base) / (Weight * (state.supply+init_out) ); - auto final_out = out_price * in; - - state.balance += final_out; - state.supply += final_out; - - return final_out; - } - - - in_token_type convert_from_converter( converter_token_type converter_in, converter_state& state ) { - in_token_type to_balance = CurrencyType::get_balance( converter_account ); - - auto init_price = (to_balance * Base) / (Weight * state.supply); - in_token_type init_out = init_price * converter_in; - - state.supply -= converter_in; - state.balance -= converter_in; - - auto out_price = ((to_balance-init_out) * Base) / ( Weight * (state.supply) ); - - return out_price * converter_in; - } - - }; - - - - /** - * This is called when we receive RELAY tokens from user and wish to - * convert to one of the connector currencies. - */ - static void on_convert( const typename converter_currency::transfer& trans, - const converter_args& args, - converter_state& state ) { - - if( args.to_currency_type == first_currency ) { - auto output = first_connector::convert_from_converter( trans.quantity, state ); - save_and_send( trans.from, state, output, args.min_return ); - } - else if( args.to_currency_type == second_currency ) { - auto output = second_connector::convert_from_converter( trans.quantity, state ); - save_and_send( trans.from, state, output, args.min_return ); - } - else { - eosio_assert( false, "invalid to currency" ); - } - } - - - /** - * This is called when the converter receives one of the connector currencies and it - * will send either converter tokens or a different connector currency in response. - */ - template - static void on_convert( const typename ConnectorType::currency_type::transfer& trans, - const converter_args& args, - converter_state& state ) - { - /// convert to converter - auto converter_out = ConnectorType::convert_to_converter( trans.quantity, state ); - - if( args.to_currency_type == converter_currency ) - { - save_and_send( trans.from, state, converter_out, args.min_return ); - } - else - { - auto output = ConnectorType::convert_from_converter( converter_out, state ); - save_and_send( trans.from, state, output, args.min_return ); - } - } - - - /** - * This method factors out the boiler plate for parsing args and loading the - * initial state before dispatching to the proper on_convert case - */ - template - static void start_convert( const typename CurrencyType::transfer_memo& trans ) { - auto args = unpack( trans.memo ); - eosio_assert( args.to_currency_type != trans.quantity.token_type(), "cannot convert to self" ); - - auto state = read_converter_state(); - on_convert( trans, args, state ); - } - - - /** - * converter_account first needs to call the currency handler to perform - * user-to-user transfers of the converter token, then if a transfer is sending - * the token back to the converter contract, it should convert like everything else. - * - * This method should be called from apply( code, action ) for each of the - * transfer types that we support (for each currency) - */ - static void on( const typename converter_currency::transfer_memo& trans ) { - converter_currency::on( trans ); - if( trans.to == converter_account ) { - start_convert( trans ); - } - } - - /** - * All other currencies simply call start_convert if to == converter_account - */ - template - static void on( const typename Currency::transfer_memo& trans ) { - if( trans.to == converter_account ) { - start_convert( trans ); - } else { - eosio_assert( trans.from == converter_account, - "received unexpected notification of transfer" ); - } - } - - static void apply( account_name code, action_name action ) { - if( !dispatch( converter_contract, - converter_currency::transfer, - converter_currency::issue, - first_currency::transfer, - second_currency::transfer ) { - eosio_assert( false, "received unexpected action" ); - } - } - }; /// converter_contract -} /// namespace bancor diff --git a/contracts/bancor/eosio.system.abi b/contracts/bancor/eosio.system.abi deleted file mode 100644 index c8d8e4b16e9..00000000000 --- a/contracts/bancor/eosio.system.abi +++ /dev/null @@ -1,37 +0,0 @@ -{ - "types": [{ - "new_type_name": "account_name", - "type": "name" - } - ], - "structs": [{ - "name": "transfer", - "base": "", - "fields": [ - {"name":"from", "type":"account_name"}, - {"name":"to", "type":"account_name"}, - {"name":"quantity", "type":"uint64"} - ] - },{ - "name": "account", - "base": "", - "fields": [ - {"name":"key", "type":"name"}, - {"name":"balance", "type":"uint64"} - ] - } - ], - "actions": [{ - "name": "transfer", - "type": "transfer" - } - ], - "tables": [{ - "name": "account", - "type": "account", - "index_type": "i64", - "key_names" : ["key"], - "key_types" : ["name"] - } - ] -} \ No newline at end of file diff --git a/contracts/eosiolib/currency.hpp b/contracts/eosiolib/currency.hpp deleted file mode 100644 index 1328c493876..00000000000 --- a/contracts/eosiolib/currency.hpp +++ /dev/null @@ -1,254 +0,0 @@ -#pragma once -#include -#include -#include - -namespace eosio { - using std::string; - using std::array; - - /** - * This contract enables the creation, issuance, and transfering of many different tokens. - * @deprecated This class is deprecated in favor of eosio.token in Dawn 3.0 - */ - class currency { - public: - currency( account_name contract ) - :_contract(contract) - { } - - struct create { - account_name issuer; - asset maximum_supply; - // array issuer_agreement_hash; - uint8_t issuer_can_freeze = true; - uint8_t issuer_can_recall = true; - uint8_t issuer_can_whitelist = true; - - /*(issuer_agreement_hash)*/ - EOSLIB_SERIALIZE( create, (issuer)(maximum_supply)(issuer_can_freeze)(issuer_can_recall)(issuer_can_whitelist) ) - }; - - struct transfer - { - account_name from; - account_name to; - asset quantity; - string memo; - - EOSLIB_SERIALIZE( transfer, (from)(to)(quantity)(memo) ) - }; - - struct issue { - account_name to; - asset quantity; - string memo; - - EOSLIB_SERIALIZE( issue, (to)(quantity)(memo) ) - }; - - struct fee_schedule { - uint64_t primary_key()const { return 0; } - - array fee_per_length; - EOSLIB_SERIALIZE( fee_schedule, (fee_per_length) ) - }; - - struct account { - asset balance; - bool frozen = false; - bool whitelist = true; - - uint64_t primary_key()const { return balance.symbol.name(); } - - EOSLIB_SERIALIZE( account, (balance)(frozen)(whitelist) ) - }; - - struct currency_stats { - asset supply; - asset max_supply; - account_name issuer; - bool can_freeze = true; - bool can_recall = true; - bool can_whitelist = true; - bool is_frozen = false; - bool enforce_whitelist = false; - - uint64_t primary_key()const { return supply.symbol.name(); } - - EOSLIB_SERIALIZE( currency_stats, (supply)(max_supply)(issuer)(can_freeze)(can_recall)(can_whitelist)(is_frozen)(enforce_whitelist) ) - }; - - typedef eosio::multi_index accounts; - typedef eosio::multi_index stats; - - - asset get_balance( account_name owner, symbol_name symbol )const { - accounts t( _contract, owner ); - return t.get(symbol).balance; - } - - asset get_supply( symbol_name symbol )const { - accounts t( _contract, symbol ); - return t.get(symbol).balance; - } - - static void inline_transfer( account_name from, account_name to, extended_asset amount, string memo = string(), permission_name perm = N(active) ) { - action act( permission_level( from, perm ), amount.contract, N(transfer), transfer{from,to,amount,memo} ); - act.send(); - } - - void inline_transfer( account_name from, account_name to, asset amount, string memo = string(), permission_name perm = N(active) ) { - action act( permission_level( from, perm ), _contract, N(transfer), transfer{from,to,amount,memo} ); - act.send(); - } - - - bool apply( account_name contract, action_name act ) { - if( contract != _contract ) - return false; - - switch( act ) { - case N(issue): - print( "issue\n"); - on( unpack_action_data() ); - return true; - case N(transfer): - print( "transfer\n"); - on( unpack_action_data() ); - return true; - case N(create): - print( "create\n"); - on( unpack_action_data() ); - return true; - } - return false; - } - - /** - * This is factored out so it can be used as a building block - */ - void create_currency( const create& c ) { - auto sym = c.maximum_supply.symbol; - eosio_assert( sym.is_valid(), "invalid symbol name" ); - - stats statstable( _contract, sym.name() ); - auto existing = statstable.find( sym.name() ); - eosio_assert( existing == statstable.end(), "token with symbol already exists" ); - - statstable.emplace( c.issuer, [&]( auto& s ) { - s.supply.symbol = c.maximum_supply.symbol; - s.max_supply = c.maximum_supply; - s.issuer = c.issuer; - s.can_freeze = c.issuer_can_freeze; - s.can_recall = c.issuer_can_recall; - s.can_whitelist = c.issuer_can_whitelist; - }); - } - - void issue_currency( const issue& i ) { - auto sym = i.quantity.symbol.name(); - stats statstable( _contract, sym ); - const auto& st = statstable.get( sym ); - - statstable.modify( st, 0, [&]( auto& s ) { - s.supply.amount += i.quantity.amount; - eosio_assert( s.supply.amount >= 0, "underflow" ); - }); - - add_balance( st.issuer, i.quantity, st, st.issuer ); - } - - - void on( const create& c ) { - require_auth( c.issuer ); - create_currency( c ); - - /* - auto fee = get_fee_schedule()[c.maximum_supply.name_length()]; - if( fee.amount > 0 ) { - inline_transfer( c.issuer, _contract, fee, "symbol registration fee" ); - } - */ - } - - void on( const issue& i ) { - auto sym = i.quantity.symbol.name(); - stats statstable( _contract, sym ); - const auto& st = statstable.get( sym ); - - require_auth( st.issuer ); - eosio_assert( i.quantity.is_valid(), "invalid quantity" ); - eosio_assert( i.quantity.amount > 0, "must issue positive quantity" ); - - statstable.modify( st, 0, [&]( auto& s ) { - s.supply.amount += i.quantity.amount; - }); - - add_balance( st.issuer, i.quantity, st, st.issuer ); - - if( i.to != st.issuer ) - { - inline_transfer( st.issuer, i.to, i.quantity, i.memo ); - } - } - - void on( const transfer& t ) { - require_auth(t.from); - auto sym = t.quantity.symbol.name(); - stats statstable( _contract, sym ); - const auto& st = statstable.get( sym ); - - require_recipient( t.to ); - - eosio_assert( t.quantity.is_valid(), "invalid quantity" ); - eosio_assert( t.quantity.amount > 0, "must transfer positive quantity" ); - sub_balance( t.from, t.quantity, st ); - add_balance( t.to, t.quantity, st, t.from ); - } - - - private: - void sub_balance( account_name owner, asset value, const currency_stats& st ) { - accounts from_acnts( _contract, owner ); - - const auto& from = from_acnts.get( value.symbol.name() ); - eosio_assert( from.balance.amount >= value.amount, "overdrawn balance" ); - - if( has_auth( owner ) ) { - eosio_assert( !st.can_freeze || !from.frozen, "account is frozen by issuer" ); - eosio_assert( !st.can_freeze || !st.is_frozen, "all transfers are frozen by issuer" ); - eosio_assert( !st.enforce_whitelist || from.whitelist, "account is not white listed" ); - } else if( has_auth( st.issuer ) ) { - eosio_assert( st.can_recall, "issuer may not recall token" ); - } else { - eosio_assert( false, "insufficient authority" ); - } - - from_acnts.modify( from, owner, [&]( auto& a ) { - a.balance.amount -= value.amount; - }); - } - - void add_balance( account_name owner, asset value, const currency_stats& st, account_name ram_payer ) - { - accounts to_acnts( _contract, owner ); - auto to = to_acnts.find( value.symbol.name() ); - if( to == to_acnts.end() ) { - eosio_assert( !st.enforce_whitelist, "can only transfer to white listed accounts" ); - to_acnts.emplace( ram_payer, [&]( auto& a ){ - a.balance = value; - }); - } else { - eosio_assert( !st.enforce_whitelist || to->whitelist, "receiver requires whitelist by issuer" ); - to_acnts.modify( to, 0, [&]( auto& a ) { - a.balance.amount += value.amount; - }); - } - } - - private: - account_name _contract; - }; - -} diff --git a/contracts/infinite/CMakeLists.txt b/contracts/infinite/CMakeLists.txt deleted file mode 100644 index af3ec44f496..00000000000 --- a/contracts/infinite/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -add_wast_executable(TARGET infinite - INCLUDE_FOLDERS "${STANDARD_INCLUDE_FOLDERS}" - LIBRARIES libc++ libc eosiolib - DESTINATION_FOLDER ${CMAKE_CURRENT_BINARY_DIR} -) diff --git a/contracts/infinite/infinite.cpp b/contracts/infinite/infinite.cpp deleted file mode 100644 index 21a542c1a28..00000000000 --- a/contracts/infinite/infinite.cpp +++ /dev/null @@ -1,16 +0,0 @@ -/** - * @file - * @copyright defined in eos/LICENSE.txt - */ -#include /// defines transfer struct (abi) - -extern "C" { - - /// The apply method just prints forever - void apply( uint64_t, uint64_t, uint64_t ) { - int idx = 0; - while(true) { - eosio::print(idx++); - } - } -} diff --git a/contracts/simple.token/CMakeLists.txt b/contracts/simple.token/CMakeLists.txt deleted file mode 100644 index 1e755eabe2f..00000000000 --- a/contracts/simple.token/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ - -file(GLOB ABI_FILES "*.abi") -configure_file("${ABI_FILES}" "${CMAKE_CURRENT_BINARY_DIR}" COPYONLY) - -add_wast_executable(TARGET simple.token - INCLUDE_FOLDERS "${STANDARD_INCLUDE_FOLDERS}" - LIBRARIES libc++ libc eosiolib - DESTINATION_FOLDER ${CMAKE_CURRENT_BINARY_DIR} -) diff --git a/contracts/simple.token/simple.token.abi b/contracts/simple.token/simple.token.abi deleted file mode 100644 index 2c63c085104..00000000000 --- a/contracts/simple.token/simple.token.abi +++ /dev/null @@ -1,2 +0,0 @@ -{ -} diff --git a/contracts/simple.token/simple.token.cpp b/contracts/simple.token/simple.token.cpp deleted file mode 100644 index 1f2ab97feca..00000000000 --- a/contracts/simple.token/simple.token.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include - -class simpletoken : public eosio::contract { - public: - simpletoken( account_name self ) - :contract(self),_accounts( _self, _self){} - - void transfer( account_name from, account_name to, uint64_t quantity ) { - require_auth( from ); - - const auto& fromacnt = _accounts.get( from ); - eosio_assert( fromacnt.balance >= quantity, "overdrawn balance" ); - _accounts.modify( fromacnt, from, [&]( auto& a ){ a.balance -= quantity; } ); - - add_balance( from, to, quantity ); - } - - void issue( account_name to, uint64_t quantity ) { - require_auth( _self ); - add_balance( _self, to, quantity ); - } - - private: - struct account { - account_name owner; - uint64_t balance; - - uint64_t primary_key()const { return owner; } - }; - - eosio::multi_index _accounts; - - void add_balance( account_name payer, account_name to, uint64_t q ) { - auto toitr = _accounts.find( to ); - if( toitr == _accounts.end() ) { - _accounts.emplace( payer, [&]( auto& a ) { - a.owner = to; - a.balance = q; - }); - } else { - _accounts.modify( toitr, 0, [&]( auto& a ) { - a.balance += q; - eosio_assert( a.balance >= q, "overflow detected" ); - }); - } - } -}; - -EOSIO_ABI( simpletoken, (transfer)(issue) ) diff --git a/contracts/social/CMakeLists.txt b/contracts/social/CMakeLists.txt deleted file mode 100644 index 72d5d9e059e..00000000000 --- a/contracts/social/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -add_wast_executable(TARGET social - INCLUDE_FOLDERS "${STANDARD_INCLUDE_FOLDERS}" - LIBRARIES eosiolib - DESTINATION_FOLDER ${CMAKE_CURRENT_BINARY_DIR} -) diff --git a/contracts/social/social.cpp b/contracts/social/social.cpp deleted file mode 100644 index 7cee3dd1ddf..00000000000 --- a/contracts/social/social.cpp +++ /dev/null @@ -1,107 +0,0 @@ -/** - * @file - * @copyright defined in eos/LICENSE.txt - */ -#include - -/** - * The purpose of this contract is to implement something like Steem on EOS, this - * means this contract defines its own currency, allows people to create posts, vote - * on posts, and stake their voting power. - * - * Unlike Steem, the goal is to enable maximum parallelism and enable the currency to - * be easily integrated with an exchange contract. - */ - -struct post_action { - account_name author; - post_name postid; - account_name reply_to_author; - int32_t reply_to_id; - uint8_t author; /// index in notify list - char[] data; /// ignored, title is embedded within - - account_name get_author()const { return get_notify(author); } -}; - -struct vote_action { - account_name voter; - account_name author; - post_name postid; - int32_t vote_power; -}; - -struct post_record { - uint64_t total_votes = 0; - uint64_t claimed_votes = 0; - uint32_t created; - - post_record( uint32_t c = now() ):created(c){} - static Name table_id() { return Name("post"); } -}; - -struct account { - uint64_t social = 0; - uint64_t social_power = 0; - int32_t vote_power = 0; - uint32_t last_vote = 0; - - static Name table_id() { return Name("account"); } -}; - - -/** - * When a user posts we create a record that tracks the total votes and the time it - * was created. A user can submit this action multiple times, but subsequent calls do - * nothing. - * - * This method only does something when called in the context of the author, if - * any other contexts are notified - */ -void apply_social_post() { - const auto& post = current_action(); - require_auth( post.author ); - - eosio_assert( current_context() == post.author, "cannot call from any other context" ); - - static post_record& existing; - if( !Db::get( post.postid, existing ) ) - Db::store( post.postid, post_record( now() ) ); -} - -/** - * This action is called when a user casts a vote, it requires that this code is executed - * in the context of both the voter and the author. When executed in the author's context it - * updates the vote total. When executed - */ -void apply_social_vote() { - const auto& vote = current_action(); - require_recipient( vote.voter, vote.author ); - disable_context_code( vote.author() ); /// prevent the author's code from rejecting the potentially negative vote - - auto context = current_context(); - auto voter = vote.getVoter(); - - if( context == vote.author ) { - static post_record post; - eosio_assert( Db::get( vote.postid, post ) > 0, "unable to find post" ); - eosio_assert( now() - post.created < days(7), "cannot vote after 7 days" ); - post.votes += vote.vote_power; - Db::store( vote.postid, post ); - } - else if( context == vote.voter ) { - static account vote_account; - Db::get( "account", vote_account ); - auto abs_vote = abs(vote.vote_power); - vote_account.vote_power = min( vote_account.social_power, - vote_account.vote_power + (vote_account.social_power * (now()-last_vote)) / days(7)); - eosio_assert( abs_vote <= vote_account.vote_power, "insufficient vote power" ); - post.votes += vote.vote_power; - vote_account.vote_power -= abs_vote; - vote_account.last_vote = now(); - Db::store( "account", vote_account ); - } else { - eosio_assert( false, "invalid context for execution of this vote" ); - } -} - diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ad77d850285..203a7cdd023 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -21,7 +21,6 @@ target_link_libraries( plugin_test eosio_testing eosio_chain chainbase eos_utili target_include_directories( plugin_test PUBLIC ${CMAKE_SOURCE_DIR}/plugins/net_plugin/include ${CMAKE_SOURCE_DIR}/plugins/chain_plugin/include ) -add_dependencies(plugin_test asserter test_api test_api_mem test_api_db test_api_multi_index proxy identity identity_test stltest infinite eosio.system eosio.token eosio.bios test.inline multi_index_test noop eosio.msig) # configure_file(${CMAKE_CURRENT_SOURCE_DIR}/core_symbol.py.in ${CMAKE_CURRENT_BINARY_DIR}/core_symbol.py) diff --git a/tests/chain_plugin_tests.cpp b/tests/chain_plugin_tests.cpp index 2b7867a07e9..5a489c255b4 100644 --- a/tests/chain_plugin_tests.cpp +++ b/tests/chain_plugin_tests.cpp @@ -12,15 +12,6 @@ #include #include -#include -#include - -#include -#include - -#include -#include - #include #include diff --git a/tests/chain_tests/proof_tests.cpp b/tests/chain_tests/proof_tests.cpp deleted file mode 100644 index 6240443d77d..00000000000 --- a/tests/chain_tests/proof_tests.cpp +++ /dev/null @@ -1,340 +0,0 @@ -#include -#include -#include -#include - -#ifdef NON_VALIDATING_TEST -#define TESTER tester -#else -#define TESTER validating_tester -#endif - -using namespace eosio; -using namespace eosio::chain; -using namespace eosio::testing; - -struct action_proof_data { - account_name receiver; - scope_name scope; - action_name name; - bytes data; - uint64_t region_id; - uint64_t cycle_index; - vector data_access; -}; -FC_REFLECT(action_proof_data, (receiver)(scope)(name)(data)(region_id)(cycle_index)(data_access)); - - -struct merkle_node { - digest_type digest; - optional left; - optional right; - optional parent; -}; - -digest_type process_merkle(vector& nodes, vector leaf_indices) { - size_t num_nodes = leaf_indices.size(); - size_t base = 0; - - if (num_nodes == 0) { - return digest_type(); - } else if (num_nodes == 1) { - return nodes[leaf_indices.front()].digest; - } - - while (num_nodes > 1) { - size_t new_nodes = (num_nodes / 2) + (num_nodes % 2); - for(size_t idx = 0; idx < new_nodes; idx++) { - size_t l_idx = idx * 2; - size_t r_idx = l_idx + 1; - size_t left = leaf_indices.at(base + l_idx); - if (r_idx >= num_nodes) { - nodes.emplace_back( - merkle_node{ - digest_type::hash(make_canonical_pair(nodes[left].digest, nodes[left].digest)), - left - }); - nodes[left].parent.emplace(nodes.size() - 1); - } else { - size_t right = leaf_indices.at(base + r_idx); - nodes.emplace_back( - merkle_node{ - digest_type::hash(make_canonical_pair(nodes[left].digest, nodes[right].digest)), - left, - right - }); - nodes[left].parent.emplace(nodes.size() - 1); - nodes[right].parent.emplace(nodes.size() - 1); - } - - leaf_indices.emplace_back(nodes.size() - 1); - } - - base += num_nodes; - num_nodes = new_nodes; - } - - return nodes.back().digest; -} - -auto get_proof_path(const vector& nodes, size_t leaf) { - vector path; - digest_type current = nodes[leaf].digest; - while (nodes[leaf].parent.valid()) { - size_t parent = *nodes[leaf].parent; - if (leaf == *nodes[parent].left) { - if (nodes[parent].right.valid()) { - size_t right = *nodes[parent].right; - path.emplace_back(make_canonical_right(nodes[right].digest)); - current = digest_type::hash(make_canonical_pair(nodes[leaf].digest, nodes[right].digest)); - } else { - path.emplace_back(make_canonical_right(nodes[leaf].digest)); - current = digest_type::hash(make_canonical_pair(nodes[leaf].digest, nodes[leaf].digest)); - } - } else { - size_t left = *nodes[parent].left; - path.emplace_back(make_canonical_left(nodes[left].digest)); - current = digest_type::hash(make_canonical_pair(nodes[left].digest, nodes[leaf].digest)); - } - - leaf = parent; - } - - return path; -} - -digest_type apply_path(const digest_type& digest, const vector& path) { - digest_type current = digest; - - for (const auto& p: path ) { - if (is_canonical_left(p)) { - current = digest_type::hash(make_canonical_pair(p, current)); - } else { - current = digest_type::hash(make_canonical_pair(current, p)); - } - } - - return current; -} - -bool proof_is_valid(const digest_type& digest, const vector& path, const digest_type& expected_root) { - return apply_path(digest, path) == expected_root; -} - -BOOST_AUTO_TEST_SUITE(proof_tests) - -BOOST_FIXTURE_TEST_CASE( prove_block_in_chain, validating_tester ) { try { - vector known_blocks; - known_blocks.reserve(100); - block_header last_block_header; - - // register a callback on new blocks to record block information - validating_node->applied_block.connect([&](const block_trace& bt){ - known_blocks.emplace_back(bt.block.id()); - last_block_header = bt.block; - }); - - produce_blocks(100); - return; - vector nodes; - vector ids; - vector block_leaves; - nodes.reserve(100); - ids.reserve(100); - for (const auto& blk_id: known_blocks) { - nodes.emplace_back(merkle_node{blk_id}); - ids.push_back(blk_id); - block_leaves.push_back(nodes.size() - 1); - } - - digest_type block_mroot = process_merkle(nodes, move(block_leaves)); - - BOOST_REQUIRE_EQUAL((std::string)block_mroot, (std::string)merkle(ids)); - - produce_block(); - - BOOST_REQUIRE_EQUAL((std::string)block_mroot, (std::string)last_block_header.block_mroot); - - for (int idx = 0; idx < 100; idx++) { - vector path = get_proof_path(nodes, idx); - - BOOST_REQUIRE_EQUAL(true, proof_is_valid(nodes[idx].digest, path, last_block_header.block_mroot)); - - /** UNCOMMENT TO PRODUCE PROOFS TO STD OUT - std::cout << idx << ":" << std::string(known_blocks.at(idx)) << " = "; - for (const auto& e: path) { - std::cout << std::string(e) << " "; - } - - std::cout << "-> " << std::string(last_block_header.block_mroot) << std::endl; - */ - } - -} FC_LOG_AND_RETHROW() } /// transfer_test - - -struct action_proof_info { - action_trace trace; - - size_t action_leaf; - size_t block_leaf; - - uint32_t cycle_index; - uint32_t region_id; -}; - -/** - * This test case will attempt to allow one account to transfer on behalf - * of another account by updating the active authority. - */ -BOOST_FIXTURE_TEST_CASE( prove_action_in_block, validating_tester ) { try { - vector nodes; - vector block_leaves; - vector known_actions; - map block_action_mroots; - block_header last_block_header; - block_id_type last_block_id; - - // register a callback on new blocks to record block information - validating_node->applied_block.connect([&](const block_trace& bt){ - nodes.emplace_back(merkle_node{bt.block.id()}); - size_t block_leaf = nodes.size() - 1; - block_leaves.push_back(block_leaf); - - vector region_leaves; - - for (uint32_t r_idx = 0; r_idx < bt.region_traces.size(); r_idx++) { - const auto& rt = bt.region_traces.at(r_idx); - - vector shard_leaves; - - for (uint32_t c_idx = 0; c_idx < rt.cycle_traces.size(); c_idx++) { - const auto& ct = rt.cycle_traces.at(c_idx); - - for (const auto& st: ct.shard_traces) { - vector action_leaves; - - for (const auto& tt: st.transaction_traces) { - for (const auto& at: tt.action_traces) { - digest_type::encoder enc; - - auto a_data = action_proof_data { - at.receiver, - at.act.account, - at.act.name, - at.act.data, - tt.region_id, - tt.cycle_index, - at.data_access - }; - fc::raw::pack(enc, a_data); - nodes.emplace_back(merkle_node{enc.result()}); - size_t action_leaf = nodes.size() - 1; - known_actions.emplace_back(action_proof_info{at, action_leaf, block_leaf, c_idx, r_idx }); - action_leaves.emplace_back(action_leaf); - } - } - - if (action_leaves.size() > 0) { - process_merkle(nodes, move(action_leaves)); - } else { - nodes.emplace_back(merkle_node{digest_type()}); - } - shard_leaves.emplace_back(nodes.size() - 1); - } - } - - if (shard_leaves.size() > 0) { - process_merkle(nodes, move(shard_leaves)); - } else { - nodes.emplace_back(merkle_node{digest_type()}); - } - - region_leaves.emplace_back(nodes.size() - 1); - } - - digest_type action_mroot = process_merkle(nodes, move(region_leaves)); - BOOST_REQUIRE_EQUAL((std::string)bt.block.action_mroot, (std::string)action_mroot); - - last_block_header = bt.block; - last_block_id = bt.block.id(); - block_action_mroots[bt.block.id()] = bt.block.action_mroot; - }); - - create_accounts( { N(alice), N(bob), N(carol), N(david), N(elvis) }); - - produce_blocks(50); - - push_dummy(N(alice), "AB"); - push_dummy(N(bob), "BC"); - push_dummy(N(carol), "CD"); - push_dummy(N(david), "DE"); - push_dummy(N(elvis), "EF"); - - produce_blocks(50); - digest_type block_mroot = process_merkle(nodes, move(block_leaves)); - - produce_block(); - BOOST_REQUIRE_EQUAL((std::string)block_mroot, (std::string)last_block_header.block_mroot); - - /* UNCOMMENT TO PRODUCE PROOFS TO STDOUT - std::cout << "Best Block ID: " << (std::string)last_block_id << std::endl; - std::cout << " Merkle Root: " << (std::string)last_block_header.block_mroot << std::endl; - - for(const auto& ai : known_actions) { - auto block_path = get_proof_path(nodes, ai.action_leaf); - auto chain_path = get_proof_path(nodes, ai.block_leaf); - - // prove action in block - auto shard_root = apply_path(nodes[ai.action_leaf].digest, block_path); - BOOST_REQUIRE_EQUAL((std::string)shard_root, (std::string)block_action_mroots[nodes[ai.block_leaf].digest]); - - // prove that block is part of the chain - auto expected_block_mroot = apply_path(nodes[ai.block_leaf].digest, chain_path); - BOOST_REQUIRE_EQUAL((std::string)expected_block_mroot, (std::string)last_block_header.block_mroot); - - std::cout << "Proof for Action:" << std::endl; - std::cout << std::setw(14) << "reciever" << ":" << (std::string) ai.trace.receiver << std::endl; - std::cout << std::setw(14) << "scope" << ":" << (std::string) ai.trace.act.scope << std::endl; - std::cout << std::setw(14) << "name" << ":" << (std::string) ai.trace.act.name << std::endl; - std::cout << std::setw(14) << "data" << ":" << fc::json::to_string(ai.trace.act.as()) << std::endl; - std::cout << std::setw(14) << "data_access" << ":" << fc::json::to_string(ai.trace.data_access) << std::endl; - std::cout << std::setw(14) << "region" << ":" << ai.region_id << std::endl; - std::cout << std::setw(14) << "cycle" << ":" << ai.cycle_index << std::endl; - std::cout << std::setw(14) << "block" << ":" << (std::string)nodes[ai.block_leaf].digest << std::endl; - std::cout << std::setw(14) << "am_root" << ":" << (std::string)block_action_mroots[nodes[ai.block_leaf].digest] << std::endl; - std::cout << std::endl; - - auto a_data = action_proof_data { - ai.trace.receiver, - ai.trace.act.scope, - ai.trace.act.name, - ai.trace.act.data, - ai.trace.region_id, - ai.trace.cycle_index, - ai.trace.data_access - }; - auto action_data = fc::raw::pack(a_data); - - std::cout << "Action Hex: " << fc::to_hex(action_data) << std::endl; - std::cout << "Action Hash: " << (std::string)nodes[ai.action_leaf].digest << std::endl; - - std::cout << "Action Path: "; - for (const auto& p: block_path) { - std::cout << (std::string)p << " "; - } - std::cout << std::endl; - - std::cout << "Block Path: "; - for (const auto& p: chain_path) { - std::cout << (std::string)p << " "; - } - std::cout << std::endl; - }*/ - - -} FC_LOG_AND_RETHROW() } - - - -BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/get_table_tests.cpp b/tests/get_table_tests.cpp index 818ca5562d2..634f736222e 100644 --- a/tests/get_table_tests.cpp +++ b/tests/get_table_tests.cpp @@ -12,12 +12,6 @@ #include #include -#include -#include - -#include -#include - #include #include diff --git a/tests/p2p_tests/fuzz_test_generic.sh b/tests/p2p_tests/fuzz_test_generic.sh deleted file mode 100644 index ae913565c86..00000000000 --- a/tests/p2p_tests/fuzz_test_generic.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -# -# This script captures the generic fuzz testing to which -# net_plugin has been subjected. It typically results in -# message buffers in excess of 1 gigabyte. -# -if ! pgrep nodeos > /dev/null; then - echo "Run nodeos with net_plugin configured for port 9876." - exit 1 -fi -for i in `seq 1 10000`; do netcat localhost 9876 < /dev/urandom; done diff --git a/tests/p2p_tests/pump/run_test.pl b/tests/p2p_tests/pump/run_test.pl deleted file mode 100755 index 11e467a5d2e..00000000000 --- a/tests/p2p_tests/pump/run_test.pl +++ /dev/null @@ -1,285 +0,0 @@ -#!/usr/bin/perl - -use strict; -use Getopt::Long; -use Env; -use File::Basename; -use File::copy; -use File::Spec; -use File::Path; -use Cwd; - -my $eos_home = defined $ENV{EOS_HOME} ? $ENV{EOS_HOME} : getcwd; -my $eosd = $eos_home . "/programs/eosd/eosd"; -my $eosc = $eos_home . "/programs/eosc/eosc"; - -my $nodes = defined $ENV{EOS_TEST_RING} ? $ENV{EOS_TEST_RING} : "1"; -my $pnodes = defined $ENV{EOS_TEST_PRODUCERS} ? $ENV{EOS_TEST_PRODUCERS} : "1"; - -my $prods = 21; -my $genesis = "$eos_home/genesis.json"; -my $http_port_base = 8888; -my $p2p_port_base = 9876; -my $data_dir_base = "tdn"; -my $hostname = "127.0.0.1"; -my $first_pause = 0; -my $launch_pause = 0; -my $run_duration = 60; -my $topo = "ring"; -my $override_gts = ""; -my $no_delay=0; -my $test=1; - -if (!GetOptions("nodes=i" => \$nodes, - "first-pause=i" => \$first_pause, - "launch-pause=i" => \$launch_pause, - "duration=i" => \$run_duration, - "topo=s" => \$topo, - "test=i" => \$test, - "time-stamp=s" => \$override_gts, - "pnodes=i" => \$pnodes)) { - print "usage: $ARGV[0] [--nodes=] [--pnodes=] [--topo=] [--first-pause=] [--launch-pause=] [--duration=] [--time-stamp=