From e9b47e26e17bb638e566b43a79abcabc4e0409fc Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 5 Aug 2019 17:05:28 -0400 Subject: [PATCH 01/21] Optimize unpacking of packed_transaction and signed_block from network --- plugins/net_plugin/net_plugin.cpp | 57 ++++++++++++++++--------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 45674b00ac1..8adbae8a734 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -728,11 +728,11 @@ namespace eosio { void handle_message( const request_message& msg ); void handle_message( const sync_request_message& msg ); void handle_message( const signed_block& msg ) = delete; // signed_block_ptr overload used instead - void handle_message( const signed_block_ptr& msg ); + void handle_message( const block_id_type& id, signed_block_ptr msg ); void handle_message( const packed_transaction& msg ) = delete; // packed_transaction_ptr overload used instead - void handle_message( const packed_transaction_ptr& msg ); + void handle_message( packed_transaction_ptr msg ); - void process_signed_block( const signed_block_ptr& msg ); + void process_signed_block( const block_id_type& id, signed_block_ptr msg ); fc::optional _logger_variant; const fc::variant_object& get_logger_variant() { @@ -772,16 +772,11 @@ namespace eosio { } void operator()( signed_block&& msg ) const { - // continue call to handle_message on connection strand - shared_ptr ptr = std::make_shared( std::move( msg ) ); - c->handle_message( ptr ); + EOS_ASSERT( false, plugin_config_exception, "operator()(signed_block&&) should call handle_message" ); } void operator()( packed_transaction&& msg ) const { - // continue call to handle_message on connection strand - fc_dlog( logger, "handle packed_transaction" ); - shared_ptr ptr = std::make_shared( std::move( msg ) ); - c->handle_message( ptr ); + EOS_ASSERT( false, plugin_config_exception, "operator()(packed_transaction&&) should call handle_message" ); } void operator()( const handshake_message& msg ) const { @@ -2409,6 +2404,7 @@ namespace eosio { bool connection::process_next_message( uint32_t message_length ) { try { // if next message is a block we already have, exit early + auto ds = pending_message_buffer.create_datastream(); auto peek_ds = pending_message_buffer.create_peek_datastream(); unsigned_int which{}; fc::raw::unpack( peek_ds, which ); @@ -2427,19 +2423,25 @@ namespace eosio { pending_message_buffer.advance_read_ptr( message_length ); return true; } - } - auto ds = pending_message_buffer.create_datastream(); - net_message msg; - fc::raw::unpack( ds, msg ); - msg_handler m( shared_from_this() ); - if( msg.contains() ) { - m( std::move( msg.get() ) ); - } else if( msg.contains() ) { - m( std::move( msg.get() ) ); + fc::raw::unpack( ds, which ); // throw away + shared_ptr ptr = std::make_shared(); + fc::raw::unpack( ds, *ptr ); + handle_message( blk_id, std::move( ptr ) ); + + } else if( which == packed_transaction_which ) { + fc::raw::unpack( ds, which ); // throw away + shared_ptr ptr = std::make_shared(); + fc::raw::unpack( ds, *ptr ); + handle_message( std::move( ptr ) ); + } else { + net_message msg; + fc::raw::unpack( ds, msg ); + msg_handler m( shared_from_this() ); msg.visit( m ); } + } catch( const fc::exception& e ) { fc_elog( logger, "Exception in handling message from ${p}: ${s}", ("p", peer_name())("s", e.to_detail_string()) ); @@ -2794,7 +2796,7 @@ namespace eosio { trx->get_signatures().size() * sizeof(signature_type); } - void connection::handle_message( const packed_transaction_ptr& trx ) { + void connection::handle_message( packed_transaction_ptr trx ) { if( my_impl->db_read_mode == eosio::db_read_mode::READ_ONLY ) { fc_dlog( logger, "got a txn in read-only mode - dropping" ); return; @@ -2811,9 +2813,9 @@ namespace eosio { } trx_in_progress_size += calc_trx_size( trx ); - app().post( priority::low, [trx, weak = weak_from_this()]() { + app().post( priority::low, [trx{std::move(trx)}, weak = weak_from_this()]() { my_impl->chain_plug->accept_transaction( trx, - [weak, trx](const static_variant& result) { + [weak, trx](const static_variant& result) mutable { // next (this lambda) called from application thread bool accepted = false; if (result.contains()) { @@ -2848,18 +2850,17 @@ namespace eosio { } // called from connection strand - void connection::handle_message( const signed_block_ptr& ptr ) { - app().post(priority::high, [ptr, weak = weak_from_this()] { + void connection::handle_message( const block_id_type& id, signed_block_ptr ptr ) { + app().post(priority::high, [ptr{std::move(ptr)}, id, weak = weak_from_this()] { connection_ptr c = weak.lock(); - if( c ) c->process_signed_block( ptr ); + if( c ) c->process_signed_block( id, std::move( ptr ) ); }); - my_impl->dispatcher->bcast_notice( ptr->id() ); + my_impl->dispatcher->bcast_notice( id ); } // called from application thread - void connection::process_signed_block( const signed_block_ptr& msg ) { + void connection::process_signed_block( const block_id_type& blk_id, signed_block_ptr msg ) { controller& cc = my_impl->chain_plug->chain(); - block_id_type blk_id = msg->id(); uint32_t blk_num = msg->block_num(); // use c in this method instead of this to highlight that all methods called on c-> must be thread safe connection_ptr c = shared_from_this(); From 48e08fb12633c9f0381fddb417efba52f427836e Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 5 Aug 2019 17:28:46 -0400 Subject: [PATCH 02/21] Remove recv_transaction and save a shared_ptr create now that recv_transaction only calls add_peer_txn --- plugins/net_plugin/net_plugin.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 8adbae8a734..53c9ec5e049 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -192,7 +192,6 @@ namespace eosio { void recv_block(const connection_ptr& conn, const block_id_type& msg, uint32_t bnum); void expire_blocks( uint32_t bnum ); - void recv_transaction(const connection_ptr& conn, const packed_transaction_ptr& txn); void recv_notice(const connection_ptr& conn, const notice_message& msg, bool generated); void retry_fetch(const connection_ptr& conn); @@ -2027,11 +2026,6 @@ namespace eosio { } ); } - void dispatch_manager::recv_transaction(const connection_ptr& c, const packed_transaction_ptr& txn) { - node_transaction_state nts = {txn->id(), txn->expiration(), 0, c->connection_id}; - add_peer_txn( nts ); - } - void dispatch_manager::rejected_transaction(const transaction_id_type& id, uint32_t head_blk_num) { fc_dlog( logger, "not sending rejected transaction ${tid}", ("tid", id) ); // keep rejected transaction around for awhile so we don't broadcast it @@ -2806,7 +2800,9 @@ namespace eosio { peer_dlog( this, "received packed_transaction ${id}", ("id", tid) ); bool have_trx = my_impl->dispatcher->have_txn( tid ); - my_impl->dispatcher->recv_transaction( shared_from_this(), trx ); + node_transaction_state nts = {tid, trx->expiration(), 0, connection_id}; + my_impl->dispatcher->add_peer_txn( nts ); + if( have_trx ) { fc_dlog( logger, "got a duplicate transaction - dropping ${id}", ("id", tid) ); return; From ec95ff6470d7005d5692cc15610e16d019498a8c Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 5 Aug 2019 17:42:05 -0400 Subject: [PATCH 03/21] Remove unneeded connection shared_ptr copy --- plugins/net_plugin/net_plugin.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 53c9ec5e049..0c4c56c9f83 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2910,8 +2910,8 @@ namespace eosio { } if( reason == no_reason ) { - boost::asio::post( my_impl->thread_pool->get_executor(), [dispatcher = my_impl->dispatcher.get(), c, msg]() { - dispatcher->add_peer_block( msg->id(), c->connection_id ); + boost::asio::post( my_impl->thread_pool->get_executor(), [dispatcher = my_impl->dispatcher.get(), cid=c->connection_id, blk_id, msg]() { + dispatcher->add_peer_block( blk_id, cid ); dispatcher->update_txns_block_num( msg ); }); c->strand.post( [sync_master = my_impl->sync_master.get(), dispatcher = my_impl->dispatcher.get(), c, blk_id, blk_num]() { From 038689c1813ab6d9813782e2a0b7e59053e0c403 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 6 Aug 2019 08:49:38 -0400 Subject: [PATCH 04/21] Remove unneeded shared_from_this() call --- plugins/net_plugin/net_plugin.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 0c4c56c9f83..79a5dedfe5f 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2408,11 +2408,10 @@ namespace eosio { block_id_type blk_id = bh.id(); if( my_impl->dispatcher->have_block( blk_id ) ) { - connection_ptr c = shared_from_this(); fc_dlog( logger, "canceling wait on ${p}, already received block ${num}", - ("p", c->peer_name())("num", block_header::num_from_id( blk_id )) ); - c->consecutive_rejected_blocks = 0; - c->cancel_wait(); + ("p", peer_name())("num", block_header::num_from_id( blk_id )) ); + consecutive_rejected_blocks = 0; + cancel_wait(); pending_message_buffer.advance_read_ptr( message_length ); return true; From a8d4beea4483bf53a77e3fe6b3f4029366e517e7 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 6 Aug 2019 09:07:13 -0400 Subject: [PATCH 05/21] Remove unneeded chain_plug from sync_manager --- plugins/net_plugin/net_plugin.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 79a5dedfe5f..35906c776b7 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -149,8 +149,6 @@ namespace eosio { connection_ptr sync_source; std::atomic sync_state; - chain_plugin* chain_plug = nullptr; - private: constexpr static auto stage_str( stages s ); void set_state( stages s ); @@ -1360,8 +1358,6 @@ namespace eosio { ,sync_source() ,sync_state(in_sync) { - chain_plug = app().find_plugin(); - EOS_ASSERT( chain_plug, chain::missing_chain_plugin_exception, "" ); } constexpr auto sync_manager::stage_str(stages s) { From ebc92806fa22592f28015a2763a4ebd71c320bfb Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 8 Aug 2019 17:11:22 -0400 Subject: [PATCH 06/21] Use c-file functions for read/write of block log instead of fstream --- libraries/chain/block_log.cpp | 258 ++++++++++++++++++++++++++-------- 1 file changed, 200 insertions(+), 58 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index d979b087fbf..1d8bc80f802 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #define LOG_READ (std::ios::in | std::ios::binary) #define LOG_WRITE (std::ios::out | std::ios::binary | std::ios::app) @@ -27,14 +29,155 @@ namespace eosio { namespace chain { namespace detail { using unique_file = std::unique_ptr; + class cfile_datastream; + + class cfile { + public: + explicit cfile() + : _file(nullptr, &fclose) + {} + + void set_file_path( fc::path file_path ) { + _file_path = std::move( file_path ); + } + + fc::path get_file_path() const { + return _file_path; + } + + bool is_open() const { return _open; } + + size_t file_size() const { + return fc::file_size( _file_path ); + } + + void open_for_append() { + _file.reset( fopen( _file_path.generic_string().c_str(), "ab+" ) ); + if( !_file ) { + throw std::runtime_error( "cfile unable to open: " + _file_path.generic_string() ); + } + _open = true; + } + + void open_for_rw() { + _file.reset( fopen( _file_path.generic_string().c_str(), "rb+" ) ); + if( !_file ) { + throw std::runtime_error( "cfile unable to open: " + _file_path.generic_string() ); + } + _open = true; + } + + uint64_t tellp() const { + return ftell( _file.get() ); + } + + uint64_t tellg() const { + fpos_t pos; + if( 0 != ::fgetpos( _file.get(), &pos ) ) { + throw std::runtime_error( "cfile: " + _file_path.generic_string() + " unable to fgetpos" ); + } + return pos; + } + + void seek( long loc ) { + if( 0 != fseek( _file.get(), loc, SEEK_SET ) ) { + throw std::runtime_error( "cfile: " + _file_path.generic_string() + + " unable to SEEK_SET to: " + std::to_string(loc) ); + } + } + + void seek_end( long loc ) { + if( 0 != fseek( _file.get(), loc, SEEK_END ) ) { + throw std::runtime_error( "cfile: " + _file_path.generic_string() + + " unable to SEEK_END to: " + std::to_string(loc) ); + } + } + + void read( char* d, size_t n ) { + size_t result = fread( d, 1, n, _file.get() ); + if( result != n ) { + throw std::runtime_error( "cfile: " + _file_path.generic_string() + + " unable to read " + std::to_string( n ) + " only read " + std::to_string( result ) ); + } + } + + void write( const char* d, size_t n ) { + size_t result = fwrite( d, 1, n, _file.get() ); + if( result != n ) { + throw std::runtime_error( "cfile: " + _file_path.generic_string() + + " unable to write " + std::to_string( n ) + " only wrote " + std::to_string( result ) ); + } + } + + void flush() { + if( 0 != fflush( _file.get() ) ) { + throw std::runtime_error( "cfile: " + _file_path.generic_string() + " unable to flush file." ); + } + } + + void close() { + _file.reset(); + _open = false; + } + + void remove() { + if( _open ) { + throw std::runtime_error( "cfile: " + _file_path.generic_string() + " Unable to remove as file is open" ); + } + fc::remove_all( _file_path ); + } + + cfile_datastream create_datastream(); + + private: + bool _open = false; + fc::path _file_path; + unique_file _file; + }; + + /* + * @brief datastream adapter that adapts cfile for use with fc unpack + * + * This class supports unpack functionality but not pack. + */ + class cfile_datastream { + public: + explicit cfile_datastream( cfile& cf ) : cf(cf) {} + + void skip( size_t s ) { + std::vector d( s ); + read( &d[0], s ); + } + + bool read( char* d, size_t s ) { + cf.read( d, s ); + return true; + } + + bool get( unsigned char& c ) { + char cc; + cf.read(&cc, 1); + c = cc; + return true; + } + + bool get( char& c ) { return read(&c, 1); } + + private: + cfile& cf; + }; + + inline cfile_datastream cfile::create_datastream() { + return cfile_datastream(*this); + } + + class block_log_impl { public: signed_block_ptr head; block_id_type head_id; - std::fstream block_stream; - std::fstream index_stream; - fc::path block_file; - fc::path index_file; + cfile block_file; + cfile index_file; bool open_files = false; bool genesis_written_to_block_log = false; uint32_t version = 0; @@ -48,10 +191,10 @@ namespace eosio { namespace chain { void reopen(); void close() { - if( block_stream.is_open() ) - block_stream.close(); - if( index_stream.is_open() ) - index_stream.close(); + if( block_file.is_open() ) + block_file.close(); + if( index_file.is_open() ) + index_file.close(); open_files = false; } }; @@ -61,13 +204,13 @@ namespace eosio { namespace chain { // open to create files if they don't exist //ilog("Opening block log at ${path}", ("path", my->block_file.generic_string())); - block_stream.open(block_file.generic_string().c_str(), LOG_WRITE); - index_stream.open(index_file.generic_string().c_str(), LOG_WRITE); + block_file.open_for_append(); + index_file.open_for_append(); close(); - block_stream.open(block_file.generic_string().c_str(), LOG_RW); - index_stream.open(index_file.generic_string().c_str(), LOG_RW); + block_file.open_for_rw(); + index_file.open_for_rw(); open_files = true; } @@ -129,8 +272,6 @@ namespace eosio { namespace chain { block_log::block_log(const fc::path& data_dir) :my(new detail::block_log_impl()) { - my->block_stream.exceptions(std::fstream::failbit | std::fstream::badbit); - my->index_stream.exceptions(std::fstream::failbit | std::fstream::badbit); open(data_dir); } @@ -152,8 +293,8 @@ namespace eosio { namespace chain { if (!fc::is_directory(data_dir)) fc::create_directories(data_dir); - my->block_file = data_dir / "blocks.log"; - my->index_file = data_dir / "blocks.index"; + my->block_file.set_file_path( data_dir / "blocks.log" ); + my->index_file.set_file_path( data_dir / "blocks.index" ); my->reopen(); @@ -175,14 +316,14 @@ namespace eosio { namespace chain { * - If the index file head is not in the log file, delete the index and replay. * - If the index file head is in the log, but not up to date, replay from index head. */ - auto log_size = fc::file_size(my->block_file); - auto index_size = fc::file_size(my->index_file); + auto log_size = my->block_file.file_size(); + auto index_size = my->index_file.file_size(); if (log_size) { ilog("Log is nonempty"); - my->block_stream.seekg( 0 ); + my->block_file.seek( 0 ); my->version = 0; - my->block_stream.read( (char*)&my->version, sizeof(my->version) ); + my->block_file.read( (char*)&my->version, sizeof(my->version) ); EOS_ASSERT( my->version > 0, block_log_exception, "Block log was not setup properly" ); EOS_ASSERT( my->version >= min_supported_version && my->version <= max_supported_version, block_log_unsupported_version, "Unsupported version of block log. Block log version is ${version} while code supports version(s) [${min},${max}]", @@ -192,7 +333,7 @@ namespace eosio { namespace chain { my->genesis_written_to_block_log = true; // Assume it was constructed properly. if (my->version > 1){ my->first_block_num = 0; - my->block_stream.read( (char*)&my->first_block_num, sizeof(my->first_block_num) ); + my->block_file.read( (char*)&my->first_block_num, sizeof(my->first_block_num) ); EOS_ASSERT(my->first_block_num > 0, block_log_exception, "Block log is malformed, first recorded block number is 0 but must be greater than or equal to 1"); } else { my->first_block_num = 1; @@ -208,12 +349,12 @@ namespace eosio { namespace chain { if (index_size) { ilog("Index is nonempty"); uint64_t block_pos; - my->block_stream.seekg(-sizeof(uint64_t), std::ios::end); - my->block_stream.read((char*)&block_pos, sizeof(block_pos)); + my->block_file.seek_end(-sizeof(uint64_t)); + my->block_file.read((char*)&block_pos, sizeof(block_pos)); uint64_t index_pos; - my->index_stream.seekg(-sizeof(uint64_t), std::ios::end); - my->index_stream.read((char*)&index_pos, sizeof(index_pos)); + my->index_file.seek_end(-sizeof(uint64_t)); + my->index_file.read((char*)&index_pos, sizeof(index_pos)); if (block_pos < index_pos) { ilog("block_pos < index_pos, close and reopen index_stream"); @@ -229,7 +370,7 @@ namespace eosio { namespace chain { } else if (index_size) { ilog("Index is nonempty, remove and recreate it"); my->close(); - fc::remove_all(my->index_file); + my->index_file.remove(); my->reopen(); } } @@ -240,18 +381,18 @@ namespace eosio { namespace chain { my->check_open_files(); - my->block_stream.seekp(0, std::ios::end); - my->index_stream.seekp(0, std::ios::end); - uint64_t pos = my->block_stream.tellp(); - EOS_ASSERT(my->index_stream.tellp() == sizeof(uint64_t) * (b->block_num() - my->first_block_num), + my->block_file.seek_end(0); + my->index_file.seek_end(0); + uint64_t pos = my->block_file.tellp(); + EOS_ASSERT(my->index_file.tellp() == sizeof(uint64_t) * (b->block_num() - my->first_block_num), block_log_append_fail, "Append to index file occuring at wrong position.", - ("position", (uint64_t) my->index_stream.tellp()) + ("position", (uint64_t) my->index_file.tellp()) ("expected", (b->block_num() - my->first_block_num) * sizeof(uint64_t))); auto data = fc::raw::pack(*b); - my->block_stream.write(data.data(), data.size()); - my->block_stream.write((char*)&pos, sizeof(pos)); - my->index_stream.write((char*)&pos, sizeof(pos)); + my->block_file.write(data.data(), data.size()); + my->block_file.write((char*)&pos, sizeof(pos)); + my->index_file.write((char*)&pos, sizeof(pos)); my->head = b; my->head_id = b->id(); @@ -263,30 +404,30 @@ namespace eosio { namespace chain { } void block_log::flush() { - my->block_stream.flush(); - my->index_stream.flush(); + my->block_file.flush(); + my->index_file.flush(); } void block_log::reset( const genesis_state& gs, const signed_block_ptr& first_block, uint32_t first_block_num ) { my->close(); - fc::remove_all(my->block_file); - fc::remove_all(my->index_file); + my->block_file.remove(); + my->index_file.remove(); my->reopen(); auto data = fc::raw::pack(gs); my->version = 0; // version of 0 is invalid; it indicates that the genesis was not properly written to the block log my->first_block_num = first_block_num; - my->block_stream.seekp(0, std::ios::end); - my->block_stream.write((char*)&my->version, sizeof(my->version)); - my->block_stream.write((char*)&my->first_block_num, sizeof(my->first_block_num)); - my->block_stream.write(data.data(), data.size()); + my->block_file.seek_end(0); + my->block_file.write((char*)&my->version, sizeof(my->version)); + my->block_file.write((char*)&my->first_block_num, sizeof(my->first_block_num)); + my->block_file.write(data.data(), data.size()); my->genesis_written_to_block_log = true; // append a totem to indicate the division between blocks and header auto totem = npos; - my->block_stream.write((char*)&totem, sizeof(totem)); + my->block_file.write((char*)&totem, sizeof(totem)); if (first_block) { append(first_block); @@ -295,24 +436,25 @@ namespace eosio { namespace chain { my->head_id = {}; } - auto pos = my->block_stream.tellp(); + auto pos = my->block_file.tellp(); static_assert( block_log::max_supported_version > 0, "a version number of zero is not supported" ); my->version = block_log::max_supported_version; - my->block_stream.seekp( 0 ); - my->block_stream.write( (char*)&my->version, sizeof(my->version) ); - my->block_stream.seekp( pos ); + my->block_file.seek( 0 ); + my->block_file.write( (char*)&my->version, sizeof(my->version) ); + my->block_file.seek( pos ); flush(); } std::pair block_log::read_block(uint64_t pos)const { my->check_open_files(); - my->block_stream.seekg(pos); + my->block_file.seek(pos); std::pair result; result.first = std::make_shared(); - fc::raw::unpack(my->block_stream, *result.first); - result.second = uint64_t(my->block_stream.tellg()) + 8; + auto ds = my->block_file.create_datastream(); + fc::raw::unpack(ds, *result.first); + result.second = uint64_t(my->block_file.tellg()) + 8; return result; } @@ -333,9 +475,9 @@ namespace eosio { namespace chain { my->check_open_files(); if (!(my->head && block_num <= block_header::num_from_id(my->head_id) && block_num >= my->first_block_num)) return npos; - my->index_stream.seekg(sizeof(uint64_t) * (block_num - my->first_block_num)); + my->index_file.seek(sizeof(uint64_t) * (block_num - my->first_block_num)); uint64_t pos; - my->index_stream.read((char*)&pos, sizeof(pos)); + my->index_file.read((char*)&pos, sizeof(pos)); return pos; } @@ -345,12 +487,12 @@ namespace eosio { namespace chain { uint64_t pos; // Check that the file is not empty - my->block_stream.seekg(0, std::ios::end); - if (my->block_stream.tellg() <= sizeof(pos)) + my->block_file.seek_end(0); + if (my->block_file.tellg() <= sizeof(pos)) return {}; - my->block_stream.seekg(-sizeof(pos), std::ios::end); - my->block_stream.read((char*)&pos, sizeof(pos)); + my->block_file.seek_end(-sizeof(pos)); + my->block_file.read((char*)&pos, sizeof(pos)); if (pos != npos) { return read_block(pos).first; } else { @@ -370,14 +512,14 @@ namespace eosio { namespace chain { ilog("Reconstructing Block Log Index..."); my->close(); - fc::remove_all(my->index_file); + my->index_file.remove(); my->reopen(); my->close(); - block_log::construct_index(my->block_file, my->index_file); + block_log::construct_index(my->block_file.get_file_path(), my->index_file.get_file_path()); my->reopen(); } // construct_index From 24581637cb4d97a2e5a62ed076c1079cc03318f4 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 8 Aug 2019 17:19:05 -0400 Subject: [PATCH 07/21] Add logs for timing --- libraries/chain/block_log.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 1d8bc80f802..3128324a730 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -377,6 +377,7 @@ namespace eosio { namespace chain { uint64_t block_log::append(const signed_block_ptr& b) { try { + elog( "start append" ); EOS_ASSERT( my->genesis_written_to_block_log, block_log_append_fail, "Cannot append to block log until the genesis is first written" ); my->check_open_files(); @@ -389,15 +390,20 @@ namespace eosio { namespace chain { "Append to index file occuring at wrong position.", ("position", (uint64_t) my->index_file.tellp()) ("expected", (b->block_num() - my->first_block_num) * sizeof(uint64_t))); + elog( "before pack" ); auto data = fc::raw::pack(*b); + elog( "after pack, before write" ); my->block_file.write(data.data(), data.size()); my->block_file.write((char*)&pos, sizeof(pos)); my->index_file.write((char*)&pos, sizeof(pos)); my->head = b; my->head_id = b->id(); + elog( "after write, before flush" ); flush(); + elog( "after flush" ); + elog( "end append" ); return pos; } FC_LOG_AND_RETHROW() From 01be54999589693a0757053a31c6b7e5ef773a1a Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 8 Aug 2019 17:36:09 -0400 Subject: [PATCH 08/21] Fix type mismatch --- libraries/chain/block_log.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 3128324a730..6dd38e812dc 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -67,11 +67,11 @@ namespace eosio { namespace chain { _open = true; } - uint64_t tellp() const { + long tellp() const { return ftell( _file.get() ); } - uint64_t tellg() const { + long tellg() const { fpos_t pos; if( 0 != ::fgetpos( _file.get(), &pos ) ) { throw std::runtime_error( "cfile: " + _file_path.generic_string() + " unable to fgetpos" ); From 4fd85944fda1f04ce4400369649f5fce014ff11d Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 8 Aug 2019 17:59:54 -0400 Subject: [PATCH 09/21] Make tellg same as tellp --- libraries/chain/block_log.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 6dd38e812dc..c7acfa0758b 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -72,11 +72,12 @@ namespace eosio { namespace chain { } long tellg() const { - fpos_t pos; - if( 0 != ::fgetpos( _file.get(), &pos ) ) { - throw std::runtime_error( "cfile: " + _file_path.generic_string() + " unable to fgetpos" ); - } - return pos; + return tellp(); +// fpos_t pos; +// if( 0 != ::fgetpos( _file.get(), &pos ) ) { +// throw std::runtime_error( "cfile: " + _file_path.generic_string() + " unable to fgetpos" ); +// } +// return pos; } void seek( long loc ) { From 4ec2c55280a325d5a62c0f19a2148775173f04e6 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 8 Aug 2019 20:26:01 -0400 Subject: [PATCH 10/21] Remove extra logging --- libraries/chain/block_log.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index c7acfa0758b..29c396baf1d 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -378,7 +378,6 @@ namespace eosio { namespace chain { uint64_t block_log::append(const signed_block_ptr& b) { try { - elog( "start append" ); EOS_ASSERT( my->genesis_written_to_block_log, block_log_append_fail, "Cannot append to block log until the genesis is first written" ); my->check_open_files(); @@ -391,20 +390,15 @@ namespace eosio { namespace chain { "Append to index file occuring at wrong position.", ("position", (uint64_t) my->index_file.tellp()) ("expected", (b->block_num() - my->first_block_num) * sizeof(uint64_t))); - elog( "before pack" ); auto data = fc::raw::pack(*b); - elog( "after pack, before write" ); my->block_file.write(data.data(), data.size()); my->block_file.write((char*)&pos, sizeof(pos)); my->index_file.write((char*)&pos, sizeof(pos)); my->head = b; my->head_id = b->id(); - elog( "after write, before flush" ); flush(); - elog( "after flush" ); - elog( "end append" ); return pos; } FC_LOG_AND_RETHROW() From 4ed6785a92c4874e19c35a87294a9e4e0296e9dc Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 8 Aug 2019 20:38:12 -0400 Subject: [PATCH 11/21] Remove tellg as it was not needed. Also read_block no longer returns a pair as .second was not used. --- libraries/chain/block_log.cpp | 23 +++++-------------- .../chain/include/eosio/chain/block_log.hpp | 2 +- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 29c396baf1d..a9d487d17d3 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -71,15 +71,6 @@ namespace eosio { namespace chain { return ftell( _file.get() ); } - long tellg() const { - return tellp(); -// fpos_t pos; -// if( 0 != ::fgetpos( _file.get(), &pos ) ) { -// throw std::runtime_error( "cfile: " + _file_path.generic_string() + " unable to fgetpos" ); -// } -// return pos; - } - void seek( long loc ) { if( 0 != fseek( _file.get(), loc, SEEK_SET ) ) { throw std::runtime_error( "cfile: " + _file_path.generic_string() + @@ -447,15 +438,13 @@ namespace eosio { namespace chain { flush(); } - std::pair block_log::read_block(uint64_t pos)const { + signed_block_ptr block_log::read_block(uint64_t pos)const { my->check_open_files(); my->block_file.seek(pos); - std::pair result; - result.first = std::make_shared(); + signed_block_ptr result = std::make_shared(); auto ds = my->block_file.create_datastream(); - fc::raw::unpack(ds, *result.first); - result.second = uint64_t(my->block_file.tellg()) + 8; + fc::raw::unpack(ds, *result); return result; } @@ -464,7 +453,7 @@ namespace eosio { namespace chain { signed_block_ptr b; uint64_t pos = get_block_pos(block_num); if (pos != npos) { - b = read_block(pos).first; + b = read_block(pos); EOS_ASSERT(b->block_num() == block_num, reversible_blocks_exception, "Wrong block was read from block log.", ("returned", b->block_num())("expected", block_num)); } @@ -489,13 +478,13 @@ namespace eosio { namespace chain { // Check that the file is not empty my->block_file.seek_end(0); - if (my->block_file.tellg() <= sizeof(pos)) + if (my->block_file.tellp() <= sizeof(pos)) return {}; my->block_file.seek_end(-sizeof(pos)); my->block_file.read((char*)&pos, sizeof(pos)); if (pos != npos) { - return read_block(pos).first; + return read_block(pos); } else { return {}; } diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index c0684d566a6..095cc73c511 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -46,7 +46,7 @@ namespace eosio { namespace chain { void flush(); void reset( const genesis_state& gs, const signed_block_ptr& genesis_block, uint32_t first_block_num = 1 ); - std::pair read_block(uint64_t file_pos)const; + signed_block_ptr read_block(uint64_t file_pos)const; signed_block_ptr read_block_by_num(uint32_t block_num)const; signed_block_ptr read_block_by_id(const block_id_type& id)const { return read_block_by_num(block_header::num_from_id(id)); From 0332b9d5f70a75bf5bc3dd9711a1ce538aff23fb Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 8 Aug 2019 22:43:51 -0400 Subject: [PATCH 12/21] Improve logging --- plugins/producer_plugin/producer_plugin.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index d093248f2ed..db736a5d90e 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -304,8 +304,9 @@ class producer_plugin_impl : public std::enable_shared_from_thisid(); + auto blk_num = block->block_num(); - fc_dlog(_log, "received incoming block ${id}", ("id", id)); + fc_dlog(_log, "received incoming block ${n} ${id}", ("n", blk_num)("id", id)); EOS_ASSERT( block->timestamp < (fc::time_point::now() + fc::seconds( 7 )), block_from_the_future, "received a block from the future, ignoring it: ${id}", ("id", id) ); @@ -354,11 +355,11 @@ class producer_plugin_impl : public std::enable_shared_from_thistimestamp < fc::minutes(5) || (block->block_num() % 1000 == 0) ) { + if( fc::time_point::now() - block->timestamp < fc::minutes(5) || (blk_num % 1000 == 0) ) { ilog("Received block ${id}... #${n} @ ${t} signed by ${p} [trxs: ${count}, lib: ${lib}, conf: ${confs}, latency: ${latency} ms]", - ("p",block->producer)("id",fc::variant(block->id()).as_string().substr(8,16)) - ("n",block_header::num_from_id(block->id()))("t",block->timestamp) - ("count",block->transactions.size())("lib",chain.last_irreversible_block_num())("confs", block->confirmed)("latency", (fc::time_point::now() - block->timestamp).count()/1000 ) ); + ("p",block->producer)("id",id.str().substr(8,16))("n",blk_num)("t",block->timestamp) + ("count",block->transactions.size())("lib",chain.last_irreversible_block_num()) + ("confs", block->confirmed)("latency", (fc::time_point::now() - block->timestamp).count()/1000 ) ); } } @@ -1300,7 +1301,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { if( chain.get_read_mode() == chain::db_read_mode::READ_ONLY ) return start_block_result::waiting; - fc_dlog(_log, "Starting block at ${time}", ("time", fc::time_point::now())); + fc_dlog(_log, "Starting block ${n} at ${time}", ("n", chain.head_block_num())("time", fc::time_point::now())); const auto& hbs = chain.head_block_state(); @@ -1732,9 +1733,9 @@ void producer_plugin_impl::schedule_production_loop() { [&chain,weak_this,cid=++_timer_corelation_id](const boost::system::error_code& ec) { auto self = weak_this.lock(); if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) { - fc_dlog( _log, "Produce block timer running at ${time}", ("time", fc::time_point::now()) ); // pending_block_state expected, but can't assert inside async_wait auto block_num = chain.is_building_block() ? chain.head_block_num() + 1 : 0; + fc_dlog( _log, "Produce block timer for ${num} running at ${time}", ("num", block_num)("time", fc::time_point::now()) ); auto res = self->maybe_produce_block(); fc_dlog( _log, "Producing Block #${num} returned: ${res}", ("num", block_num)( "res", res ) ); } @@ -1866,7 +1867,7 @@ void producer_plugin_impl::produce_block() { _producer_watermarks[new_bs->header.producer] = chain.head_block_num(); ilog("Produced block ${id}... #${n} @ ${t} signed by ${p} [trxs: ${count}, lib: ${lib}, confirmed: ${confs}]", - ("p",new_bs->header.producer)("id",fc::variant(new_bs->id).as_string().substr(0,16)) + ("p",new_bs->header.producer)("id",new_bs->id.str().substr(0,16)) ("n",new_bs->block_num)("t",new_bs->header.timestamp) ("count",new_bs->block->transactions.size())("lib",chain.last_irreversible_block_num())("confs", new_bs->header.confirmed)); From 5b0f6c75c0d804e341e953236bd8e92a3d9132b9 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 9 Aug 2019 08:42:16 -0400 Subject: [PATCH 13/21] Report correct block num --- plugins/producer_plugin/producer_plugin.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index db736a5d90e..e5ec7a5782d 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -1301,10 +1301,10 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { if( chain.get_read_mode() == chain::db_read_mode::READ_ONLY ) return start_block_result::waiting; - fc_dlog(_log, "Starting block ${n} at ${time}", ("n", chain.head_block_num())("time", fc::time_point::now())); - const auto& hbs = chain.head_block_state(); + fc_dlog(_log, "Starting block ${n} at ${time}", ("n", hbs->block_num + 1)("time", fc::time_point::now())); + //Schedule for the next second's tick regardless of chain state // If we would wait less than 50ms (1/10 of block_interval), wait for the whole block interval. const fc::time_point now = fc::time_point::now(); From fd3e6f942ff13c005cb1cab2dd49c5915edf3c30 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 12 Aug 2019 11:28:47 -0500 Subject: [PATCH 14/21] Move cfile to fc --- libraries/chain/block_log.cpp | 153 +++------------------------------- libraries/fc | 2 +- 2 files changed, 11 insertions(+), 144 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index a9d487d17d3..75b3934060b 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -6,13 +6,15 @@ #include #include #include +#include #include -#include -#include + #define LOG_READ (std::ios::in | std::ios::binary) #define LOG_WRITE (std::ios::out | std::ios::binary | std::ios::app) #define LOG_RW ( std::ios::in | std::ios::out | std::ios::binary ) +#define LOG_WRITE_C "ab+" +#define LOG_RW_C "rb+" namespace eosio { namespace chain { @@ -29,147 +31,12 @@ namespace eosio { namespace chain { namespace detail { using unique_file = std::unique_ptr; - class cfile_datastream; - - class cfile { - public: - explicit cfile() - : _file(nullptr, &fclose) - {} - - void set_file_path( fc::path file_path ) { - _file_path = std::move( file_path ); - } - - fc::path get_file_path() const { - return _file_path; - } - - bool is_open() const { return _open; } - - size_t file_size() const { - return fc::file_size( _file_path ); - } - - void open_for_append() { - _file.reset( fopen( _file_path.generic_string().c_str(), "ab+" ) ); - if( !_file ) { - throw std::runtime_error( "cfile unable to open: " + _file_path.generic_string() ); - } - _open = true; - } - - void open_for_rw() { - _file.reset( fopen( _file_path.generic_string().c_str(), "rb+" ) ); - if( !_file ) { - throw std::runtime_error( "cfile unable to open: " + _file_path.generic_string() ); - } - _open = true; - } - - long tellp() const { - return ftell( _file.get() ); - } - - void seek( long loc ) { - if( 0 != fseek( _file.get(), loc, SEEK_SET ) ) { - throw std::runtime_error( "cfile: " + _file_path.generic_string() + - " unable to SEEK_SET to: " + std::to_string(loc) ); - } - } - - void seek_end( long loc ) { - if( 0 != fseek( _file.get(), loc, SEEK_END ) ) { - throw std::runtime_error( "cfile: " + _file_path.generic_string() + - " unable to SEEK_END to: " + std::to_string(loc) ); - } - } - - void read( char* d, size_t n ) { - size_t result = fread( d, 1, n, _file.get() ); - if( result != n ) { - throw std::runtime_error( "cfile: " + _file_path.generic_string() + - " unable to read " + std::to_string( n ) + " only read " + std::to_string( result ) ); - } - } - - void write( const char* d, size_t n ) { - size_t result = fwrite( d, 1, n, _file.get() ); - if( result != n ) { - throw std::runtime_error( "cfile: " + _file_path.generic_string() + - " unable to write " + std::to_string( n ) + " only wrote " + std::to_string( result ) ); - } - } - - void flush() { - if( 0 != fflush( _file.get() ) ) { - throw std::runtime_error( "cfile: " + _file_path.generic_string() + " unable to flush file." ); - } - } - - void close() { - _file.reset(); - _open = false; - } - - void remove() { - if( _open ) { - throw std::runtime_error( "cfile: " + _file_path.generic_string() + " Unable to remove as file is open" ); - } - fc::remove_all( _file_path ); - } - - cfile_datastream create_datastream(); - - private: - bool _open = false; - fc::path _file_path; - unique_file _file; - }; - - /* - * @brief datastream adapter that adapts cfile for use with fc unpack - * - * This class supports unpack functionality but not pack. - */ - class cfile_datastream { - public: - explicit cfile_datastream( cfile& cf ) : cf(cf) {} - - void skip( size_t s ) { - std::vector d( s ); - read( &d[0], s ); - } - - bool read( char* d, size_t s ) { - cf.read( d, s ); - return true; - } - - bool get( unsigned char& c ) { - char cc; - cf.read(&cc, 1); - c = cc; - return true; - } - - bool get( char& c ) { return read(&c, 1); } - - private: - cfile& cf; - }; - - inline cfile_datastream cfile::create_datastream() { - return cfile_datastream(*this); - } - - class block_log_impl { public: signed_block_ptr head; block_id_type head_id; - cfile block_file; - cfile index_file; + fc::cfile block_file; + fc::cfile index_file; bool open_files = false; bool genesis_written_to_block_log = false; uint32_t version = 0; @@ -196,13 +63,13 @@ namespace eosio { namespace chain { // open to create files if they don't exist //ilog("Opening block log at ${path}", ("path", my->block_file.generic_string())); - block_file.open_for_append(); - index_file.open_for_append(); + block_file.open( LOG_WRITE_C ); + index_file.open( LOG_WRITE_C ); close(); - block_file.open_for_rw(); - index_file.open_for_rw(); + block_file.open( LOG_RW_C ); + index_file.open( LOG_RW_C ); open_files = true; } diff --git a/libraries/fc b/libraries/fc index 0732c19245f..1e5604c2b6d 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit 0732c19245fad322b8785f9d43efda2117066490 +Subproject commit 1e5604c2b6da7f04ac167634cf696dec63ab3083 From c898ca554b2a233ff073c8d8d9f2e64abaf1c02c Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 12 Aug 2019 11:31:57 -0500 Subject: [PATCH 15/21] Remove unused variable --- plugins/producer_plugin/producer_plugin.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index c482ff0682c..e77bcff1991 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -734,7 +734,6 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ { try { my->chain_plug = app().find_plugin(); EOS_ASSERT( my->chain_plug, plugin_config_exception, "chain_plugin not found" ); - chain_plugin* chain_plug = my->chain_plug; my->_options = &options; LOAD_VALUE_SET(options, "producer-name", my->_producers) From ac2b96fb67a0c28378118091475f4dd1f387913c Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 12 Aug 2019 11:57:12 -0500 Subject: [PATCH 16/21] Add similar fix as #7640 (untested) --- libraries/chain/block_log.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 75b3934060b..0271376442c 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -16,6 +16,14 @@ #define LOG_WRITE_C "ab+" #define LOG_RW_C "rb+" +#ifndef _WIN32 +#define FC_FOPEN(p, m) fopen(p, m) +#else +#define FC_CAT(s1, s2) s1 ## s2 +#define FC_PREL(s) FC_CAT(L, s) +#define FC_FOPEN(p, m) _wfopen(p, FC_PREL(m)) +#endif + namespace eosio { namespace chain { const uint32_t block_log::min_supported_version = 1; @@ -600,7 +608,7 @@ namespace eosio { namespace chain { uint32_t detail::reverse_iterator::open(const fc::path& block_file_name) { _block_file_name = block_file_name.generic_string(); - _file.reset(fopen(_block_file_name.c_str(), "r")); + _file.reset( FC_FOPEN(_block_file_name.c_str(), "r")); EOS_ASSERT( _file, block_log_exception, "Could not open Block log file at '${blocks_log}'", ("blocks_log", _block_file_name) ); _end_of_buffer_position = _unset_position; @@ -728,7 +736,7 @@ namespace eosio { namespace chain { void detail::index_writer::prepare_buffer() { if (_file == nullptr) { - _file.reset(fopen(_block_index_name.c_str(), "w")); + _file.reset(FC_FOPEN(_block_index_name.c_str(), "w")); EOS_ASSERT( _file, block_log_exception, "Could not open Block index file at '${blocks_index}'", ("blocks_index", _block_index_name) ); // allocate 8 bytes for each block position to store const auto full_file_size = buffer_location_to_file_location(_blocks_expected); From 9fb0e77f1ccb4ddc7298b72fe472228388cab63d Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 12 Aug 2019 14:33:24 -0500 Subject: [PATCH 17/21] Removed remove() and file_size() from cfile --- libraries/chain/block_log.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 0271376442c..d18162294c2 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -163,6 +163,9 @@ namespace eosio { namespace chain { my->block_file.set_file_path( data_dir / "blocks.log" ); my->index_file.set_file_path( data_dir / "blocks.index" ); + auto log_size = fc::file_size( my->block_file.get_file_path() ); + auto index_size = fc::file_size( my->index_file.get_file_path() ); + my->reopen(); /* On startup of the block log, there are several states the log file and the index file can be @@ -183,8 +186,6 @@ namespace eosio { namespace chain { * - If the index file head is not in the log file, delete the index and replay. * - If the index file head is in the log, but not up to date, replay from index head. */ - auto log_size = my->block_file.file_size(); - auto index_size = my->index_file.file_size(); if (log_size) { ilog("Log is nonempty"); @@ -237,7 +238,7 @@ namespace eosio { namespace chain { } else if (index_size) { ilog("Index is nonempty, remove and recreate it"); my->close(); - my->index_file.remove(); + fc::remove_all( my->index_file.get_file_path() ); my->reopen(); } } @@ -278,8 +279,8 @@ namespace eosio { namespace chain { void block_log::reset( const genesis_state& gs, const signed_block_ptr& first_block, uint32_t first_block_num ) { my->close(); - my->block_file.remove(); - my->index_file.remove(); + fc::remove_all( my->block_file.get_file_path() ); + fc::remove_all( my->index_file.get_file_path() ); my->reopen(); @@ -377,7 +378,7 @@ namespace eosio { namespace chain { ilog("Reconstructing Block Log Index..."); my->close(); - my->index_file.remove(); + fc::remove_all( my->index_file.get_file_path() ); my->reopen(); From 411f7d3c554e29aa91cc71a7075fd206e13739e1 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 13 Aug 2019 13:23:30 -0500 Subject: [PATCH 18/21] Update to fc with cfile --- libraries/fc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/fc b/libraries/fc index 1e5604c2b6d..ff8686c4a0a 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit 1e5604c2b6da7f04ac167634cf696dec63ab3083 +Subproject commit ff8686c4a0a6c32b6a6318fc2b280b9f6f143c16 From 3b12f1b6f1b1d876470eca0ba2bcea09dbde13d5 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 13 Aug 2019 14:50:13 -0500 Subject: [PATCH 19/21] Call file_size after reopen otherwise file may not exist --- libraries/chain/block_log.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index d18162294c2..2a8fb7d2d6c 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -163,9 +163,6 @@ namespace eosio { namespace chain { my->block_file.set_file_path( data_dir / "blocks.log" ); my->index_file.set_file_path( data_dir / "blocks.index" ); - auto log_size = fc::file_size( my->block_file.get_file_path() ); - auto index_size = fc::file_size( my->index_file.get_file_path() ); - my->reopen(); /* On startup of the block log, there are several states the log file and the index file can be @@ -186,6 +183,8 @@ namespace eosio { namespace chain { * - If the index file head is not in the log file, delete the index and replay. * - If the index file head is in the log, but not up to date, replay from index head. */ + auto log_size = fc::file_size( my->block_file.get_file_path() ); + auto index_size = fc::file_size( my->index_file.get_file_path() ); if (log_size) { ilog("Log is nonempty"); From 7c90551ce27772794998a2002d6515773d2c457e Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 14 Aug 2019 07:10:57 -0500 Subject: [PATCH 20/21] Better assert message. Move datastream construction cloer to use --- plugins/net_plugin/net_plugin.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 35906c776b7..740a9a28724 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -756,24 +756,22 @@ namespace eosio { explicit msg_handler( const connection_ptr& conn) : c(conn) {} void operator()( const signed_block& msg ) const { - EOS_ASSERT( false, plugin_config_exception, "operator()(signed_block&&) should be called" ); + EOS_ASSERT( false, plugin_config_exception, "Not implemented, call handle_message directly instead" ); } void operator()( signed_block& msg ) const { - EOS_ASSERT( false, plugin_config_exception, "operator()(signed_block&&) should be called" ); + EOS_ASSERT( false, plugin_config_exception, "Not implemented, call handle_message directly instead" ); } void operator()( const packed_transaction& msg ) const { - EOS_ASSERT( false, plugin_config_exception, "operator()(packed_transaction&&) should be called" ); + EOS_ASSERT( false, plugin_config_exception, "Not implemented, call handle_message directly instead" ); } void operator()( packed_transaction& msg ) const { - EOS_ASSERT( false, plugin_config_exception, "operator()(packed_transaction&&) should be called" ); + EOS_ASSERT( false, plugin_config_exception, "Not implemented, call handle_message directly instead" ); } - void operator()( signed_block&& msg ) const { - EOS_ASSERT( false, plugin_config_exception, "operator()(signed_block&&) should call handle_message" ); + EOS_ASSERT( false, plugin_config_exception, "Not implemented, call handle_message directly instead" ); } - void operator()( packed_transaction&& msg ) const { - EOS_ASSERT( false, plugin_config_exception, "operator()(packed_transaction&&) should call handle_message" ); + EOS_ASSERT( false, plugin_config_exception, "Not implemented, call handle_message directly instead" ); } void operator()( const handshake_message& msg ) const { @@ -2394,7 +2392,6 @@ namespace eosio { bool connection::process_next_message( uint32_t message_length ) { try { // if next message is a block we already have, exit early - auto ds = pending_message_buffer.create_datastream(); auto peek_ds = pending_message_buffer.create_peek_datastream(); unsigned_int which{}; fc::raw::unpack( peek_ds, which ); @@ -2413,18 +2410,21 @@ namespace eosio { return true; } + auto ds = pending_message_buffer.create_datastream(); fc::raw::unpack( ds, which ); // throw away shared_ptr ptr = std::make_shared(); fc::raw::unpack( ds, *ptr ); handle_message( blk_id, std::move( ptr ) ); } else if( which == packed_transaction_which ) { + auto ds = pending_message_buffer.create_datastream(); fc::raw::unpack( ds, which ); // throw away shared_ptr ptr = std::make_shared(); fc::raw::unpack( ds, *ptr ); handle_message( std::move( ptr ) ); } else { + auto ds = pending_message_buffer.create_datastream(); net_message msg; fc::raw::unpack( ds, msg ); msg_handler m( shared_from_this() ); From 40a8be81c082c07038551b67da5adc23636bccd0 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 14 Aug 2019 09:33:21 -0500 Subject: [PATCH 21/21] Remove duplicate code via templated operator() --- plugins/net_plugin/net_plugin.cpp | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 740a9a28724..d81f5bc35a7 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -755,22 +755,8 @@ namespace eosio { connection_ptr c; explicit msg_handler( const connection_ptr& conn) : c(conn) {} - void operator()( const signed_block& msg ) const { - EOS_ASSERT( false, plugin_config_exception, "Not implemented, call handle_message directly instead" ); - } - void operator()( signed_block& msg ) const { - EOS_ASSERT( false, plugin_config_exception, "Not implemented, call handle_message directly instead" ); - } - void operator()( const packed_transaction& msg ) const { - EOS_ASSERT( false, plugin_config_exception, "Not implemented, call handle_message directly instead" ); - } - void operator()( packed_transaction& msg ) const { - EOS_ASSERT( false, plugin_config_exception, "Not implemented, call handle_message directly instead" ); - } - void operator()( signed_block&& msg ) const { - EOS_ASSERT( false, plugin_config_exception, "Not implemented, call handle_message directly instead" ); - } - void operator()( packed_transaction&& msg ) const { + template + void operator()( const T& ) const { EOS_ASSERT( false, plugin_config_exception, "Not implemented, call handle_message directly instead" ); }