From a7fc838b09ee05240ba05a1001f914697b910ca1 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 5 Feb 2019 17:44:01 -0500 Subject: [PATCH 01/17] Consolidated Security Fixes for 1.6.1 - net_plugin security fixes - Additional checktime calls to limit cpu usage - Limit memory usage in producer_plugin Co-Authored-By: Kevin Heifner Co-Authored-By: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Co-authored-by: Kayan --- libraries/chain/controller.cpp | 41 +-- .../chain/include/eosio/chain/controller.hpp | 18 +- .../eosio/chain/wasm_eosio_injection.hpp | 11 +- libraries/chain/wasm_eosio_injection.cpp | 2 +- .../testing/include/eosio/testing/tester.hpp | 11 + libraries/testing/tester.cpp | 21 +- plugins/net_plugin/net_plugin.cpp | 316 ++++++++++++------ plugins/producer_plugin/producer_plugin.cpp | 275 ++++++++------- unittests/api_tests.cpp | 4 +- unittests/delay_tests.cpp | 2 +- 10 files changed, 419 insertions(+), 282 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 98b2065d1e1..2d2af25f0c1 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -143,7 +143,7 @@ struct controller_impl { * are removed from this list if they are re-applied in other blocks. Producers * can query this list when scheduling new transactions into blocks. */ - map unapplied_transactions; + unapplied_transactions_type unapplied_transactions; void pop_block() { auto prev = fork_db.get_block( head->header.previous ); @@ -2106,41 +2106,12 @@ const account_object& controller::get_account( account_name name )const return my->db.get(name); } FC_CAPTURE_AND_RETHROW( (name) ) } -vector controller::get_unapplied_transactions() const { - vector result; - if ( my->read_mode == db_read_mode::SPECULATIVE ) { - result.reserve(my->unapplied_transactions.size()); - for ( const auto& entry: my->unapplied_transactions ) { - result.emplace_back(entry.second); - } - } else { - EOS_ASSERT( my->unapplied_transactions.empty(), transaction_exception, "not empty unapplied_transactions in non-speculative mode" ); //should never happen - } - return result; -} - -void controller::drop_unapplied_transaction(const transaction_metadata_ptr& trx) { - my->unapplied_transactions.erase(trx->signed_id); -} - -void controller::drop_all_unapplied_transactions() { - my->unapplied_transactions.clear(); -} - -vector controller::get_scheduled_transactions() const { - const auto& idx = db().get_index(); - - vector result; - - static const size_t max_reserve = 64; - result.reserve(std::min(idx.size(), max_reserve)); - - auto itr = idx.begin(); - while( itr != idx.end() && itr->delay_until <= pending_block_time() ) { - result.emplace_back(itr->trx_id); - ++itr; +unapplied_transactions_type& controller::get_unapplied_transactions() { + if ( my->read_mode != db_read_mode::SPECULATIVE ) { + EOS_ASSERT( my->unapplied_transactions.empty(), transaction_exception, + "not empty unapplied_transactions in non-speculative mode" ); //should never happen } - return result; + return my->unapplied_transactions; } bool controller::sender_avoids_whitelist_blacklist_enforcement( account_name sender )const { diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index f0d5a53f52d..9e6947fdff9 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -33,6 +33,7 @@ namespace eosio { namespace chain { class account_object; using resource_limits::resource_limits_manager; using apply_handler = std::function; + using unapplied_transactions_type = map; class fork_database; @@ -111,22 +112,9 @@ namespace eosio { namespace chain { * The caller is responsible for calling drop_unapplied_transaction on a failing transaction that * they never intend to retry * - * @return vector of transactions which have been unapplied + * @return map of transactions which have been unapplied */ - vector get_unapplied_transactions() const; - void drop_unapplied_transaction(const transaction_metadata_ptr& trx); - void drop_all_unapplied_transactions(); - - /** - * These transaction IDs represent transactions available in the head chain state as scheduled - * or otherwise generated transactions. - * - * calling push_scheduled_transaction with these IDs will remove the associated transaction from - * the chain state IFF it succeeds or objectively fails - * - * @return - */ - vector get_scheduled_transactions() const; + unapplied_transactions_type& get_unapplied_transactions(); /** * diff --git a/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp b/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp index f5ebf01c1f7..a67ca9ef696 100644 --- a/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp +++ b/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp @@ -272,7 +272,7 @@ namespace eosio { namespace chain { namespace wasm_injections { }; - struct call_depth_check { + struct call_depth_check_and_insert_checktime { static constexpr bool kills = true; static constexpr bool post = false; static int32_t global_idx; @@ -290,6 +290,7 @@ namespace eosio { namespace chain { namespace wasm_injections { injector_utils::add_import(*(arg.module), "call_depth_assert", assert_idx); wasm_ops::op_types<>::call_t call_assert; + wasm_ops::op_types<>::call_t call_checktime; wasm_ops::op_types<>::get_global_t get_global_inst; wasm_ops::op_types<>::set_global_t set_global_inst; @@ -301,6 +302,7 @@ namespace eosio { namespace chain { namespace wasm_injections { wasm_ops::op_types<>::else__t else_inst; call_assert.field = assert_idx; + call_checktime.field = checktime_injection::chktm_idx; get_global_inst.field = global_idx; set_global_inst.field = global_idx; const_inst.field = -1; @@ -334,6 +336,7 @@ namespace eosio { namespace chain { namespace wasm_injections { INSERT_INJECTED(const_inst); INSERT_INJECTED(add_inst); INSERT_INJECTED(set_global_inst); + INSERT_INJECTED(call_checktime); #undef INSERT_INJECTED } @@ -679,8 +682,8 @@ namespace eosio { namespace chain { namespace wasm_injections { }; struct pre_op_injectors : wasm_ops::op_types { - using call_t = wasm_ops::call ; - using call_indirect_t = wasm_ops::call_indirect ; + using call_t = wasm_ops::call ; + using call_indirect_t = wasm_ops::call_indirect ; // float binops using f32_add_t = wasm_ops::f32_add >; @@ -785,7 +788,7 @@ namespace eosio { namespace chain { namespace wasm_injections { // initialize static fields of injectors injector_utils::init( mod ); checktime_injection::init(); - call_depth_check::init(); + call_depth_check_and_insert_checktime::init(); } void inject() { diff --git a/libraries/chain/wasm_eosio_injection.cpp b/libraries/chain/wasm_eosio_injection.cpp index a4afa44d46d..2c627e13ea7 100644 --- a/libraries/chain/wasm_eosio_injection.cpp +++ b/libraries/chain/wasm_eosio_injection.cpp @@ -35,7 +35,7 @@ void max_memory_injection_visitor::inject( Module& m ) { } void max_memory_injection_visitor::initializer() {} -int32_t call_depth_check::global_idx = -1; +int32_t call_depth_check_and_insert_checktime::global_idx = -1; uint32_t instruction_counter::icnt = 0; uint32_t instruction_counter::tcnt = 0; uint32_t instruction_counter::bcnt = 0; diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index 10e7d4499e2..c9a4591d6c9 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -99,6 +99,17 @@ namespace eosio { namespace testing { void produce_min_num_of_blocks_to_spend_time_wo_inactive_prod(const fc::microseconds target_elapsed_time = fc::microseconds()); signed_block_ptr push_block(signed_block_ptr b); + /** + * These transaction IDs represent transactions available in the head chain state as scheduled + * or otherwise generated transactions. + * + * calling push_scheduled_transaction with these IDs will remove the associated transaction from + * the chain state IFF it succeeds or objectively fails + * + * @return + */ + vector get_scheduled_transactions() const; + transaction_trace_ptr push_transaction( packed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US ); transaction_trace_ptr push_transaction( signed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US ); action_result push_action(action&& cert_act, uint64_t authorizer); // TODO/QUESTION: Is this needed? diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index bcf811434d5..a632cb40643 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -163,16 +164,16 @@ namespace eosio { namespace testing { } if( !skip_pending_trxs ) { - auto unapplied_trxs = control->get_unapplied_transactions(); - for (const auto& trx : unapplied_trxs ) { - auto trace = control->push_transaction(trx, fc::time_point::maximum()); + unapplied_transactions_type unapplied_trxs = control->get_unapplied_transactions(); // make copy of map + for (const auto& entry : unapplied_trxs ) { + auto trace = control->push_transaction(entry.second, fc::time_point::maximum()); if(trace->except) { trace->except->dynamic_rethrow_exception(); } } vector scheduled_trxs; - while( (scheduled_trxs = control->get_scheduled_transactions() ).size() > 0 ) { + while( (scheduled_trxs = get_scheduled_transactions() ).size() > 0 ) { for (const auto& trx : scheduled_trxs ) { auto trace = control->push_scheduled_transaction(trx, fc::time_point::maximum()); if(trace->except) { @@ -237,6 +238,18 @@ namespace eosio { namespace testing { } } + vector base_tester::get_scheduled_transactions() const { + const auto& idx = control->db().get_index(); + + vector result; + + auto itr = idx.begin(); + while( itr != idx.end() && itr->delay_until <= control->pending_block_time() ) { + result.emplace_back(itr->trx_id); + ++itr; + } + return result; + } void base_tester::produce_blocks_until_end_of_round() { uint64_t blocks_per_round; diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 0c00495c48d..1b398a8b53a 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -283,6 +283,10 @@ namespace eosio { */ constexpr auto def_send_buffer_size_mb = 4; constexpr auto def_send_buffer_size = 1024*1024*def_send_buffer_size_mb; + constexpr auto def_max_write_queue_size = def_send_buffer_size*10; + constexpr boost::asio::chrono::milliseconds def_read_delay_for_full_write_queue{100}; + constexpr auto def_max_reads_in_flight = 1000; + constexpr auto def_max_trx_in_progress_size = 100*1024*1024; // 100 MB constexpr auto def_max_clients = 25; // 0 for unlimited clients constexpr auto def_max_nodes_per_host = 1; constexpr auto def_conn_retry_wait = 30; @@ -400,6 +404,86 @@ namespace eosio { static void populate(handshake_message &hello); }; + class queued_buffer : boost::noncopyable { + public: + void clear_write_queue() { + _write_queue.clear(); + _sync_write_queue.clear(); + _write_queue_size = 0; + } + + void clear_out_queue() { + while ( _out_queue.size() > 0 ) { + _out_queue.pop_front(); + } + } + + uint32_t write_queue_size() const { return _write_queue_size; } + + bool is_out_queue_empty() const { return _out_queue.empty(); } + + bool ready_to_send() const { + // if out_queue is not empty then async_write is in progress + return ((!_sync_write_queue.empty() || !_write_queue.empty()) && _out_queue.empty()); + } + + bool add_write_queue( const std::shared_ptr>& buff, + std::function callback, + bool to_sync_queue ) { + if( to_sync_queue ) { + _sync_write_queue.push_back( {buff, callback} ); + } else { + _write_queue.push_back( {buff, callback} ); + } + _write_queue_size += buff->size(); + if( _write_queue_size > 2 * def_max_write_queue_size ) { + return false; + } + return true; + } + + void fill_out_buffer( std::vector& bufs ) { + if( _sync_write_queue.size() > 0 ) { // always send msgs from sync_write_queue first + fill_out_buffer( bufs, _sync_write_queue ); + } else { // postpone real_time write_queue if sync queue is not empty + fill_out_buffer( bufs, _write_queue ); + EOS_ASSERT( _write_queue_size == 0, plugin_exception, "write queue size expected to be zero" ); + } + } + + void out_callback( boost::system::error_code ec, std::size_t w ) { + for( auto& m : _out_queue ) { + m.callback( ec, w ); + } + } + + private: + struct queued_write; + void fill_out_buffer( std::vector& bufs, + deque& w_queue ) { + while ( w_queue.size() > 0 ) { + auto& m = w_queue.front(); + bufs.push_back( boost::asio::buffer( *m.buff )); + _write_queue_size -= m.buff->size(); + _out_queue.emplace_back( m ); + w_queue.pop_front(); + } + } + + private: + struct queued_write { + std::shared_ptr> buff; + std::function callback; + }; + + uint32_t _write_queue_size = 0; + deque _write_queue; + deque _sync_write_queue; // sync_write_queue will be sent first + deque _out_queue; + + }; // queued_buffer + + class connection : public std::enable_shared_from_this { public: explicit connection( string endpoint ); @@ -416,12 +500,11 @@ namespace eosio { fc::message_buffer<1024*1024> pending_message_buffer; fc::optional outstanding_read_bytes; - struct queued_write { - std::shared_ptr> buff; - std::function callback; - }; - deque write_queue; - deque out_queue; + + queued_buffer buffer_queue; + + uint32_t reads_in_flight = 0; + uint32_t trx_in_progress_size = 0; fc::sha256 node_id; handshake_message last_handshake_recv; handshake_message last_handshake_sent; @@ -431,6 +514,7 @@ namespace eosio { uint16_t protocol_version = 0; string peer_addr; unique_ptr response_expected; + unique_ptr read_delay_timer; optional pending_fetch; go_away_reason no_retry = no_reason; block_id_type fork_head; @@ -497,12 +581,14 @@ namespace eosio { void txn_send(const vector& txn_lis); void blk_send_branch(); - void blk_send(const vector &txn_lis); + void blk_send(const block_id_type& blkid); void stop_send(); void enqueue( const net_message &msg, bool trigger_send = true ); - void enqueue_block( const signed_block_ptr& sb, bool trigger_send = true ); - void enqueue_buffer( const std::shared_ptr>& send_buffer, bool trigger_send, go_away_reason close_after_send ); + void enqueue_block( const signed_block_ptr& sb, bool trigger_send = true, bool to_sync_queue = false ); + void enqueue_buffer( const std::shared_ptr>& send_buffer, + bool trigger_send, go_away_reason close_after_send, + bool to_sync_queue = false); void cancel_sync(go_away_reason); void flush_queues(); bool enqueue_sync_block(); @@ -516,7 +602,8 @@ namespace eosio { void queue_write(const std::shared_ptr>& buff, bool trigger_send, - std::function callback); + std::function callback, + bool to_sync_queue = false); void do_queue_write(); /** \brief Process the next message from the pending message buffer @@ -530,7 +617,7 @@ namespace eosio { */ bool process_next_message(net_plugin_impl& impl, uint32_t message_length); - bool add_peer_block(const peer_block_state &pbs); + bool add_peer_block(const peer_block_state& pbs); fc::optional _logger_variant; const fc::variant_object& get_logger_variant() { @@ -659,6 +746,7 @@ namespace eosio { protocol_version(0), peer_addr(endpoint), response_expected(), + read_delay_timer(), pending_fetch(), no_retry(no_reason), fork_head(), @@ -683,6 +771,7 @@ namespace eosio { protocol_version(0), peer_addr(), response_expected(), + read_delay_timer(), pending_fetch(), no_retry(no_reason), fork_head(), @@ -699,6 +788,7 @@ namespace eosio { auto *rnd = node_id.data(); rnd[0] = 0; response_expected.reset(new boost::asio::steady_timer(app().get_io_service())); + read_delay_timer.reset(new boost::asio::steady_timer(app().get_io_service())); } bool connection::connected() { @@ -716,7 +806,7 @@ namespace eosio { } void connection::flush_queues() { - write_queue.clear(); + buffer_queue.clear_write_queue(); } void connection::close() { @@ -739,6 +829,7 @@ namespace eosio { my_impl->sync_master->reset_lib_num(shared_from_this()); fc_dlog(logger, "canceling wait on ${p}", ("p",peer_name())); cancel_wait(); + if( read_delay_timer ) read_delay_timer->cancel(); pending_message_buffer.reset(); } @@ -803,75 +894,43 @@ namespace eosio { catch (...) { } - vector bstack; - block_id_type null_id; - for (auto bid = head_id; bid != null_id && bid != lib_id; ) { - try { - - // if the last handshake received indicates that we are catching up on a fork - // that the peer is already partially aware of, no need to resend blocks - if (remote_head_id == bid) { - break; - } - - signed_block_ptr b = cc.fetch_block_by_id(bid); - if ( b ) { - bid = b->previous; - bstack.push_back(b); - } - else { - break; - } - } catch (...) { - break; - } - } - size_t count = 0; - if (!bstack.empty()) { - if (bstack.back()->previous == lib_id || bstack.back()->previous == remote_head_id) { - count = bstack.size(); - while (bstack.size()) { - enqueue_block( bstack.back() ); - bstack.pop_back(); - } - } - fc_ilog(logger, "Sent ${n} blocks on my fork",("n",count)); + if( !peer_requested ) { + peer_requested = sync_state( block_header::num_from_id(lib_id)+1, + block_header::num_from_id(head_id), + block_header::num_from_id(lib_id) ); } else { - fc_ilog(logger, "Nothing to send on fork request"); + uint32_t start = std::min( peer_requested->last + 1, block_header::num_from_id(lib_id)+1 ); + uint32_t end = std::max( peer_requested->end_block, block_header::num_from_id(head_id) ); + peer_requested = sync_state( start, end, start - 1 ); } + enqueue_sync_block(); + // still want to send transactions along during blk branch sync syncing = false; } - void connection::blk_send(const vector &ids) { + void connection::blk_send(const block_id_type& blkid) { controller &cc = my_impl->chain_plug->chain(); - int count = 0; - for(auto &blkid : ids) { - ++count; - try { - signed_block_ptr b = cc.fetch_block_by_id(blkid); - if(b) { - fc_dlog(logger,"found block for id at num ${n}",("n",b->block_num())); - enqueue_block( b ); - } - else { - ilog("fetch block by id returned null, id ${id} on block ${c} of ${s} for ${p}", - ("id",blkid)("c",count)("s",ids.size())("p",peer_name())); - break; - } - } - catch (const assert_exception &ex) { - elog( "caught assert on fetch_block_by_id, ${ex}, id ${id} on block ${c} of ${s} for ${p}", - ("ex",ex.to_string())("id",blkid)("c",count)("s",ids.size())("p",peer_name())); - break; - } - catch (...) { - elog( "caught othser exception fetching block id ${id} on block ${c} of ${s} for ${p}", - ("id",blkid)("c",count)("s",ids.size())("p",peer_name())); - break; + try { + signed_block_ptr b = cc.fetch_block_by_id(blkid); + if(b) { + fc_dlog(logger,"found block for id at num ${n}",("n",b->block_num())); + peer_block_state pbstate = {blkid, block_header::num_from_id(blkid), true, true, time_point()}; + add_peer_block(pbstate); + enqueue_block( b ); + } else { + ilog("fetch block by id returned null, id ${id} for ${p}", + ("id",blkid)("p",peer_name())); } } - + catch (const assert_exception &ex) { + elog( "caught assert on fetch_block_by_id, ${ex}, id ${id} for ${p}", + ("ex",ex.to_string())("id",blkid)("p",peer_name())); + } + catch (...) { + elog( "caught other exception fetching block id ${id} for ${p}", + ("id",blkid)("p",peer_name())); + } } void connection::stop_send() { @@ -905,14 +964,21 @@ namespace eosio { void connection::queue_write(const std::shared_ptr>& buff, bool trigger_send, - std::function callback) { - write_queue.push_back({buff, callback}); - if(out_queue.empty() && trigger_send) + std::function callback, + bool to_sync_queue) { + if( !buffer_queue.add_write_queue( buff, callback, to_sync_queue )) { + fc_wlog( logger, "write_queue full ${s} bytes, giving up on connection ${p}", + ("s", buffer_queue.write_queue_size())("p", peer_name()) ); + my_impl->close( shared_from_this() ); + return; + } + if( buffer_queue.is_out_queue_empty() && trigger_send) { do_queue_write(); + } } void connection::do_queue_write() { - if(write_queue.empty() || !out_queue.empty()) + if( !buffer_queue.ready_to_send() ) return; connection_wptr c(shared_from_this()); if(!socket->is_open()) { @@ -921,21 +987,14 @@ namespace eosio { return; } std::vector bufs; - while (write_queue.size() > 0) { - auto& m = write_queue.front(); - bufs.push_back(boost::asio::buffer(*m.buff)); - out_queue.push_back(m); - write_queue.pop_front(); - } + buffer_queue.fill_out_buffer( bufs ); boost::asio::async_write(*socket, bufs, [c](boost::system::error_code ec, std::size_t w) { try { auto conn = c.lock(); if(!conn) return; - for (auto& m: conn->out_queue) { - m.callback(ec, w); - } + conn->buffer_queue.out_callback( ec, w ); if(ec) { string pname = conn ? conn->peer_name() : "no connection name"; @@ -948,9 +1007,7 @@ namespace eosio { my_impl->close(conn); return; } - while (conn->out_queue.size() > 0) { - conn->out_queue.pop_front(); - } + conn->buffer_queue.clear_out_queue(); conn->enqueue_sync_block(); conn->do_queue_write(); } @@ -973,8 +1030,8 @@ namespace eosio { } void connection::cancel_sync(go_away_reason reason) { - fc_dlog(logger,"cancel sync reason = ${m}, write queue size ${o} peer ${p}", - ("m",reason_str(reason)) ("o", write_queue.size())("p", peer_name())); + fc_dlog(logger,"cancel sync reason = ${m}, write queue size ${o} bytes peer ${p}", + ("m",reason_str(reason)) ("o", buffer_queue.write_queue_size())("p", peer_name())); cancel_wait(); flush_queues(); switch (reason) { @@ -1002,7 +1059,7 @@ namespace eosio { controller& cc = my_impl->chain_plug->chain(); signed_block_ptr sb = cc.fetch_block_by_number(num); if(sb) { - enqueue_block( sb, trigger_send); + enqueue_block( sb, trigger_send, true); return true; } } catch ( ... ) { @@ -1031,7 +1088,7 @@ namespace eosio { enqueue_buffer( send_buffer, trigger_send, close_after_send ); } - void connection::enqueue_block( const signed_block_ptr& sb, bool trigger_send ) { + void connection::enqueue_block( const signed_block_ptr& sb, bool trigger_send, bool to_sync_queue ) { // this implementation is to avoid copy of signed_block to net_message int which = 7; // matches which of net_message for signed_block @@ -1048,10 +1105,13 @@ namespace eosio { fc::raw::pack( ds, unsigned_int( which )); fc::raw::pack( ds, *sb ); - enqueue_buffer( send_buffer, trigger_send, no_reason ); + enqueue_buffer( send_buffer, trigger_send, no_reason, to_sync_queue ); } - void connection::enqueue_buffer( const std::shared_ptr>& send_buffer, bool trigger_send, go_away_reason close_after_send ) { + void connection::enqueue_buffer( const std::shared_ptr>& send_buffer, bool trigger_send, + go_away_reason close_after_send, + bool to_sync_queue ) + { connection_wptr weak_this = shared_from_this(); queue_write(send_buffer,trigger_send, [weak_this, close_after_send](boost::system::error_code ec, std::size_t ) { @@ -1065,7 +1125,8 @@ namespace eosio { } else { fc_wlog(logger, "connection expired before enqueued net_message called callback!"); } - }); + }, + to_sync_queue); } void connection::cancel_wait() { @@ -1165,7 +1226,7 @@ namespace eosio { return true; } - bool connection::add_peer_block(const peer_block_state &entry) { + bool connection::add_peer_block(const peer_block_state& entry) { auto bptr = blk_state.get().find(entry.id); bool added = (bptr == blk_state.end()); if (added){ @@ -1476,11 +1537,15 @@ namespace eosio { void sync_manager::recv_notice(const connection_ptr& c, const notice_message& msg) { fc_ilog(logger, "sync_manager got ${m} block notice",("m",modes_str(msg.known_blocks.mode))); + if( msg.known_blocks.ids.size() > 1 ) { + elog( "Invalid notice_message, known_blocks.ids.size ${s}", ("s", msg.known_blocks.ids.size()) ); + my_impl->close(c); + return; + } if (msg.known_blocks.mode == catch_up) { if (msg.known_blocks.ids.size() == 0) { elog("got a catch up with ids size = 0"); - } - else { + } else { verify_catchup(c, msg.known_blocks.pending, msg.known_blocks.ids.back()); } } @@ -1685,7 +1750,9 @@ namespace eosio { if (msg.known_blocks.mode == normal) { req.req_blocks.mode = normal; controller& cc = my_impl->chain_plug->chain(); - for( const auto& blkid : msg.known_blocks.ids) { + // known_blocks.ids is never > 1 + if( !msg.known_blocks.ids.empty() ) { + const block_id_type& blkid = msg.known_blocks.ids.back(); signed_block_ptr b; peer_block_state entry = {blkid,0,true,true,fc::time_point()}; try { @@ -1952,6 +2019,37 @@ namespace eosio { } }; + if( conn->buffer_queue.write_queue_size() > def_max_write_queue_size || + conn->reads_in_flight > def_max_reads_in_flight || + conn->trx_in_progress_size > def_max_trx_in_progress_size ) + { + // too much queued up, reschedule + if( conn->buffer_queue.write_queue_size() > def_max_write_queue_size ) { + peer_wlog( conn, "write_queue full ${s} bytes", ("s", conn->buffer_queue.write_queue_size()) ); + } else if( conn->reads_in_flight > def_max_reads_in_flight ) { + peer_wlog( conn, "max reads in flight ${s}", ("s", conn->reads_in_flight) ); + } else { + peer_wlog( conn, "max trx in progress ${s} bytes", ("s", conn->trx_in_progress_size) ); + } + if( conn->buffer_queue.write_queue_size() > 2*def_max_write_queue_size || + conn->reads_in_flight > 2*def_max_reads_in_flight || + conn->trx_in_progress_size > 2*def_max_trx_in_progress_size ) + { + fc_wlog( logger, "queues over full, giving up on connection ${p}", ("p", conn->peer_name()) ); + my_impl->close( conn ); + return; + } + if( !conn->read_delay_timer ) return; + conn->read_delay_timer->expires_from_now( def_read_delay_for_full_write_queue ); + conn->read_delay_timer->async_wait([this, weak_conn]( boost::system::error_code ) { + auto conn = weak_conn.lock(); + if( !conn ) return; + start_read_message( conn ); + } ); + return; + } + + ++conn->reads_in_flight; boost::asio::async_read(*conn->socket, conn->pending_message_buffer.get_buffer_sequence_for_boost_async_read(), completion_handler, [this,weak_conn]( boost::system::error_code ec, std::size_t bytes_transferred ) { @@ -1960,6 +2058,7 @@ namespace eosio { return; } + --conn->reads_in_flight; conn->outstanding_read_bytes.reset(); try { @@ -2308,6 +2407,12 @@ namespace eosio { } void net_plugin_impl::handle_message(const connection_ptr& c, const request_message& msg) { + if( msg.req_blocks.ids.size() > 1 ) { + elog( "Invalid request_message, req_blocks.ids.size ${s}", ("s", msg.req_blocks.ids.size()) ); + close(c); + return; + } + switch (msg.req_blocks.mode) { case catch_up : peer_ilog(c, "received request_message:catch_up"); @@ -2315,7 +2420,9 @@ namespace eosio { break; case normal : peer_ilog(c, "received request_message:normal"); - c->blk_send(msg.req_blocks.ids); + if( !msg.req_blocks.ids.empty() ) { + c->blk_send(msg.req_blocks.ids.back()); + } break; default:; } @@ -2347,6 +2454,13 @@ namespace eosio { } } + size_t calc_trx_size( const packed_transaction_ptr& trx ) { + // transaction is stored packed and unpacked, double packed_size and size of signed as an approximation of use + return (trx->get_packed_transaction().size() * 2 + sizeof(trx->get_signed_transaction())) * 2 + + trx->get_packed_context_free_data().size() * 4 + + trx->get_signatures().size() * sizeof(signature_type); + } + void net_plugin_impl::handle_message(const connection_ptr& c, const packed_transaction_ptr& trx) { fc_dlog(logger, "got a packed transaction, cancel wait"); peer_ilog(c, "received packed_transaction"); @@ -2369,7 +2483,9 @@ namespace eosio { return; } dispatcher->recv_transaction(c, tid); + c->trx_in_progress_size += calc_trx_size( ptrx->packed_trx ); chain_plug->accept_transaction(ptrx, [c, this, ptrx](const static_variant& result) { + c->trx_in_progress_size -= calc_trx_size( ptrx->packed_trx ); if (result.contains()) { peer_dlog(c, "bad packed_transaction : ${m}", ("m",result.get()->what())); } else { diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 71583aee0c5..947ef48f46a 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -1143,6 +1144,10 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { int orig_count = _persistent_transactions.size(); while(!persisted_by_expiry.empty() && persisted_by_expiry.begin()->expiry <= pbs->header.timestamp.to_time_point()) { + if (preprocess_deadline <= fc::time_point::now()) { + exhausted = true; + break; + } auto const& txid = persisted_by_expiry.begin()->trx_id; if (_pending_block_mode == pending_block_mode::producing) { fc_dlog(_trx_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} is EXPIRING PERSISTED tx: ${txid}", @@ -1158,9 +1163,15 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { num_expired_persistent++; } - fc_dlog(_log, "Processed ${n} persisted transactions, Expired ${expired}", - ("n", orig_count) - ("expired", num_expired_persistent)); + if( exhausted ) { + fc_wlog( _log, "Unable to process all ${n} persisted transactions before deadline, Expired ${expired}", + ( "n", orig_count ) + ( "expired", num_expired_persistent ) ); + } else { + fc_dlog( _log, "Processed ${n} persisted transactions, Expired ${expired}", + ( "n", orig_count ) + ( "expired", num_expired_persistent ) ); + } } try { @@ -1171,13 +1182,15 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { if (_producers.empty() && persisted_by_id.empty()) { // if this node can never produce and has no persisted transactions, // there is no need for unapplied transactions they can be dropped - chain.drop_all_unapplied_transactions(); + chain.get_unapplied_transactions().clear(); } else { - std::vector apply_trxs; - { // derive appliable transactions from unapplied_transactions and drop droppable transactions - auto unapplied_trxs = chain.get_unapplied_transactions(); - apply_trxs.reserve(unapplied_trxs.size()); - + // derive appliable transactions from unapplied_transactions and drop droppable transactions + unapplied_transactions_type& unapplied_trxs = chain.get_unapplied_transactions(); + if( !unapplied_trxs.empty() ) { + auto unapplied_trxs_size = unapplied_trxs.size(); + int num_applied = 0; + int num_failed = 0; + int num_processed = 0; auto calculate_transaction_category = [&](const transaction_metadata_ptr& trx) { if (trx->packed_trx->expiration() < pbs->header.timestamp.to_time_point()) { return tx_category::EXPIRED; @@ -1188,64 +1201,65 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { } }; - for (auto& trx: unapplied_trxs) { + auto itr = unapplied_trxs.begin(); + while( itr != unapplied_trxs.end() ) { + auto itr_next = itr; // save off next since itr may be invalidated by loop + ++itr_next; + + if( preprocess_deadline <= fc::time_point::now() ) exhausted = true; + if( exhausted ) break; + const auto& trx = itr->second; auto category = calculate_transaction_category(trx); - if (category == tx_category::EXPIRED || (category == tx_category::UNEXPIRED_UNPERSISTED && _producers.empty())) { + if (category == tx_category::EXPIRED || + (category == tx_category::UNEXPIRED_UNPERSISTED && _producers.empty())) + { if (!_producers.empty()) { fc_dlog(_trx_trace_log, "[TRX_TRACE] Node with producers configured is dropping an EXPIRED transaction that was PREVIOUSLY ACCEPTED : ${txid}", ("txid", trx->id)); } - chain.drop_unapplied_transaction(trx); - } else if (category == tx_category::PERSISTED || (category == tx_category::UNEXPIRED_UNPERSISTED && _pending_block_mode == pending_block_mode::producing)) { - apply_trxs.emplace_back(std::move(trx)); - } - } - } - - if (!apply_trxs.empty()) { - int num_applied = 0; - int num_failed = 0; - int num_processed = 0; - - for (const auto& trx: apply_trxs) { - if (preprocess_deadline <= fc::time_point::now()) exhausted = true; - if (exhausted) { - break; - } - - num_processed++; - - try { - auto deadline = fc::time_point::now() + fc::milliseconds(_max_transaction_time_ms); - bool deadline_is_subjective = false; - if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && preprocess_deadline < deadline)) { - deadline_is_subjective = true; - deadline = preprocess_deadline; - } + itr = unapplied_trxs.erase( itr ); // unapplied_trxs map has not been modified, so simply erase and continue + continue; + } else if (category == tx_category::PERSISTED || + (category == tx_category::UNEXPIRED_UNPERSISTED && _pending_block_mode == pending_block_mode::producing)) + { + ++num_processed; + + try { + auto deadline = fc::time_point::now() + fc::milliseconds(_max_transaction_time_ms); + bool deadline_is_subjective = false; + if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && preprocess_deadline < deadline)) { + deadline_is_subjective = true; + deadline = preprocess_deadline; + } - auto trace = chain.push_transaction(trx, deadline); - if (trace->except) { - if (failure_is_subjective(*trace->except, deadline_is_subjective)) { - exhausted = true; + auto trace = chain.push_transaction(trx, deadline); + if (trace->except) { + if (failure_is_subjective(*trace->except, deadline_is_subjective)) { + exhausted = true; + break; + } else { + // this failed our configured maximum transaction time, we don't want to replay it + // chain.plus_transactions can modify unapplied_trxs, so erase by id + unapplied_trxs.erase( trx->signed_id ); + ++num_failed; + } } else { - // this failed our configured maximum transaction time, we don't want to replay it - chain.drop_unapplied_transaction(trx); - num_failed++; + ++num_applied; } - } else { - num_applied++; - } - } catch ( const guard_exception& e ) { - chain_plug->handle_guard_exception(e); - return start_block_result::failed; - } FC_LOG_AND_DROP(); + } catch ( const guard_exception& e ) { + chain_plug->handle_guard_exception(e); + return start_block_result::failed; + } FC_LOG_AND_DROP(); + } + + itr = itr_next; } fc_dlog(_log, "Processed ${m} of ${n} previously applied transactions, Applied ${applied}, Failed/Dropped ${failed}", - ("m", num_processed) - ("n", apply_trxs.size()) - ("applied", num_applied) - ("failed", num_failed)); + ("m", num_processed) + ("n", unapplied_trxs_size) + ("applied", num_applied) + ("failed", num_failed)); } } @@ -1258,6 +1272,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { int orig_count = _blacklisted_transactions.size(); while (!blacklist_by_expiry.empty() && blacklist_by_expiry.begin()->expiry <= now) { + if (preprocess_deadline <= fc::time_point::now()) break; blacklist_by_expiry.erase(blacklist_by_expiry.begin()); num_expired++; } @@ -1267,85 +1282,105 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { ("expired", num_expired)); } - auto scheduled_trxs = chain.get_scheduled_transactions(); - if (!scheduled_trxs.empty()) { - int num_applied = 0; - int num_failed = 0; - int num_processed = 0; + // scheduled transactions + int num_applied = 0; + int num_failed = 0; + int num_processed = 0; + + auto scheduled_trx_deadline = preprocess_deadline; + if (_max_scheduled_transaction_time_per_block_ms >= 0) { + scheduled_trx_deadline = std::min( + scheduled_trx_deadline, + fc::time_point::now() + fc::milliseconds(_max_scheduled_transaction_time_per_block_ms) + ); + } + time_point pending_block_time = chain.pending_block_time(); + const auto& sch_idx = chain.db().get_index(); + const auto scheduled_trxs_size = sch_idx.size(); + auto sch_itr = sch_idx.begin(); + while( sch_itr != sch_idx.end() ) { + if( sch_itr->delay_until > pending_block_time) break; // not scheduled yet + if( sch_itr->published >= pending_block_time ) { + ++sch_itr; + continue; // do not allow schedule and execute in same block + } + if( scheduled_trx_deadline <= fc::time_point::now() ) { + exhausted = true; + break; + } - auto scheduled_trx_deadline = preprocess_deadline; - if (_max_scheduled_transaction_time_per_block_ms >= 0) { - scheduled_trx_deadline = std::min( - scheduled_trx_deadline, - fc::time_point::now() + fc::milliseconds(_max_scheduled_transaction_time_per_block_ms) - ); + const transaction_id_type trx_id = sch_itr->trx_id; // make copy since reference could be invalidated + if (blacklist_by_id.find(trx_id) != blacklist_by_id.end()) { + ++sch_itr; + continue; } - for (const auto& trx : scheduled_trxs) { - if (scheduled_trx_deadline <= fc::time_point::now()) exhausted = true; - if (exhausted) { - break; - } + auto sch_itr_next = sch_itr; // save off next since sch_itr may be invalidated by loop + ++sch_itr_next; + const auto next_delay_until = sch_itr_next != sch_idx.end() ? sch_itr_next->delay_until : sch_itr->delay_until; + const auto next_id = sch_itr_next != sch_idx.end() ? sch_itr_next->id : sch_itr->id; - num_processed++; + num_processed++; - // configurable ratio of incoming txns vs deferred txns - while (_incoming_trx_weight >= 1.0 && orig_pending_txn_size && _pending_incoming_transactions.size()) { - if (scheduled_trx_deadline <= fc::time_point::now()) break; + // configurable ratio of incoming txns vs deferred txns + while (_incoming_trx_weight >= 1.0 && orig_pending_txn_size && _pending_incoming_transactions.size()) { + if (scheduled_trx_deadline <= fc::time_point::now()) break; - auto e = _pending_incoming_transactions.front(); - _pending_incoming_transactions.pop_front(); - --orig_pending_txn_size; - _incoming_trx_weight -= 1.0; - process_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)); - } + auto e = _pending_incoming_transactions.front(); + _pending_incoming_transactions.pop_front(); + --orig_pending_txn_size; + _incoming_trx_weight -= 1.0; + process_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)); + } - if (scheduled_trx_deadline <= fc::time_point::now()) { - exhausted = true; - break; - } + if (scheduled_trx_deadline <= fc::time_point::now()) { + exhausted = true; + break; + } - if (blacklist_by_id.find(trx) != blacklist_by_id.end()) { - continue; + try { + auto deadline = fc::time_point::now() + fc::milliseconds(_max_transaction_time_ms); + bool deadline_is_subjective = false; + if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && scheduled_trx_deadline < deadline)) { + deadline_is_subjective = true; + deadline = scheduled_trx_deadline; } - try { - auto deadline = fc::time_point::now() + fc::milliseconds(_max_transaction_time_ms); - bool deadline_is_subjective = false; - if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && scheduled_trx_deadline < deadline)) { - deadline_is_subjective = true; - deadline = scheduled_trx_deadline; - } - - auto trace = chain.push_scheduled_transaction(trx, deadline); - if (trace->except) { - if (failure_is_subjective(*trace->except, deadline_is_subjective)) { - exhausted = true; - } else { - auto expiration = fc::time_point::now() + fc::seconds(chain.get_global_properties().configuration.deferred_trx_expiration_window); - // this failed our configured maximum transaction time, we don't want to replay it add it to a blacklist - _blacklisted_transactions.insert(transaction_id_with_expiry{trx, expiration}); - num_failed++; - } + auto trace = chain.push_scheduled_transaction(trx_id, deadline); + if (trace->except) { + if (failure_is_subjective(*trace->except, deadline_is_subjective)) { + exhausted = true; + break; } else { - num_applied++; + auto expiration = fc::time_point::now() + fc::seconds(chain.get_global_properties().configuration.deferred_trx_expiration_window); + // this failed our configured maximum transaction time, we don't want to replay it add it to a blacklist + _blacklisted_transactions.insert(transaction_id_with_expiry{trx_id, expiration}); + num_failed++; } - } catch ( const guard_exception& e ) { - chain_plug->handle_guard_exception(e); - return start_block_result::failed; - } FC_LOG_AND_DROP(); + } else { + num_applied++; + } + } catch ( const guard_exception& e ) { + chain_plug->handle_guard_exception(e); + return start_block_result::failed; + } FC_LOG_AND_DROP(); - _incoming_trx_weight += _incoming_defer_ratio; - if (!orig_pending_txn_size) _incoming_trx_weight = 0.0; - } + _incoming_trx_weight += _incoming_defer_ratio; + if (!orig_pending_txn_size) _incoming_trx_weight = 0.0; - fc_dlog(_log, "Processed ${m} of ${n} scheduled transactions, Applied ${applied}, Failed/Dropped ${failed}", - ("m", num_processed) - ("n", scheduled_trxs.size()) - ("applied", num_applied) - ("failed", num_failed)); + if( sch_itr_next == sch_idx.end() ) break; + sch_itr = sch_idx.lower_bound( boost::make_tuple( next_delay_until, next_id ) ); + } + if( scheduled_trxs_size > 0 ) { + fc_dlog( _log, + "Processed ${m} of ${n} scheduled transactions, Applied ${applied}, Failed/Dropped ${failed}", + ( "m", num_processed ) + ( "n", scheduled_trxs_size ) + ( "applied", num_applied ) + ( "failed", num_failed ) ); } + } if (exhausted || preprocess_deadline <= fc::time_point::now()) { @@ -1357,11 +1392,11 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { if (!_pending_incoming_transactions.empty()) { fc_dlog(_log, "Processing ${n} pending transactions"); while (orig_pending_txn_size && _pending_incoming_transactions.size()) { + if (preprocess_deadline <= fc::time_point::now()) return start_block_result::exhausted; auto e = _pending_incoming_transactions.front(); _pending_incoming_transactions.pop_front(); --orig_pending_txn_size; process_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)); - if (preprocess_deadline <= fc::time_point::now()) return start_block_result::exhausted; } } return start_block_result::succeeded; diff --git a/unittests/api_tests.cpp b/unittests/api_tests.cpp index 89c94d6f393..955130130fa 100644 --- a/unittests/api_tests.cpp +++ b/unittests/api_tests.cpp @@ -1089,7 +1089,7 @@ BOOST_FIXTURE_TEST_CASE(deferred_transaction_tests, TESTER) { try { produce_blocks( 3 ); //check that only one deferred transaction executed - auto dtrxs = control->get_scheduled_transactions(); + auto dtrxs = get_scheduled_transactions(); BOOST_CHECK_EQUAL(dtrxs.size(), 1); for (const auto& trx: dtrxs) { control->push_scheduled_transaction(trx, fc::time_point::maximum()); @@ -1114,7 +1114,7 @@ BOOST_FIXTURE_TEST_CASE(deferred_transaction_tests, TESTER) { try { produce_blocks( 3 ); //check that only one deferred transaction executed - auto dtrxs = control->get_scheduled_transactions(); + auto dtrxs = get_scheduled_transactions(); BOOST_CHECK_EQUAL(dtrxs.size(), 1); for (const auto& trx: dtrxs) { control->push_scheduled_transaction(trx, fc::time_point::maximum()); diff --git a/unittests/delay_tests.cpp b/unittests/delay_tests.cpp index 9f14de4107c..ae1a3d114b5 100644 --- a/unittests/delay_tests.cpp +++ b/unittests/delay_tests.cpp @@ -72,7 +72,7 @@ BOOST_FIXTURE_TEST_CASE( delay_error_create_account, validating_tester) { try { produce_blocks(6); - auto scheduled_trxs = control->get_scheduled_transactions(); + auto scheduled_trxs = get_scheduled_transactions(); BOOST_REQUIRE_EQUAL(scheduled_trxs.size(), 1); auto dtrace = control->push_scheduled_transaction(scheduled_trxs.front(), fc::time_point::maximum()); BOOST_REQUIRE_EQUAL(dtrace->except.valid(), true); From 3dd25b47ae8e7414d27b16e32105930ac29b7098 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 6 Feb 2019 18:06:39 -0500 Subject: [PATCH 02/17] Bump version to 1.6.1 --- CMakeLists.txt | 2 +- Docker/README.md | 44 +++----------------------------------------- README.md | 16 ++++++++-------- 3 files changed, 12 insertions(+), 50 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fc57979b0e9..c2bd0d7e57f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,7 @@ set( CXX_STANDARD_REQUIRED ON) set(VERSION_MAJOR 1) set(VERSION_MINOR 6) -set(VERSION_PATCH 0) +set(VERSION_PATCH 1) if(VERSION_SUFFIX) set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_SUFFIX}") diff --git a/Docker/README.md b/Docker/README.md index 834f44ac39e..201bee96431 100644 --- a/Docker/README.md +++ b/Docker/README.md @@ -20,10 +20,10 @@ cd eos/Docker docker build . -t eosio/eos ``` -The above will build off the most recent commit to the master branch by default. If you would like to target a specific branch/tag, you may use a build argument. For example, if you wished to generate a docker image based off of the 1.6.0 tag, you could do the following: +The above will build off the most recent commit to the master branch by default. If you would like to target a specific branch/tag, you may use a build argument. For example, if you wished to generate a docker image based off of the 1.6.1 tag, you could do the following: ```bash -docker build -t eosio/eos:1.6.0 --build-arg branch=1.6.0 . +docker build -t eosio/eos:v1.6.1 --build-arg branch=1.6.1 . ``` By default, the symbol in eosio.system is set to SYS. You can override this using the symbol argument while building the docker image. @@ -133,45 +133,7 @@ docker volume rm keosd-data-volume ### Docker Hub -Docker Hub image available from [docker hub](https://hub.docker.com/r/eosio/eos/). -Create a new `docker-compose.yaml` file with the content below - -```bash -version: "3" - -services: - nodeosd: - image: eosio/eos:latest - command: /opt/eosio/bin/nodeosd.sh --data-dir /opt/eosio/bin/data-dir -e --http-alias=nodeosd:8888 --http-alias=127.0.0.1:8888 --http-alias=localhost:8888 - hostname: nodeosd - ports: - - 8888:8888 - - 9876:9876 - expose: - - "8888" - volumes: - - nodeos-data-volume:/opt/eosio/bin/data-dir - - keosd: - image: eosio/eos:latest - command: /opt/eosio/bin/keosd --wallet-dir /opt/eosio/bin/data-dir --http-server-address=127.0.0.1:8900 --http-alias=localhost:8900 --http-alias=keosd:8900 - hostname: keosd - links: - - nodeosd - volumes: - - keosd-data-volume:/opt/eosio/bin/data-dir - -volumes: - nodeos-data-volume: - keosd-data-volume: - -``` - -*NOTE:* the default version is the latest, you can change it to what you want - -run `docker pull eosio/eos:latest` - -run `docker-compose up` +Docker Hub images are now deprecated. New build images were discontinued on January 1st, 2019. The existing old images will be removed on June 1st, 2019. ### EOSIO Testnet diff --git a/README.md b/README.md index 17dfe24052b..8c7a7e1f2dc 100644 --- a/README.md +++ b/README.md @@ -39,13 +39,13 @@ $ brew remove eosio ``` #### Ubuntu 18.04 Debian Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.6.0/eosio_1.6.0-1-ubuntu-18.04_amd64.deb -$ sudo apt install ./eosio_1.6.0-1-ubuntu-18.04_amd64.deb +$ wget https://github.com/eosio/eos/releases/download/v1.6.1/eosio_1.6.1-1-ubuntu-18.04_amd64.deb +$ sudo apt install ./eosio_1.6.1-1-ubuntu-18.04_amd64.deb ``` #### Ubuntu 16.04 Debian Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.6.0/eosio_1.6.0-1-ubuntu-16.04_amd64.deb -$ sudo apt install ./eosio_1.6.0-1-ubuntu-16.04_amd64.deb +$ wget https://github.com/eosio/eos/releases/download/v1.6.1/eosio_1.6.1-1-ubuntu-16.04_amd64.deb +$ sudo apt install ./eosio_1.6.1-1-ubuntu-16.04_amd64.deb ``` #### Debian Package Uninstall ```sh @@ -53,8 +53,8 @@ $ sudo apt remove eosio ``` #### Centos RPM Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.6.0/eosio-1.6.0-1.el7.x86_64.rpm -$ sudo yum install ./eosio-1.6.0-1.el7.x86_64.rpm +$ wget https://github.com/eosio/eos/releases/download/v1.6.1/eosio-1.6.1-1.el7.x86_64.rpm +$ sudo yum install ./eosio-1.6.1-1.el7.x86_64.rpm ``` #### Centos RPM Package Uninstall ```sh @@ -62,8 +62,8 @@ $ sudo yum remove eosio.cdt ``` #### Fedora RPM Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.6.0/eosio-1.6.0-1.fc27.x86_64.rpm -$ sudo yum install ./eosio-1.6.0-1.fc27.x86_64.rpm +$ wget https://github.com/eosio/eos/releases/download/v1.6.1/eosio-1.6.1-1.fc27.x86_64.rpm +$ sudo yum install ./eosio-1.6.1-1.fc27.x86_64.rpm ``` #### Fedora RPM Package Uninstall ```sh From f7ecd76d9af94c8ef866e24cc02f057140e07a67 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Mon, 11 Feb 2019 10:14:56 -0500 Subject: [PATCH 03/17] Fix boost 1.67 brew install The latest homebrew code balks at something in the old 1.67 package file. Fix the package file and store it locally in our repo for now. We try and pin the boost version because boost upgrades invalidate nodeos data files. --- scripts/boost.rb | 131 ++++++++++++++++++++++++++++++++++ scripts/eosio_build_darwin.sh | 2 +- 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 scripts/boost.rb diff --git a/scripts/boost.rb b/scripts/boost.rb new file mode 100644 index 00000000000..a322acbdd11 --- /dev/null +++ b/scripts/boost.rb @@ -0,0 +1,131 @@ +class Boost < Formula + desc "Collection of portable C++ source libraries" + homepage "https://www.boost.org/" + revision 1 + head "https://github.com/boostorg/boost.git" + + stable do + url "https://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_67_0.tar.bz2" + sha256 "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba" + + # Remove for > 1.67.0 + # Fix "error: no member named 'next' in namespace 'boost'" + # Upstream commit from 1 Dec 2017 "Add #include ; no + # longer in utility.hpp" + patch :p2 do + url "https://github.com/boostorg/lockfree/commit/12726cd.patch?full_index=1" + sha256 "f165823d961a588b622b20520668b08819eb5fdc08be7894c06edce78026ce0a" + end + end + + bottle do + cellar :any + sha256 "265ab8beaa6fa26a7c305ef2e6aec8bd26ca1db105aca0aaca028f32c5245a90" => :high_sierra + sha256 "567f3e9a294413c1701b698d666a521cfdeec846e256c6e66576d5b70eb26f08" => :sierra + sha256 "3f3f687a620f656fe2ac54f01306e00e6bbc0e9797db284a8d272648d427e640" => :el_capitan + end + + option "with-icu4c", "Build regexp engine with icu support" + option "without-single", "Disable building single-threading variant" + option "without-static", "Disable building static library variant" + + deprecated_option "with-icu" => "with-icu4c" + + depends_on "icu4c" => :optional + + def install + # Force boost to compile with the desired compiler + open("user-config.jam", "a") do |file| + file.write "using darwin : : #{ENV.cxx} ;\n" + end + + # libdir should be set by --prefix but isn't + bootstrap_args = ["--prefix=#{prefix}", "--libdir=#{lib}"] + + if build.with? "icu4c" + icu4c_prefix = Formula["icu4c"].opt_prefix + bootstrap_args << "--with-icu=#{icu4c_prefix}" + else + bootstrap_args << "--without-icu" + end + + # Handle libraries that will not be built. + without_libraries = ["python", "mpi"] + + # Boost.Log cannot be built using Apple GCC at the moment. Disabled + # on such systems. + without_libraries << "log" if ENV.compiler == :gcc + + bootstrap_args << "--without-libraries=#{without_libraries.join(",")}" + + # layout should be synchronized with boost-python and boost-mpi + args = ["--prefix=#{prefix}", + "--libdir=#{lib}", + "-d2", + "-j#{ENV.make_jobs}", + "--layout=tagged", + "--user-config=user-config.jam", + "-sNO_LZMA=1", + "install"] + + if build.with? "single" + args << "threading=multi,single" + else + args << "threading=multi" + end + + if build.with? "static" + args << "link=shared,static" + else + args << "link=shared" + end + + # Trunk starts using "clang++ -x c" to select C compiler which breaks C++11 + # handling using ENV.cxx11. Using "cxxflags" and "linkflags" still works. + args << "cxxflags=-std=c++11" + if ENV.compiler == :clang + args << "cxxflags=-stdlib=libc++" << "linkflags=-stdlib=libc++" + end + + system "./bootstrap.sh", *bootstrap_args + system "./b2", "headers" + system "./b2", *args + end + + def caveats + s = "" + # ENV.compiler doesn't exist in caveats. Check library availability + # instead. + if Dir["#{lib}/libboost_log*"].empty? + s += <<~EOS + Building of Boost.Log is disabled because it requires newer GCC or Clang. + EOS + end + + s + end + + test do + (testpath/"test.cpp").write <<~EOS + #include + #include + #include + #include + using namespace boost::algorithm; + using namespace std; + + int main() + { + string str("a,b"); + vector strVec; + split(strVec, str, is_any_of(",")); + assert(strVec.size()==2); + assert(strVec[0]=="a"); + assert(strVec[1]=="b"); + return 0; + } + EOS + system ENV.cxx, "test.cpp", "-std=c++1y", "-L#{lib}", "-lboost_system", "-o", "test" + system "./test" + end +end diff --git a/scripts/eosio_build_darwin.sh b/scripts/eosio_build_darwin.sh index a91e1611d44..af36abe38d0 100644 --- a/scripts/eosio_build_darwin.sh +++ b/scripts/eosio_build_darwin.sh @@ -218,7 +218,7 @@ done fi printf "\\tInstalling boost libraries.\\n" - if ! "${BREW}" install https://raw.githubusercontent.com/Homebrew/homebrew-core/f946d12e295c8a27519b73cc810d06593270a07f/Formula/boost.rb + if ! "${BREW}" install "${SOURCE_DIR}/scripts/boost.rb" then printf "\\tUnable to install boost 1.67 libraries at this time. 0\\n" printf "\\tExiting now.\\n\\n" From 74a143cc1d0cd7a6cc795030ea3a51b979a6bd43 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Mon, 11 Feb 2019 15:24:34 -0500 Subject: [PATCH 04/17] Set proper directory for baked in macOS LLVM_DIR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some users (including myself) were seeing llvm@4 unpacked to 4.0.1_1 instead of 4.0.1. Stuff unpacked to the Cellar directory appears to be a kind of implementation detail — /usr/local/opt is the proper place to reference here. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c2bd0d7e57f..59b3952809a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") if (UNIX) if (APPLE) if (LLVM_DIR STREQUAL "" OR NOT LLVM_DIR) - set(LLVM_DIR "/usr/local/Cellar/llvm@4/4.0.1/lib/cmake/llvm") + set(LLVM_DIR "/usr/local/opt/llvm@4/lib/cmake/llvm/") endif() endif() endif() From de26672781cb9c506c63376da5813f1f7d40d2d3 Mon Sep 17 00:00:00 2001 From: Emory Barlow Date: Tue, 12 Feb 2019 23:25:59 -0500 Subject: [PATCH 05/17] Add bk step to gather brew files for automatic update --- .buildkite/pipeline.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 3c7f63def07..893a19f9ff9 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -398,6 +398,7 @@ steps: - "os=high-sierra" artifact_paths: - "build/packages/*.tar.gz" + - "build/packages/*.rb" timeout: 60 - command: | @@ -412,6 +413,7 @@ steps: - "os=mojave" artifact_paths: - "build/packages/*.tar.gz" + - "build/packages/*.rb" timeout: 60 - command: | @@ -507,3 +509,18 @@ steps: OS: "el7" PKGTYPE: "rpm" timeout: 60 + + - wait + + - command: | + echo "--- :arrow_down: Downloading brew files" && \ + buildkite-agent artifact download "build/packages/eosio.rb" . --step ":darwin: High Sierra Package Builder" && \ + mv build/packages/eosio.rb build/packages/eosio_highsierra.rb && \ + buildkite-agent artifact download "build/packages/eosio.rb" . --step ":darwin: Mojave Package Builder" + label: ":darwin: Brew Updater" + agents: + queue: "automation-large-builder-fleet" + artifact_paths: + - "build/packages/eosio_highsierra.rb" + - "build/packages/eosio.rb" + timeout: 60 From f2d45399dbdbf713fc2c9e655a1b40c908cf8266 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 14 Feb 2019 13:29:37 -0600 Subject: [PATCH 06/17] Update version to 1.6.2 --- CMakeLists.txt | 2 +- Docker/README.md | 4 ++-- README.md | 16 ++++++++-------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59b3952809a..e35a98973f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,7 @@ set( CXX_STANDARD_REQUIRED ON) set(VERSION_MAJOR 1) set(VERSION_MINOR 6) -set(VERSION_PATCH 1) +set(VERSION_PATCH 2) if(VERSION_SUFFIX) set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_SUFFIX}") diff --git a/Docker/README.md b/Docker/README.md index 201bee96431..45459b11b4b 100644 --- a/Docker/README.md +++ b/Docker/README.md @@ -20,10 +20,10 @@ cd eos/Docker docker build . -t eosio/eos ``` -The above will build off the most recent commit to the master branch by default. If you would like to target a specific branch/tag, you may use a build argument. For example, if you wished to generate a docker image based off of the 1.6.1 tag, you could do the following: +The above will build off the most recent commit to the master branch by default. If you would like to target a specific branch/tag, you may use a build argument. For example, if you wished to generate a docker image based off of the 1.6.2 tag, you could do the following: ```bash -docker build -t eosio/eos:v1.6.1 --build-arg branch=1.6.1 . +docker build -t eosio/eos:v1.6.2 --build-arg branch=1.6.2 . ``` By default, the symbol in eosio.system is set to SYS. You can override this using the symbol argument while building the docker image. diff --git a/README.md b/README.md index 8c7a7e1f2dc..3009bfb4c01 100644 --- a/README.md +++ b/README.md @@ -39,13 +39,13 @@ $ brew remove eosio ``` #### Ubuntu 18.04 Debian Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.6.1/eosio_1.6.1-1-ubuntu-18.04_amd64.deb -$ sudo apt install ./eosio_1.6.1-1-ubuntu-18.04_amd64.deb +$ wget https://github.com/eosio/eos/releases/download/v1.6.2/eosio_1.6.2-1-ubuntu-18.04_amd64.deb +$ sudo apt install ./eosio_1.6.2-1-ubuntu-18.04_amd64.deb ``` #### Ubuntu 16.04 Debian Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.6.1/eosio_1.6.1-1-ubuntu-16.04_amd64.deb -$ sudo apt install ./eosio_1.6.1-1-ubuntu-16.04_amd64.deb +$ wget https://github.com/eosio/eos/releases/download/v1.6.2/eosio_1.6.2-1-ubuntu-16.04_amd64.deb +$ sudo apt install ./eosio_1.6.2-1-ubuntu-16.04_amd64.deb ``` #### Debian Package Uninstall ```sh @@ -53,8 +53,8 @@ $ sudo apt remove eosio ``` #### Centos RPM Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.6.1/eosio-1.6.1-1.el7.x86_64.rpm -$ sudo yum install ./eosio-1.6.1-1.el7.x86_64.rpm +$ wget https://github.com/eosio/eos/releases/download/v1.6.2/eosio-1.6.2-1.el7.x86_64.rpm +$ sudo yum install ./eosio-1.6.2-1.el7.x86_64.rpm ``` #### Centos RPM Package Uninstall ```sh @@ -62,8 +62,8 @@ $ sudo yum remove eosio.cdt ``` #### Fedora RPM Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.6.1/eosio-1.6.1-1.fc27.x86_64.rpm -$ sudo yum install ./eosio-1.6.1-1.fc27.x86_64.rpm +$ wget https://github.com/eosio/eos/releases/download/v1.6.2/eosio-1.6.2-1.fc27.x86_64.rpm +$ sudo yum install ./eosio-1.6.2-1.fc27.x86_64.rpm ``` #### Fedora RPM Package Uninstall ```sh From a8c01c810e8730fefadf65e9c4bd5d5d2499ad66 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Sat, 16 Feb 2019 22:52:25 -0500 Subject: [PATCH 07/17] add noninteractive option for build scripts --- eosio_build.sh | 11 +++++++++-- scripts/eosio_build_amazon.sh | 1 + scripts/eosio_build_centos.sh | 4 ++++ scripts/eosio_build_darwin.sh | 3 +++ scripts/eosio_build_fedora.sh | 1 + scripts/eosio_build_ubuntu.sh | 1 + 6 files changed, 19 insertions(+), 2 deletions(-) diff --git a/eosio_build.sh b/eosio_build.sh index d1e77b6e85f..173005a15d4 100755 --- a/eosio_build.sh +++ b/eosio_build.sh @@ -34,10 +34,14 @@ function usage() { - printf "\\tUsage: %s \\n\\t[Build Option -o ] \\n\\t[CodeCoverage -c] \\n\\t[Doxygen -d] \\n\\t[CoreSymbolName -s <1-7 characters>] \\n\\t[Avoid Compiling -a]\\n\\n" "$0" 1>&2 + printf "\\tUsage: %s \\n\\t[Build Option -o ] \\n\\t[CodeCoverage -c] \\n\\t[Doxygen -d] \\n\\t[CoreSymbolName -s <1-7 characters>] \\n\\t[Avoid Compiling -a]\\n\\t[Noninteractive -y]\\n\\n" "$0" 1>&2 exit 1 } + is_noninteractive() { + [[ -n "${EOSIO_BUILD_NONINTERACTIVE+1}" ]] + } + ARCH=$( uname ) if [ "${SOURCE_DIR}" == "${PWD}" ]; then BUILD_DIR="${PWD}/build" @@ -66,7 +70,7 @@ txtrst=$(tput sgr0) if [ $# -ne 0 ]; then - while getopts ":cdo:s:ah" opt; do + while getopts ":cdo:s:ahy" opt; do case "${opt}" in o ) options=( "Debug" "Release" "RelWithDebInfo" "MinSizeRel" ) @@ -100,6 +104,9 @@ usage exit 1 ;; + y) + EOSIO_BUILD_NONINTERACTIVE=1 + ;; \? ) printf "\\n\\tInvalid Option: %s\\n" "-${OPTARG}" 1>&2 usage diff --git a/scripts/eosio_build_amazon.sh b/scripts/eosio_build_amazon.sh index 1c96024b847..b4ee2e5321f 100644 --- a/scripts/eosio_build_amazon.sh +++ b/scripts/eosio_build_amazon.sh @@ -87,6 +87,7 @@ printf "\\n\\tThe following dependencies are required to install EOSIO.\\n" printf "\\n\\t${DISPLAY}\\n\\n" printf "\\tDo you wish to install these dependencies?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) diff --git a/scripts/eosio_build_centos.sh b/scripts/eosio_build_centos.sh index fa5e3c61378..6c3f11ee1ca 100644 --- a/scripts/eosio_build_centos.sh +++ b/scripts/eosio_build_centos.sh @@ -63,6 +63,7 @@ SCL=$( rpm -qa | grep -E 'centos-release-scl-[0-9].*' ) if [ -z "${SCL}" ]; then printf "\\t - Do you wish to install and enable this repository?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) @@ -87,6 +88,7 @@ DEVTOOLSET=$( rpm -qa | grep -E 'devtoolset-7-[0-9].*' ) if [ -z "${DEVTOOLSET}" ]; then printf "\\tDo you wish to install devtoolset-7?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) @@ -118,6 +120,7 @@ PYTHON33=$( rpm -qa | grep -E 'python33-[0-9].*' ) if [ -z "${PYTHON33}" ]; then printf "\\tDo you wish to install python33?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) @@ -170,6 +173,7 @@ printf "\\tThe following dependencies are required to install EOSIO.\\n" printf "\\t${DISPLAY}\\n\\n" printf "\\tDo you wish to install these dependencies?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) diff --git a/scripts/eosio_build_darwin.sh b/scripts/eosio_build_darwin.sh index af36abe38d0..661238f18bc 100644 --- a/scripts/eosio_build_darwin.sh +++ b/scripts/eosio_build_darwin.sh @@ -70,6 +70,7 @@ then printf "\\tHomebrew must be installed to compile EOS.IO\\n\\n" printf "\\tDo you wish to install Home Brew?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case "${yn}" in [Yy]* ) @@ -138,6 +139,7 @@ printf "\\n\\tThe following dependencies are required to install EOSIO.\\n" printf "\\n\\t${DISPLAY}\\n\\n" echo "Do you wish to install these packages?" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) @@ -184,6 +186,7 @@ printf "\\tFound Boost Version %s.\\n" "${BVERSION}" printf "\\tEOS.IO requires Boost version 1.67.\\n" printf "\\tWould you like to uninstall version %s and install Boost version 1.67.\\n" "${BVERSION}" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) diff --git a/scripts/eosio_build_fedora.sh b/scripts/eosio_build_fedora.sh index 661efea9fc0..4492e545c17 100644 --- a/scripts/eosio_build_fedora.sh +++ b/scripts/eosio_build_fedora.sh @@ -86,6 +86,7 @@ printf "\\n\\tThe following dependencies are required to install EOSIO.\\n" printf "\\n\\t${DISPLAY}\\n\\n" printf "\\tDo you wish to install these dependencies?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) diff --git a/scripts/eosio_build_ubuntu.sh b/scripts/eosio_build_ubuntu.sh index 4c9873a60a1..f87397e8625 100644 --- a/scripts/eosio_build_ubuntu.sh +++ b/scripts/eosio_build_ubuntu.sh @@ -91,6 +91,7 @@ printf "\\n\\tThe following dependencies are required to install EOSIO.\\n" printf "\\n\\t${DISPLAY}\\n\\n" printf "\\tDo you wish to install these packages?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) From 2aa4bed95492d2333b95113943a3b5301d5a8298 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 18 Feb 2019 12:00:15 -0600 Subject: [PATCH 08/17] Limit assert message to 1024 chars --- libraries/chain/wasm_interface.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libraries/chain/wasm_interface.cpp b/libraries/chain/wasm_interface.cpp index 28de213c1c2..c8f81cacca3 100644 --- a/libraries/chain/wasm_interface.cpp +++ b/libraries/chain/wasm_interface.cpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace eosio { namespace chain { using namespace webassembly; @@ -901,6 +902,8 @@ class system_api : public context_aware_api { }; +constexpr size_t max_assert_message = 1024; + class context_free_system_api : public context_aware_api { public: explicit context_free_system_api( apply_context& ctx ) @@ -913,14 +916,16 @@ class context_free_system_api : public context_aware_api { // Kept as intrinsic rather than implementing on WASM side (using eosio_assert_message and strlen) because strlen is faster on native side. void eosio_assert( bool condition, null_terminated_ptr msg ) { if( BOOST_UNLIKELY( !condition ) ) { - std::string message( msg ); + const size_t sz = strnlen( msg, max_assert_message ); + std::string message( msg, sz ); EOS_THROW( eosio_assert_message_exception, "assertion failure with message: ${s}", ("s",message) ); } } void eosio_assert_message( bool condition, array_ptr msg, size_t msg_len ) { if( BOOST_UNLIKELY( !condition ) ) { - std::string message( msg, msg_len ); + const size_t sz = msg_len > max_assert_message ? max_assert_message : msg_len; + std::string message( msg, sz ); EOS_THROW( eosio_assert_message_exception, "assertion failure with message: ${s}", ("s",message) ); } } From 0cbccbb03a04b509790f6a0aec6f4eab52f39794 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Tue, 19 Feb 2019 09:52:08 -0500 Subject: [PATCH 09/17] Don't unlink what we install via brew unlinking eveything we install makes no sense -- it means things like cmake aren't in the path any longer like the script expects. So don't do that any more. Unfortuately this old script requires that gettext be force linked. So implement that behavior explictly for now --- scripts/eosio_build_darwin.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/eosio_build_darwin.sh b/scripts/eosio_build_darwin.sh index 661238f18bc..adcae78a25c 100644 --- a/scripts/eosio_build_darwin.sh +++ b/scripts/eosio_build_darwin.sh @@ -161,13 +161,12 @@ printf "\\tExiting now.\\n\\n" exit 1; fi - if [[ "$DEP" == "llvm@4" ]]; then - "${BREW}" unlink ${DEP} - elif ! "${BREW}" unlink ${DEP} && "${BREW}" link --force ${DEP} - then - printf "\\tHomebrew exited with the above errors.\\n" - printf "\\tExiting now.\\n\\n" - exit 1; + if [ $PERMISSION_GETTEXT -eq 1 ]; then + if ! "${BREW}" link --force gettext; then + printf "\\tHomebrew exited with the above errors.\\n" + printf "\\tExiting now.\\n\\n" + exit 1; + fi fi break;; [Nn]* ) echo "User aborting installation of required dependencies, Exiting now."; exit;; From 56726b0a4b45515705e35025120bdf9afcbb493c Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Tue, 19 Feb 2019 15:45:03 -0500 Subject: [PATCH 10/17] Make sure python-devel is installed for amazon linux builds --- scripts/eosio_build_amazon.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/eosio_build_amazon.sh b/scripts/eosio_build_amazon.sh index b4ee2e5321f..73c163330dd 100644 --- a/scripts/eosio_build_amazon.sh +++ b/scripts/eosio_build_amazon.sh @@ -56,11 +56,11 @@ if [[ "${OS_NAME}" == "Amazon Linux AMI" ]]; then DEP_ARRAY=( git gcc72.x86_64 gcc72-c++.x86_64 autoconf automake libtool make bzip2 \ bzip2-devel.x86_64 openssl-devel.x86_64 gmp-devel.x86_64 libstdc++72.x86_64 \ - python27.x86_64 python36-devel.x86_64 libedit-devel.x86_64 doxygen.x86_64 graphviz.x86_64) + python27.x86_64 python27-devel.x86_64 python36-devel.x86_64 libedit-devel.x86_64 doxygen.x86_64 graphviz.x86_64) else DEP_ARRAY=( git gcc gcc-c++ autoconf automake libtool make bzip2 \ bzip2-devel openssl-devel gmp-devel libstdc++ \ - python3 python3-devel libedit-devel doxygen graphviz) + python3 python3-devel python-devel libedit-devel doxygen graphviz) fi COUNT=1 DISPLAY="" From 3b15c4f0d7bbd4c0138060a85dacce5711ba1b1c Mon Sep 17 00:00:00 2001 From: fsword Date: Sat, 23 Feb 2019 14:44:29 -0800 Subject: [PATCH 11/17] Wrong tag/branch name there is no tag or branch named "1.6.2", I think it should be "v1.6.2"? --- Docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/README.md b/Docker/README.md index 45459b11b4b..9e01fe71f65 100644 --- a/Docker/README.md +++ b/Docker/README.md @@ -23,7 +23,7 @@ docker build . -t eosio/eos The above will build off the most recent commit to the master branch by default. If you would like to target a specific branch/tag, you may use a build argument. For example, if you wished to generate a docker image based off of the 1.6.2 tag, you could do the following: ```bash -docker build -t eosio/eos:v1.6.2 --build-arg branch=1.6.2 . +docker build -t eosio/eos:v1.6.2 --build-arg branch=v1.6.2 . ``` By default, the symbol in eosio.system is set to SYS. You can override this using the symbol argument while building the docker image. From 7c1e14a81af8cf0f9da8e64774f411f4ccacc8f7 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 5 Mar 2019 09:00:21 -0500 Subject: [PATCH 12/17] Call recover keys before transactions execution so trx->sig_cpu_usage is set correctly --- libraries/chain/controller.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 2d2af25f0c1..8be2fa4ad7e 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -999,6 +999,9 @@ struct controller_impl { transaction_trace_ptr trace; try { auto start = fc::time_point::now(); + const bool check_auth = !self.skip_auth_check() && !trx->implicit; + // call recover keys so that trx->sig_cpu_usage is set correctly + const flat_set& recovered_keys = check_auth ? trx->recover_keys( chain_id ) : flat_set(); if( !explicit_billed_cpu_time ) { fc::microseconds already_consumed_time( EOS_PERCENT(trx->sig_cpu_usage.count(), conf.sig_cpu_bill_pct) ); @@ -1031,10 +1034,10 @@ struct controller_impl { trx_context.delay = fc::seconds(trn.delay_sec); - if( !self.skip_auth_check() && !trx->implicit ) { + if( check_auth ) { authorization.check_authorization( trn.actions, - trx->recover_keys( chain_id ), + recovered_keys, {}, trx_context.delay, [](){} From 12d996150d233bff65d6bcc328e742a6d564b0cf Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 5 Mar 2019 16:23:34 -0500 Subject: [PATCH 13/17] Consolidated Security Fixes for 1.6.3 - Fix small memory leak in net_plugin. - Add additional deadline checks to transaction authorization. --- libraries/chain/controller.cpp | 4 +--- plugins/net_plugin/net_plugin.cpp | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 2d2af25f0c1..cf904d6fb5b 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1037,9 +1037,7 @@ struct controller_impl { trx->recover_keys( chain_id ), {}, trx_context.delay, - [](){} - /*std::bind(&transaction_context::add_cpu_usage_and_check_time, &trx_context, - std::placeholders::_1)*/, + [&trx_context](){ trx_context.checktime(); }, false ); } diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 1b398a8b53a..ab36d06cc6a 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -724,6 +724,7 @@ namespace eosio { void rejected_block(const block_id_type& id); void recv_block(const connection_ptr& conn, const block_id_type& msg, uint32_t bnum); + void expire_blocks( uint32_t bnum ); void recv_transaction(const connection_ptr& conn, const transaction_id_type& id); void recv_notice(const connection_ptr& conn, const notice_message& msg, bool generated); @@ -1656,11 +1657,23 @@ namespace eosio { } void dispatch_manager::rejected_block(const block_id_type& id) { - fc_dlog(logger,"not sending rejected transaction ${tid}",("tid",id)); + fc_dlog( logger, "rejected block ${id}", ("id", id) ); auto range = received_blocks.equal_range(id); received_blocks.erase(range.first, range.second); } + void dispatch_manager::expire_blocks( uint32_t lib_num ) { + for( auto i = received_blocks.begin(); i != received_blocks.end(); ) { + const block_id_type& blk_id = i->first; + uint32_t blk_num = block_header::num_from_id( blk_id ); + if( blk_num <= lib_num ) { + i = received_blocks.erase( i ); + } else { + ++i; + } + } + } + void dispatch_manager::bcast_transaction(const transaction_metadata_ptr& ptrx) { std::set skips; const auto& id = ptrx->id; @@ -2565,6 +2578,7 @@ namespace eosio { } else { sync_master->rejected_block(c, blk_num); + dispatcher->rejected_block( blk_id ); } } @@ -2626,6 +2640,7 @@ namespace eosio { controller& cc = chain_plug->chain(); uint32_t lib = cc.last_irreversible_block_num(); + dispatcher->expire_blocks( lib ); for ( auto &c : connections ) { auto &stale_txn = c->trx_state.get(); stale_txn.erase( stale_txn.lower_bound(1), stale_txn.upper_bound(lib) ); From bf85b4fcea6527882a0217fc0adab02cd3cd5bc2 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 6 Mar 2019 17:14:48 -0500 Subject: [PATCH 14/17] Bump version to 1.6.3 --- CMakeLists.txt | 2 +- Docker/README.md | 4 ++-- README.md | 16 ++++++++-------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e35a98973f4..3a03c7b0ed2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,7 @@ set( CXX_STANDARD_REQUIRED ON) set(VERSION_MAJOR 1) set(VERSION_MINOR 6) -set(VERSION_PATCH 2) +set(VERSION_PATCH 3) if(VERSION_SUFFIX) set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_SUFFIX}") diff --git a/Docker/README.md b/Docker/README.md index 9e01fe71f65..f697f1862a0 100644 --- a/Docker/README.md +++ b/Docker/README.md @@ -20,10 +20,10 @@ cd eos/Docker docker build . -t eosio/eos ``` -The above will build off the most recent commit to the master branch by default. If you would like to target a specific branch/tag, you may use a build argument. For example, if you wished to generate a docker image based off of the 1.6.2 tag, you could do the following: +The above will build off the most recent commit to the master branch by default. If you would like to target a specific branch/tag, you may use a build argument. For example, if you wished to generate a docker image based off of the 1.6.3 tag, you could do the following: ```bash -docker build -t eosio/eos:v1.6.2 --build-arg branch=v1.6.2 . +docker build -t eosio/eos:v1.6.3 --build-arg branch=v1.6.3 . ``` By default, the symbol in eosio.system is set to SYS. You can override this using the symbol argument while building the docker image. diff --git a/README.md b/README.md index 3009bfb4c01..9b39e234348 100644 --- a/README.md +++ b/README.md @@ -39,13 +39,13 @@ $ brew remove eosio ``` #### Ubuntu 18.04 Debian Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.6.2/eosio_1.6.2-1-ubuntu-18.04_amd64.deb -$ sudo apt install ./eosio_1.6.2-1-ubuntu-18.04_amd64.deb +$ wget https://github.com/eosio/eos/releases/download/v1.6.3/eosio_1.6.3-1-ubuntu-18.04_amd64.deb +$ sudo apt install ./eosio_1.6.3-1-ubuntu-18.04_amd64.deb ``` #### Ubuntu 16.04 Debian Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.6.2/eosio_1.6.2-1-ubuntu-16.04_amd64.deb -$ sudo apt install ./eosio_1.6.2-1-ubuntu-16.04_amd64.deb +$ wget https://github.com/eosio/eos/releases/download/v1.6.3/eosio_1.6.3-1-ubuntu-16.04_amd64.deb +$ sudo apt install ./eosio_1.6.3-1-ubuntu-16.04_amd64.deb ``` #### Debian Package Uninstall ```sh @@ -53,8 +53,8 @@ $ sudo apt remove eosio ``` #### Centos RPM Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.6.2/eosio-1.6.2-1.el7.x86_64.rpm -$ sudo yum install ./eosio-1.6.2-1.el7.x86_64.rpm +$ wget https://github.com/eosio/eos/releases/download/v1.6.3/eosio-1.6.3-1.el7.x86_64.rpm +$ sudo yum install ./eosio-1.6.3-1.el7.x86_64.rpm ``` #### Centos RPM Package Uninstall ```sh @@ -62,8 +62,8 @@ $ sudo yum remove eosio.cdt ``` #### Fedora RPM Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.6.2/eosio-1.6.2-1.fc27.x86_64.rpm -$ sudo yum install ./eosio-1.6.2-1.fc27.x86_64.rpm +$ wget https://github.com/eosio/eos/releases/download/v1.6.3/eosio-1.6.3-1.fc27.x86_64.rpm +$ sudo yum install ./eosio-1.6.3-1.fc27.x86_64.rpm ``` #### Fedora RPM Package Uninstall ```sh From 254cde20265da710f172a0178e73e1baacb611f2 Mon Sep 17 00:00:00 2001 From: Matias Romeo Date: Wed, 3 Apr 2019 12:39:12 -0300 Subject: [PATCH 15/17] cleos: handle no Content-length header in http response --- programs/cleos/httpc.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/programs/cleos/httpc.cpp b/programs/cleos/httpc.cpp index 5503c8fe8ec..9a9cdb8f866 100644 --- a/programs/cleos/httpc.cpp +++ b/programs/cleos/httpc.cpp @@ -89,17 +89,22 @@ namespace eosio { namespace client { namespace http { if(std::regex_search(header, match, clregex)) response_content_length = std::stoi(match[1]); } - EOS_ASSERT(response_content_length >= 0, invalid_http_response, "Invalid content-length response"); std::stringstream re; - // Write whatever content we already have to output. - response_content_length -= response.size(); - if (response.size() > 0) - re << &response; + if( response_content_length >= 0 ) { + // Write whatever content we already have to output. + response_content_length -= response.size(); + if (response.size() > 0) + re << &response; - boost::asio::read(socket, response, boost::asio::transfer_exactly(response_content_length)); - re << &response; + boost::asio::read(socket, response, boost::asio::transfer_exactly(response_content_length)); + } else { + boost::system::error_code ec; + boost::asio::read(socket, response, boost::asio::transfer_all(), ec); + EOS_ASSERT(!ec || ec == boost::asio::ssl::error::stream_truncated, http_exception, "Unable to read http response: ${err}", ("err",ec.message())); + } + re << &response; return re.str(); } From 5e8c459607f88cd51d226faf85fc12b1e04d7062 Mon Sep 17 00:00:00 2001 From: Matias Romeo Date: Tue, 9 Apr 2019 21:57:25 -0300 Subject: [PATCH 16/17] cleos: simplify http response body reading --- programs/cleos/httpc.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/programs/cleos/httpc.cpp b/programs/cleos/httpc.cpp index 9a9cdb8f866..7d9326b9ed7 100644 --- a/programs/cleos/httpc.cpp +++ b/programs/cleos/httpc.cpp @@ -90,20 +90,19 @@ namespace eosio { namespace client { namespace http { response_content_length = std::stoi(match[1]); } - std::stringstream re; - if( response_content_length >= 0 ) { - // Write whatever content we already have to output. + // Attempt to read the response body using the length indicated by the + // Content-length header. If the header was not present just read all available bytes. + if( response_content_length != -1 ) { response_content_length -= response.size(); - if (response.size() > 0) - re << &response; - - boost::asio::read(socket, response, boost::asio::transfer_exactly(response_content_length)); + if( response_content_length > 0 ) + boost::asio::read(socket, response, boost::asio::transfer_exactly(response_content_length)); } else { boost::system::error_code ec; boost::asio::read(socket, response, boost::asio::transfer_all(), ec); EOS_ASSERT(!ec || ec == boost::asio::ssl::error::stream_truncated, http_exception, "Unable to read http response: ${err}", ("err",ec.message())); } + std::stringstream re; re << &response; return re.str(); } From 698251caf943b56ddd8eac7c899ea30a641db2dc Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Wed, 10 Apr 2019 12:41:57 -0400 Subject: [PATCH 17/17] Revert "Allow cleos to query an API node behind Cloudflare" --- programs/cleos/httpc.cpp | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/programs/cleos/httpc.cpp b/programs/cleos/httpc.cpp index 7d9326b9ed7..5503c8fe8ec 100644 --- a/programs/cleos/httpc.cpp +++ b/programs/cleos/httpc.cpp @@ -89,21 +89,17 @@ namespace eosio { namespace client { namespace http { if(std::regex_search(header, match, clregex)) response_content_length = std::stoi(match[1]); } - - // Attempt to read the response body using the length indicated by the - // Content-length header. If the header was not present just read all available bytes. - if( response_content_length != -1 ) { - response_content_length -= response.size(); - if( response_content_length > 0 ) - boost::asio::read(socket, response, boost::asio::transfer_exactly(response_content_length)); - } else { - boost::system::error_code ec; - boost::asio::read(socket, response, boost::asio::transfer_all(), ec); - EOS_ASSERT(!ec || ec == boost::asio::ssl::error::stream_truncated, http_exception, "Unable to read http response: ${err}", ("err",ec.message())); - } + EOS_ASSERT(response_content_length >= 0, invalid_http_response, "Invalid content-length response"); std::stringstream re; + // Write whatever content we already have to output. + response_content_length -= response.size(); + if (response.size() > 0) + re << &response; + + boost::asio::read(socket, response, boost::asio::transfer_exactly(response_content_length)); re << &response; + return re.str(); }