diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 0b742fd2369..807260e6d86 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1403,7 +1403,8 @@ struct controller_impl { transaction_trace_ptr push_transaction( const transaction_metadata_ptr& trx, fc::time_point deadline, uint32_t billed_cpu_time_us, - bool explicit_billed_cpu_time ) + bool explicit_billed_cpu_time, + uint32_t subjective_cpu_bill_us ) { EOS_ASSERT(deadline != fc::time_point(), transaction_exception, "deadline cannot be uninitialized"); @@ -1432,6 +1433,7 @@ struct controller_impl { trx_context.deadline = deadline; trx_context.explicit_billed_cpu_time = explicit_billed_cpu_time; trx_context.billed_cpu_time_us = billed_cpu_time_us; + trx_context.subjective_cpu_bill_us = subjective_cpu_bill_us; trace = trx_context.trace; try { if( trx->implicit ) { @@ -1504,6 +1506,7 @@ struct controller_impl { trace->error_code = controller::convert_exception_to_error_code( e ); trace->except = e; trace->except_ptr = std::current_exception(); + trace->elapsed = fc::time_point::now() - trx_context.start; } emit( self.accepted_transaction, trx ); @@ -1645,7 +1648,7 @@ struct controller_impl { in_trx_requiring_checks = old_value; }); in_trx_requiring_checks = true; - push_transaction( onbtrx, fc::time_point::maximum(), self.get_global_properties().configuration.min_transaction_cpu_usage, true ); + push_transaction( onbtrx, fc::time_point::maximum(), self.get_global_properties().configuration.min_transaction_cpu_usage, true, 0 ); } catch( const std::bad_alloc& e ) { elog( "on block transaction failed due to a std::bad_alloc" ); throw; @@ -1903,7 +1906,7 @@ struct controller_impl { : ( !!std::get<0>( trx_metas.at( packed_idx ) ) ? std::get<0>( trx_metas.at( packed_idx ) ) : std::get<1>( trx_metas.at( packed_idx ) ).get() ) ); - trace = push_transaction( trx_meta, fc::time_point::maximum(), receipt.cpu_usage_us, true ); + trace = push_transaction( trx_meta, fc::time_point::maximum(), receipt.cpu_usage_us, true, 0 ); ++packed_idx; } else if( receipt.trx.contains() ) { trace = push_scheduled_transaction( receipt.trx.get(), fc::time_point::maximum(), receipt.cpu_usage_us, true ); @@ -2688,11 +2691,12 @@ void controller::push_block( std::future& block_state_future, } transaction_trace_ptr controller::push_transaction( const transaction_metadata_ptr& trx, fc::time_point deadline, - uint32_t billed_cpu_time_us, bool explicit_billed_cpu_time ) { + uint32_t billed_cpu_time_us, bool explicit_billed_cpu_time, + uint32_t subjective_cpu_bill_us ) { validate_db_available_size(); EOS_ASSERT( get_read_mode() != db_read_mode::IRREVERSIBLE, transaction_type_exception, "push transaction not allowed in irreversible mode" ); EOS_ASSERT( trx && !trx->implicit && !trx->scheduled, transaction_type_exception, "Implicit/Scheduled transaction not allowed" ); - return my->push_transaction(trx, deadline, billed_cpu_time_us, explicit_billed_cpu_time ); + return my->push_transaction(trx, deadline, billed_cpu_time_us, explicit_billed_cpu_time, subjective_cpu_bill_us ); } transaction_trace_ptr controller::push_scheduled_transaction( const transaction_id_type& trxid, fc::time_point deadline, diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index e6bb848bd72..2d08e4239a4 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -144,7 +144,8 @@ namespace eosio { namespace chain { * */ transaction_trace_ptr push_transaction( const transaction_metadata_ptr& trx, fc::time_point deadline, - uint32_t billed_cpu_time_us, bool explicit_billed_cpu_time ); + uint32_t billed_cpu_time_us, bool explicit_billed_cpu_time, + uint32_t subjective_cpu_bill_us ); /** * Attempt to execute a specific transaction in our deferred trx database diff --git a/libraries/chain/include/eosio/chain/resource_limits.hpp b/libraries/chain/include/eosio/chain/resource_limits.hpp index 5e1f3ca8423..9ea8521147a 100644 --- a/libraries/chain/include/eosio/chain/resource_limits.hpp +++ b/libraries/chain/include/eosio/chain/resource_limits.hpp @@ -78,6 +78,8 @@ namespace eosio { namespace chain { namespace resource_limits { bool set_account_limits( const account_name& account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight); void get_account_limits( const account_name& account, int64_t& ram_bytes, int64_t& net_weight, int64_t& cpu_weight) const; + bool is_unlimited_cpu( const account_name& account ) const; + void process_account_limit_updates(); void process_block_usage( uint32_t block_num ); diff --git a/libraries/chain/include/eosio/chain/resource_limits_private.hpp b/libraries/chain/include/eosio/chain/resource_limits_private.hpp index ccaed484ce5..e65de0fcdaf 100644 --- a/libraries/chain/include/eosio/chain/resource_limits_private.hpp +++ b/libraries/chain/include/eosio/chain/resource_limits_private.hpp @@ -120,6 +120,71 @@ namespace eosio { namespace chain { namespace resource_limits { } }; + /** + * This class accumulates a value that decays over quantums based on inputs + * The decay is linear between updates and exponential if the set of inputs has no gaps + * + * The value stored is Precision times the sum of the inputs. + */ + template + struct exponential_decay_accumulator + { + static_assert( Precision > 0, "Precision must be positive" ); + static constexpr uint64_t max_raw_value = std::numeric_limits::max() / Precision; + + exponential_decay_accumulator() + : last_ordinal(0) + , value_ex(0) + { + } + + uint32_t last_ordinal; ///< The ordinal of the last period which has contributed to the accumulator + uint64_t value_ex; ///< The current accumulated value pre-multiplied by Precision + + /** + * return the extended value at a current or future ordinal + */ + uint64_t value_ex_at( uint32_t ordinal, uint32_t window_size ) const { + if( last_ordinal < ordinal ) { + if( (uint64_t)last_ordinal + window_size > (uint64_t)ordinal ) { + const auto delta = ordinal - last_ordinal; // clearly 0 < delta < window_size + const auto decay = make_ratio( + (uint128_t)window_size - delta, + (uint128_t)window_size + ); + + return downgrade_cast((uint128_t)value_ex * decay); + } else { + return 0; + } + } else { + return value_ex; + } + } + + /** + * return the value at a current or future ordinal + */ + uint64_t value_at( uint32_t ordinal, uint32_t window_size ) const { + return integer_divide_ceil(value_ex_at(ordinal, window_size), Precision); + } + + void add( uint64_t units, uint32_t ordinal, uint32_t window_size /* must be positive */ ) + { + // check for some numerical limits before doing any state mutations + EOS_ASSERT(units <= max_raw_value, rate_limiting_state_inconsistent, "Usage exceeds maximum value representable after extending for precision"); + + uint128_t units_ex = (uint128_t)units * Precision; + if (last_ordinal < ordinal) { + value_ex = value_ex_at(ordinal, window_size); + last_ordinal = ordinal; + } + + // saturate the value + uint128_t new_value_ex = std::min(units_ex + (uint128_t)value_ex, std::numeric_limits::max()); + value_ex = downgrade_cast(new_value_ex); + } + }; } using usage_accumulator = impl::exponential_moving_average_accumulator<>; diff --git a/libraries/chain/include/eosio/chain/transaction_context.hpp b/libraries/chain/include/eosio/chain/transaction_context.hpp index a10c7ba9e9a..d9cdff7b5e4 100644 --- a/libraries/chain/include/eosio/chain/transaction_context.hpp +++ b/libraries/chain/include/eosio/chain/transaction_context.hpp @@ -129,6 +129,7 @@ namespace eosio { namespace chain { fc::time_point deadline = fc::time_point::maximum(); fc::microseconds leeway = fc::microseconds( config::default_subjective_cpu_leeway_us ); int64_t billed_cpu_time_us = 0; + uint32_t subjective_cpu_bill_us = 0; bool explicit_billed_cpu_time = false; transaction_checktime_timer transaction_timer; diff --git a/libraries/chain/resource_limits.cpp b/libraries/chain/resource_limits.cpp index e9ec714fabf..d834ac42420 100644 --- a/libraries/chain/resource_limits.cpp +++ b/libraries/chain/resource_limits.cpp @@ -285,6 +285,13 @@ void resource_limits_manager::get_account_limits( const account_name& account, i } } +bool resource_limits_manager::is_unlimited_cpu( const account_name& account ) const { + const auto* buo = _db.find( boost::make_tuple(false, account) ); + if (buo) { + return buo->cpu_weight == -1; + } + return false; +} void resource_limits_manager::process_account_limit_updates() { auto& multi_index = _db.get_mutable_index(); diff --git a/libraries/chain/transaction_context.cpp b/libraries/chain/transaction_context.cpp index 99f5c57c17e..17d09d4a4dc 100644 --- a/libraries/chain/transaction_context.cpp +++ b/libraries/chain/transaction_context.cpp @@ -174,7 +174,9 @@ namespace eosio { namespace chain { if( !explicit_billed_cpu_time ) { // Fail early if amount of the previous speculative execution is within 10% of remaining account cpu available - int64_t validate_account_cpu_limit = account_cpu_limit - EOS_PERCENT( account_cpu_limit, 10 * config::percent_1 ); + int64_t validate_account_cpu_limit = account_cpu_limit - subjective_cpu_bill_us; + if( validate_account_cpu_limit > 0 ) + validate_account_cpu_limit -= EOS_PERCENT( validate_account_cpu_limit, 10 * config::percent_1 ); if( validate_account_cpu_limit < 0 ) validate_account_cpu_limit = 0; validate_account_cpu_usage( billed_cpu_time_us, validate_account_cpu_limit, true ); } diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index ca67bdb0580..ae88d56b737 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -329,7 +329,7 @@ namespace eosio { namespace testing { if( !skip_pending_trxs ) { for( auto itr = unapplied_transactions.begin(); itr != unapplied_transactions.end(); ) { - auto trace = control->push_transaction( itr->trx_meta, fc::time_point::maximum(), DEFAULT_BILLED_CPU_TIME_US, true ); + auto trace = control->push_transaction( itr->trx_meta, fc::time_point::maximum(), DEFAULT_BILLED_CPU_TIME_US, true, 0 ); traces.emplace_back( trace ); if(!no_throw && trace->except) { // this always throws an fc::exception, since the original exception is copied into an fc::exception @@ -549,7 +549,7 @@ namespace eosio { namespace testing { fc::microseconds::maximum() : fc::microseconds( deadline - fc::time_point::now() ); auto fut = transaction_metadata::start_recover_keys( ptrx, control->get_thread_pool(), control->get_chain_id(), time_limit ); - auto r = control->push_transaction( fut.get(), deadline, billed_cpu_time_us, billed_cpu_time_us > 0 ); + auto r = control->push_transaction( fut.get(), deadline, billed_cpu_time_us, billed_cpu_time_us > 0, 0 ); if( r->except_ptr ) std::rethrow_exception( r->except_ptr ); if( r->except ) throw *r->except; return r; @@ -574,7 +574,7 @@ namespace eosio { namespace testing { fc::microseconds( deadline - fc::time_point::now() ); auto ptrx = std::make_shared( trx, c ); auto fut = transaction_metadata::start_recover_keys( ptrx, control->get_thread_pool(), control->get_chain_id(), time_limit ); - auto r = control->push_transaction( fut.get(), deadline, billed_cpu_time_us, billed_cpu_time_us > 0 ); + auto r = control->push_transaction( fut.get(), deadline, billed_cpu_time_us, billed_cpu_time_us > 0, 0 ); if (no_throw) return r; if( r->except_ptr ) std::rethrow_exception( r->except_ptr ); if( r->except) throw *r->except; diff --git a/plugins/chain_plugin/account_query_db.cpp b/plugins/chain_plugin/account_query_db.cpp index 28d7b546902..eaea2214e77 100644 --- a/plugins/chain_plugin/account_query_db.cpp +++ b/plugins/chain_plugin/account_query_db.cpp @@ -22,13 +22,13 @@ using namespace boost::bimaps; namespace { /** * Structure to hold indirect reference to a `property_object` via {owner,name} as well as a non-standard - * index over `last_updated` for roll-back support + * index over `last_updated_height` (which is truncated at the LIB during initialization) for roll-back support */ struct permission_info { // indexed data chain::name owner; chain::name name; - fc::time_point last_updated; + uint32_t last_updated_height; // un-indexed data uint32_t threshold; @@ -37,10 +37,10 @@ namespace { }; struct by_owner_name; - struct by_last_updated; + struct by_last_updated_height; /** - * Multi-index providing fast lookup for {owner,name} as well as {last_updated} + * Multi-index providing fast lookup for {owner,name} as well as {last_updated_height} */ using permission_info_index_t = multi_index_container< permission_info, @@ -53,8 +53,8 @@ namespace { > >, ordered_non_unique< - tag, - member + tag, + member > > >; @@ -143,8 +143,19 @@ namespace eosio::chain_apis { auto start = fc::time_point::now(); const auto& index = controller.db().get_index().indices().get(); + // build a initial time to block number map + const auto lib_num = controller.last_irreversible_block_num(); + const auto head_num = controller.head_block_num(); + + for (uint32_t block_num = lib_num + 1; block_num <= head_num; block_num++) { + const auto block_p = controller.fetch_block_by_number(block_num); + EOS_ASSERT(block_p, chain::plugin_exception, "cannot fetch reversible block ${block_num}, required for account_db initialization", ("block_num", block_num)); + time_to_block_num.emplace(block_p->timestamp.to_time_point(), block_num); + } + for (const auto& po : index ) { - const auto& pi = permission_info_index.emplace( permission_info{ po.owner, po.name, po.last_updated, po.auth.threshold } ).first; + uint32_t last_updated_height = last_updated_time_to_height(po.last_updated); + const auto& pi = permission_info_index.emplace( permission_info{ po.owner, po.name, last_updated_height, po.auth.threshold } ).first; add_to_bimaps(*pi, po); } auto duration = fc::time_point::now() - start; @@ -185,14 +196,14 @@ namespace eosio::chain_apis { bool is_rollback_required( const chain::block_state_ptr& bsp ) const { std::shared_lock read_lock(rw_mutex); - const auto t = bsp->block->timestamp.to_time_point(); - const auto& index = permission_info_index.get(); + const auto bnum = bsp->block->block_num(); + const auto& index = permission_info_index.get(); if (index.empty()) { return false; } else { const auto& pi = (*index.rbegin()); - if (pi.last_updated < t) { + if (pi.last_updated_height < bnum) { return false; } } @@ -200,22 +211,42 @@ namespace eosio::chain_apis { return true; } + uint32_t last_updated_time_to_height( const fc::time_point& last_updated) { + const auto lib_num = controller.last_irreversible_block_num(); + const auto lib_time = controller.last_irreversible_block_time(); + + uint32_t last_updated_height = lib_num; + if (last_updated > lib_time) { + const auto iter = time_to_block_num.find(last_updated); + EOS_ASSERT(iter != time_to_block_num.end(), chain::plugin_exception, "invalid block time encountered in on-chain accounts ${time}", ("time", last_updated)); + last_updated_height = iter->second; + } + + return last_updated_height; + } + /** - * Given a time_point, remove all permissions that were last updated at or after that time_point - * this will effectively remove any updates that happened at or after that time point + * Given a block number, remove all permissions that were last updated at or after that block number + * this will effectively roll back the database to just before the incoming block * * For each removed entry, this will create a new entry if there exists an equivalent {owner, name} permission * at the HEAD state of the chain. * @param bsp - the block to rollback before */ void rollback_to_before( const chain::block_state_ptr& bsp ) { - const auto t = bsp->block->timestamp.to_time_point(); - auto& index = permission_info_index.get(); + const auto bnum = bsp->block->block_num(); + auto& index = permission_info_index.get(); const auto& permission_by_owner = controller.db().get_index().indices().get(); + // roll back time-map + auto time_iter = time_to_block_num.rbegin(); + while (time_iter != time_to_block_num.rend() && time_iter->second >= bnum) { + time_iter = decltype(time_iter){time_to_block_num.erase( std::next(time_iter).base() )}; + } + while (!index.empty()) { const auto& pi = (*index.rbegin()); - if (pi.last_updated < t) { + if (pi.last_updated_height < bnum) { break; } @@ -228,8 +259,11 @@ namespace eosio::chain_apis { index.erase(index.iterator_to(pi)); } else { const auto& po = *itr; - index.modify(index.iterator_to(pi), [&po](auto& mutable_pi) { - mutable_pi.last_updated = po.last_updated; + + uint32_t last_updated_height = po.last_updated == bsp->header.timestamp ? bsp->block_num : last_updated_time_to_height(po.last_updated); + + index.modify(index.iterator_to(pi), [&po, last_updated_height](auto& mutable_pi) { + mutable_pi.last_updated_height = last_updated_height; mutable_pi.threshold = po.auth.threshold; }); add_to_bimaps(pi, po); @@ -331,6 +365,11 @@ namespace eosio::chain_apis { std::unique_lock write_lock(rw_mutex); rollback_to_before(bsp); + + // insert this blocks time into the time map + time_to_block_num.emplace(bsp->header.timestamp, bsp->block_num); + + const auto bnum = bsp->block_num; auto& index = permission_info_index.get(); const auto& permission_by_owner = controller.db().get_index().indices().get(); @@ -342,11 +381,11 @@ namespace eosio::chain_apis { auto itr = index.find(key); if (itr == index.end()) { const auto& po = *source_itr; - itr = index.emplace(permission_info{ po.owner, po.name, po.last_updated, po.auth.threshold }).first; + itr = index.emplace(permission_info{ po.owner, po.name, bnum, po.auth.threshold }).first; } else { remove_from_bimaps(*itr); index.modify(itr, [&](auto& mutable_pi){ - mutable_pi.last_updated = source_itr->last_updated; + mutable_pi.last_updated_height = bnum; mutable_pi.threshold = source_itr->auth.threshold; }); } @@ -440,6 +479,10 @@ namespace eosio::chain_apis { cached_trace_map_t cached_trace_map; ///< temporary cache of uncommitted traces onblock_trace_t onblock_trace; ///< temporary cache of on_block trace + using time_map_t = std::map; + time_map_t time_to_block_num; + + using name_bimap_t = bimap>, multiset_of>; using key_bimap_t = bimap>, multiset_of>; diff --git a/plugins/chain_plugin/test/test_account_query_db.cpp b/plugins/chain_plugin/test/test_account_query_db.cpp index 36cceb67647..e0ef730f839 100644 --- a/plugins/chain_plugin/test/test_account_query_db.cpp +++ b/plugins/chain_plugin/test/test_account_query_db.cpp @@ -40,26 +40,26 @@ BOOST_AUTO_TEST_SUITE(account_query_db_tests) BOOST_FIXTURE_TEST_CASE(newaccount_test, TESTER) { try { - // instantiate an account_query_db - auto aq_db = account_query_db(*control); + // instantiate an account_query_db + auto aq_db = account_query_db(*control); //link aq_db to the `accepted_block` signal on the controller - auto c2 = control->accepted_block.connect([&](const block_state_ptr& blk) { + auto c2 = control->accepted_block.connect([&](const block_state_ptr& blk) { aq_db.commit_block( blk); - }); + }); - produce_blocks(10); + produce_blocks(10); - account_name tester_account = N(tester); - const auto trace_ptr = create_account(tester_account); - aq_db.cache_transaction_trace(trace_ptr); - produce_block(); + account_name tester_account = N(tester); + const auto trace_ptr = create_account(tester_account); + aq_db.cache_transaction_trace(trace_ptr); + produce_block(); - params pars; - pars.keys.emplace_back(get_public_key(tester_account, "owner")); - const auto results = aq_db.get_accounts_by_authorizers(pars); + params pars; + pars.keys.emplace_back(get_public_key(tester_account, "owner")); + const auto results = aq_db.get_accounts_by_authorizers(pars); - BOOST_TEST_REQUIRE(find_account_name(results, tester_account) == true); + BOOST_TEST_REQUIRE(find_account_name(results, tester_account) == true); } FC_LOG_AND_RETHROW() } @@ -75,25 +75,72 @@ BOOST_FIXTURE_TEST_CASE(updateauth_test, TESTER) { try { produce_blocks(10); - const auto& tester_account = N(tester); - const string role = "first"; - produce_block(); - create_account(tester_account); - - const auto trace_ptr = push_action(config::system_account_name, updateauth::get_name(), tester_account, fc::mutable_variant_object() - ("account", tester_account) - ("permission", N(role)) - ("parent", "active") - ("auth", authority(get_public_key(tester_account, role), 5)) - ); - aq_db.cache_transaction_trace(trace_ptr); - produce_block(); - - params pars; - pars.keys.emplace_back(get_public_key(tester_account, role)); - const auto results = aq_db.get_accounts_by_authorizers(pars); - - BOOST_TEST_REQUIRE(find_account_auth(results, tester_account, N(role)) == true); + const auto& tester_account = N(tester); + const string role = "first"; + produce_block(); + create_account(tester_account); + + const auto trace_ptr = push_action(config::system_account_name, updateauth::get_name(), tester_account, fc::mutable_variant_object() + ("account", tester_account) + ("permission", N(role)) + ("parent", "active") + ("auth", authority(get_public_key(tester_account, role), 5)) + ); + aq_db.cache_transaction_trace(trace_ptr); + produce_block(); + + params pars; + pars.keys.emplace_back(get_public_key(tester_account, role)); + const auto results = aq_db.get_accounts_by_authorizers(pars); + + BOOST_TEST_REQUIRE(find_account_auth(results, tester_account, N(role)) == true); + +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_CASE(future_fork_test) { try { + tester node_a(setup_policy::none); + tester node_b(setup_policy::none); + + // instantiate an account_query_db + auto aq_db = account_query_db(*node_a.control); + + //link aq_db to the `accepted_block` signal on the controller + auto c = node_a.control->accepted_block.connect([&](const block_state_ptr& blk) { + aq_db.commit_block( blk); + }); + + // create 10 blocks synced + for (int i = 0; i < 10; i++) { + node_b.push_block(node_a.produce_block()); + } + + // produce a block on node A with a new account and permission + const auto& tester_account = N(tester); + const string role = "first"; + node_a.create_account(tester_account); + + const auto trace_ptr = node_a.push_action(config::system_account_name, updateauth::get_name(), tester_account, fc::mutable_variant_object() + ("account", tester_account) + ("permission", N(role)) + ("parent", "active") + ("auth", authority(node_a.get_public_key(tester_account, role), 5)) + ); + aq_db.cache_transaction_trace(trace_ptr); + node_a.produce_block(); + + params pars; + pars.keys.emplace_back(node_a.get_public_key(tester_account, role)); + + const auto pre_results = aq_db.get_accounts_by_authorizers(pars); + BOOST_TEST_REQUIRE(find_account_auth(pre_results, tester_account, N(role)) == true); + + // have node B take over from head-1 and produce "future" blocks to overtake + node_a.push_block(node_b.produce_block(fc::milliseconds(config::block_interval_ms * 100))); + node_a.push_block(node_b.produce_block()); + + // ensure the account was forked away + const auto post_results = aq_db.get_accounts_by_authorizers(pars); + BOOST_TEST_REQUIRE(post_results.accounts.size() == 0); } FC_LOG_AND_RETHROW() } diff --git a/plugins/producer_plugin/CMakeLists.txt b/plugins/producer_plugin/CMakeLists.txt index fe161f8d913..4e5c23efe58 100644 --- a/plugins/producer_plugin/CMakeLists.txt +++ b/plugins/producer_plugin/CMakeLists.txt @@ -8,3 +8,6 @@ add_library( producer_plugin target_link_libraries( producer_plugin chain_plugin http_client_plugin appbase eosio_chain ) target_include_directories( producer_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../chain_interface/include" ) + +add_subdirectory( test ) + diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp new file mode 100644 index 00000000000..0be8de26983 --- /dev/null +++ b/plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp @@ -0,0 +1,205 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace eosio { + +namespace bmi = boost::multi_index; +using chain::transaction_id_type; +using chain::account_name; +using chain::block_state_ptr; +using chain::packed_transaction; +namespace config = chain::config; + +class subjective_billing { +private: + + struct trx_cache_entry { + transaction_id_type trx_id; + account_name account; + uint32_t subjective_cpu_bill; + fc::time_point expiry; + }; + struct by_id; + struct by_expiry; + + using trx_cache_index = bmi::multi_index_container< + trx_cache_entry, + indexed_by< + bmi::hashed_unique, BOOST_MULTI_INDEX_MEMBER( trx_cache_entry, transaction_id_type, trx_id ) >, + ordered_non_unique, BOOST_MULTI_INDEX_MEMBER( trx_cache_entry, fc::time_point, expiry ) > + > + >; + + using decaying_accumulator = chain::resource_limits::impl::exponential_decay_accumulator<>; + + struct subjective_billing_info { + uint64_t pending_cpu_us; // tracked cpu us for transactions that may still succeed in a block + decaying_accumulator expired_accumulator; // accumulator used to account for transactions that have expired + + bool empty(uint32_t time_ordinal) { + return pending_cpu_us == 0 && expired_accumulator.value_at(time_ordinal, expired_accumulator_average_window) == 0; + } + }; + + using account_subjective_bill_cache = std::map; + using block_subjective_bill_cache = std::map; + + bool _disabled = false; + trx_cache_index _trx_cache_index; + account_subjective_bill_cache _account_subjective_bill_cache; + block_subjective_bill_cache _block_subjective_bill_cache; + +private: + uint32_t time_ordinal_for( const fc::time_point& t ) const { + auto ordinal = t.time_since_epoch().count() / (1000U * (uint64_t)subjective_time_interval_ms); + EOS_ASSERT(ordinal <= std::numeric_limits::max(), chain::tx_resource_exhaustion, "overflow of quantized time in subjective billing"); + return ordinal; + } + + void remove_subjective_billing( const trx_cache_entry& entry, uint32_t time_ordinal ) { + auto aitr = _account_subjective_bill_cache.find( entry.account ); + if( aitr != _account_subjective_bill_cache.end() ) { + aitr->second.pending_cpu_us -= entry.subjective_cpu_bill; + EOS_ASSERT( aitr->second.pending_cpu_us >= 0, chain::tx_resource_exhaustion, + "Logic error in subjective account billing ${a}", ("a", entry.account) ); + if( aitr->second.empty(time_ordinal) ) _account_subjective_bill_cache.erase( aitr ); + } + } + + void transition_to_expired( const trx_cache_entry& entry, uint32_t time_ordinal ) { + auto aitr = _account_subjective_bill_cache.find( entry.account ); + if( aitr != _account_subjective_bill_cache.end() ) { + aitr->second.pending_cpu_us -= entry.subjective_cpu_bill; + aitr->second.expired_accumulator.add(entry.subjective_cpu_bill, time_ordinal, expired_accumulator_average_window); + } + } + + void remove_subjective_billing( const block_state_ptr& bsp, uint32_t time_ordinal ) { + if( !_trx_cache_index.empty() ) { + for( const auto& receipt : bsp->block->transactions ) { + if( receipt.trx.contains() ) { + const auto& pt = receipt.trx.get(); + remove_subjective_billing( pt.id(), time_ordinal ); + } + } + } + } + +public: // public for tests + static constexpr uint32_t subjective_time_interval_ms = 5'000; + static constexpr uint32_t expired_accumulator_average_window = config::account_cpu_usage_average_window_ms / subjective_time_interval_ms; + + void remove_subjective_billing( const transaction_id_type& trx_id, uint32_t time_ordinal ) { + auto& idx = _trx_cache_index.get(); + auto itr = idx.find( trx_id ); + if( itr != idx.end() ) { + remove_subjective_billing( *itr, time_ordinal ); + idx.erase( itr ); + } + } + +public: + void disable() { _disabled = true; } + + /// @param in_pending_block pass true if pt's bill time is accounted for in the pending block + void subjective_bill( const transaction_id_type& id, const fc::time_point& expire, const account_name& first_auth, + const fc::microseconds& elapsed, bool in_pending_block ) + { + if( !_disabled ) { + uint32_t bill = std::max( 0, elapsed.count() ); + auto p = _trx_cache_index.emplace( + trx_cache_entry{id, + first_auth, + bill, + expire} ); + if( p.second ) { + _account_subjective_bill_cache[first_auth].pending_cpu_us += bill; + if( in_pending_block ) { + _block_subjective_bill_cache[first_auth] += bill; + } + } + } + } + + void subjective_bill_failure( const account_name& first_auth, const fc::microseconds& elapsed, const fc::time_point& now ) + { + if( !_disabled ) { + uint32_t bill = std::max( 0, elapsed.count() ); + const auto time_ordinal = time_ordinal_for(now); + _account_subjective_bill_cache[first_auth].expired_accumulator.add(bill, time_ordinal, expired_accumulator_average_window); + } + } + + uint32_t get_subjective_bill( const account_name& first_auth, const fc::time_point& now ) const { + if( _disabled ) return 0; + const auto time_ordinal = time_ordinal_for(now); + const subjective_billing_info* sub_bill_info = nullptr; + auto aitr = _account_subjective_bill_cache.find( first_auth ); + if( aitr != _account_subjective_bill_cache.end() ) { + sub_bill_info = &aitr->second; + } + uint64_t in_block_pending_cpu_us = 0; + auto bitr = _block_subjective_bill_cache.find( first_auth ); + if( bitr != _block_subjective_bill_cache.end() ) { + in_block_pending_cpu_us = bitr->second; + } + + if (sub_bill_info) { + EOS_ASSERT(sub_bill_info->pending_cpu_us >= in_block_pending_cpu_us, chain::tx_resource_exhaustion, "Logic error subjective billing ${a}", ("a", first_auth) ); + uint32_t sub_bill = sub_bill_info->pending_cpu_us - in_block_pending_cpu_us + sub_bill_info->expired_accumulator.value_at(time_ordinal, expired_accumulator_average_window ); + return sub_bill; + } else { + return 0; + } + } + + void abort_block() { + _block_subjective_bill_cache.clear(); + } + + void on_block( const block_state_ptr& bsp, const fc::time_point& now ) { + if( bsp == nullptr ) return; + const auto time_ordinal = time_ordinal_for(now); + remove_subjective_billing( bsp, time_ordinal ); + } + + bool remove_expired( fc::logger& log, const fc::time_point& pending_block_time, const fc::time_point& now, const fc::time_point& deadline ) { + bool exhausted = false; + auto& idx = _trx_cache_index.get(); + if( !idx.empty() ) { + const auto time_ordinal = time_ordinal_for(now); + const auto orig_count = _trx_cache_index.size(); + uint32_t num_expired = 0; + + while( !idx.empty() ) { + if( deadline <= fc::time_point::now() ) { + exhausted = true; + break; + } + auto b = idx.begin(); + if( b->expiry > pending_block_time ) break; + transition_to_expired( *b, time_ordinal ); + idx.erase( b ); + num_expired++; + } + + fc_dlog( log, "Processed ${n} subjective billed transactions, Expired ${expired}", + ("n", orig_count)( "expired", num_expired ) ); + } + return !exhausted; + } +}; + +} //eosio diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 5a738957cc8..b33e3264cd8 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -174,6 +175,7 @@ enum class pending_block_mode { speculating }; + class producer_plugin_impl : public std::enable_shared_from_this { public: producer_plugin_impl(boost::asio::io_service& io) @@ -234,6 +236,7 @@ class producer_plugin_impl : public std::enable_shared_from_this _accepted_block_connection; fc::optional _accepted_block_header_connection; @@ -277,6 +280,7 @@ class producer_plugin_impl : public std::enable_shared_from_this - auto publish_results_of(const Type &data, Channel& channel, F f) { - auto publish_success = fc::make_scoped_exit([&, this](){ - channel.publish(std::pair(nullptr, data)); - }); + void abort_block() { + auto& chain = chain_plug->chain(); - try { - auto trace = f(); - if (trace->except) { - publish_success.cancel(); - channel.publish(std::pair(trace->except->dynamic_copy_exception(), data)); - } - return trace; - } catch (const fc::exception& e) { - publish_success.cancel(); - channel.publish(std::pair(e.dynamic_copy_exception(), data)); - throw e; - } catch( const std::exception& e ) { - publish_success.cancel(); - auto fce = fc::exception( - FC_LOG_MESSAGE( info, "Caught std::exception: ${what}", ("what",e.what())), - fc::std_exception_code, - BOOST_CORE_TYPEID(e).name(), - e.what() - ); - channel.publish(std::pair(fce.dynamic_copy_exception(),data)); - throw fce; - } catch( ... ) { - publish_success.cancel(); - auto fce = fc::unhandled_exception( - FC_LOG_MESSAGE( info, "Caught unknown exception"), - std::current_exception() - ); - - channel.publish(std::pair(fce.dynamic_copy_exception(), data)); - throw fce; - } - }; + _unapplied_transactions.add_aborted( chain.abort_block() ); + _subjective_billing.abort_block(); + } bool on_incoming_block(const signed_block_ptr& block, const std::optional& block_id) { auto& chain = chain_plug->chain(); @@ -366,7 +338,7 @@ class producer_plugin_impl : public std::enable_shared_from_thisid(); fc::time_point bt = chain.is_building_block() ? chain.pending_block_time() : chain.head_block_time(); - if( fc::time_point( trx->packed_trx()->expiration()) < bt ) { + const fc::time_point expire = trx->packed_trx()->expiration(); + if( expire < bt ) { send_response( std::static_pointer_cast( std::make_shared( FC_LOG_MESSAGE( error, "expired transaction ${id}, expiration ${e}, block time ${bt}", - ("id", id)("e", trx->packed_trx()->expiration())( "bt", bt ))))); + ("id", id)("e", expire)( "bt", bt ))))); return true; } @@ -565,7 +538,12 @@ class producer_plugin_impl : public std::enable_shared_from_thisbilled_cpu_time_us, false ); + auto first_auth = trx->packed_trx()->get_transaction().first_authorizer(); + uint32_t sub_bill = 0; + if( _pending_block_mode != pending_block_mode::producing) + sub_bill = _subjective_billing.get_subjective_bill( first_auth, fc::time_point::now() ); + + auto trace = chain.push_transaction( trx, deadline, trx->billed_cpu_time_us, false, sub_bill ); if( trace->except ) { if( exception_is_exhausted( *trace->except, deadline_is_subjective )) { _pending_incoming_transactions.add( trx, persist_until_expired, next ); @@ -578,9 +556,9 @@ class producer_plugin_impl : public std::enable_shared_from_thisid())); } - if( !exhausted ) - exhausted = block_is_exhausted(); + exhausted = block_is_exhausted(); } else { + _subjective_billing.subjective_bill_failure( first_auth, trace->elapsed, fc::time_point::now() ); auto e_ptr = trace->except->dynamic_copy_exception(); send_response( e_ptr ); } @@ -588,7 +566,12 @@ class producer_plugin_impl : public std::enable_shared_from_thisid(), expire, first_auth, trace->elapsed, + chain.get_read_mode() == chain::db_read_mode::SPECULATIVE ); } send_response( trace ); } @@ -730,6 +713,8 @@ void producer_plugin::set_program_options( "Maximum size (in MiB) of the incoming transaction queue. Exceeding this value will subjectively drop transaction with resource exhaustion.") ("disable-api-persisted-trx", bpo::bool_switch()->default_value(false), "Disable the re-apply of API transactions.") + ("disable-subjective-billing", bpo::bool_switch()->default_value(false), + "Disable subjective billing.") ("producer-threads", bpo::value()->default_value(config::default_controller_thread_pool_size), "Number of worker threads in producer thread pool") ("snapshots-dir", bpo::value()->default_value("snapshots"), @@ -914,6 +899,7 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ my->_incoming_defer_ratio = options.at("incoming-defer-ratio").as(); my->_disable_persist_until_expired = options.at("disable-api-persisted-trx").as(); + if( options.at("disable-subjective-billing").as() ) my->_subjective_billing.disable(); auto thread_pool_size = options.at( "producer-threads" ).as(); EOS_ASSERT( thread_pool_size > 0, plugin_config_exception, @@ -1055,8 +1041,7 @@ void producer_plugin::resume() { // re-evaluate that now // if (my->_pending_block_mode == pending_block_mode::speculating) { - chain::controller& chain = my->chain_plug->chain(); - my->_unapplied_transactions.add_aborted( chain.abort_block() ); + my->abort_block(); fc_ilog(_log, "Producer resumed. Scheduling production."); my->schedule_production_loop(); } else { @@ -1098,7 +1083,7 @@ void producer_plugin::update_runtime_options(const runtime_options& options) { } if (check_speculating && my->_pending_block_mode == pending_block_mode::speculating) { - my->_unapplied_transactions.add_aborted( chain.abort_block() ); + my->abort_block(); my->schedule_production_loop(); } @@ -1182,7 +1167,7 @@ producer_plugin::integrity_hash_information producer_plugin::get_integrity_hash( if (chain.is_building_block()) { // abort the pending block - my->_unapplied_transactions.add_aborted( chain.abort_block() ); + my->abort_block(); } else { reschedule.cancel(); } @@ -1211,7 +1196,7 @@ void producer_plugin::create_snapshot(producer_plugin::next_function_unapplied_transactions.add_aborted( chain.abort_block() ); + my->abort_block(); } else { reschedule.cancel(); } @@ -1567,7 +1552,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { blocks_to_confirm = (uint16_t)(std::min(blocks_to_confirm, (uint32_t)(hbs->block_num - hbs->dpos_irreversible_blocknum))); } - _unapplied_transactions.add_aborted( chain.abort_block() ); + abort_block(); auto features_to_activate = chain.get_preactivated_protocol_features(); if( _pending_block_mode == pending_block_mode::producing && _protocol_features_to_activate.size() > 0 ) { @@ -1621,6 +1606,8 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { return start_block_result::exhausted; if( !remove_expired_blacklisted_trxs( preprocess_deadline ) ) return start_block_result::exhausted; + if( !_subjective_billing.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(), preprocess_deadline ) ) + return start_block_result::exhausted; // limit execution of pending incoming to once per block size_t pending_incoming_process_limit = _pending_incoming_transactions.size(); @@ -1818,6 +1805,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin if( !_unapplied_transactions.empty() ) { account_failures account_fails; chain::controller& chain = chain_plug->chain(); + const auto& rl = chain.get_resource_limits_manager(); int num_applied = 0, num_failed = 0, num_processed = 0; auto unapplied_trxs_size = _unapplied_transactions.size(); auto itr = (_pending_block_mode == pending_block_mode::producing) ? @@ -1835,14 +1823,16 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin try { auto start = fc::time_point::now(); auto trx_deadline = start + fc::milliseconds( _max_transaction_time_ms ); - if( account_fails.failure_limit( trx->packed_trx()->get_transaction().first_authorizer() ) ) { + + auto first_auth = trx->packed_trx()->get_transaction().first_authorizer(); + if( account_fails.failure_limit( first_auth ) ) { ++num_failed; itr = _unapplied_transactions.erase( itr ); continue; } auto prev_billed_cpu_time_us = trx->billed_cpu_time_us; - if( prev_billed_cpu_time_us > 0 ) { + if( prev_billed_cpu_time_us > 0 && !rl.is_unlimited_cpu( first_auth )) { auto prev_billed_plus100 = prev_billed_cpu_time_us + EOS_PERCENT( prev_billed_cpu_time_us, 100 * config::percent_1 ); auto trx_dl = start + fc::microseconds( prev_billed_plus100 ); if( trx_dl < trx_deadline ) trx_deadline = trx_dl; @@ -1853,8 +1843,10 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin deadline_is_subjective = true; trx_deadline = deadline; } + // no subjective billing since we are producing or processing persisted trxs + const uint32_t sub_bill = 0; - auto trace = chain.push_transaction( trx, trx_deadline, prev_billed_cpu_time_us, false ); + auto trace = chain.push_transaction( trx, trx_deadline, prev_billed_cpu_time_us, false, sub_bill ); if( trace->except ) { if( exception_is_exhausted( *trace->except, deadline_is_subjective ) ) { if( block_is_exhausted() ) { @@ -1864,16 +1856,22 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin } } else { auto failure_code = trace->except->code(); - // this failed our configured maximum transaction time, we don't want to replay it - fc_dlog( _log, "Failed ${c} trx, prev billed: ${p}us, ran: ${r}us, id: ${id}", - ("c", trace->except->code())("p", prev_billed_cpu_time_us) - ("r", fc::time_point::now() - start)("id", trx->id()) ); - account_fails.add( trx->packed_trx()->get_transaction().first_authorizer(), failure_code ); + if( failure_code != tx_duplicate::code_value ) { + // this failed our configured maximum transaction time, we don't want to replay it + fc_dlog( _log, "Failed ${c} trx, prev billed: ${p}us, ran: ${r}us, id: ${id}", + ("c", trace->except->code())("p", prev_billed_cpu_time_us) + ("r", fc::time_point::now() - start)("id", trx->id()) ); + account_fails.add( first_auth, failure_code ); + _subjective_billing.subjective_bill_failure( first_auth, trace->elapsed, fc::time_point::now() ); + } ++num_failed; itr = _unapplied_transactions.erase( itr ); continue; } } else { + // if db_read_mode SPECULATIVE then trx is in the pending block and not immediately reverted + _subjective_billing.subjective_bill( trx->id(), trx->packed_trx()->expiration(), first_auth, trace->elapsed, + chain.get_read_mode() == chain::db_read_mode::SPECULATIVE ); ++num_applied; itr = _unapplied_transactions.erase( itr ); continue; @@ -2157,8 +2155,7 @@ bool producer_plugin_impl::maybe_produce_block() { } LOG_AND_DROP(); fc_dlog(_log, "Aborting block due to produce_block error"); - chain::controller& chain = chain_plug->chain(); - _unapplied_transactions.add_aborted( chain.abort_block() ); + abort_block(); return false; } diff --git a/plugins/producer_plugin/test/CMakeLists.txt b/plugins/producer_plugin/test/CMakeLists.txt new file mode 100644 index 00000000000..689dca3981f --- /dev/null +++ b/plugins/producer_plugin/test/CMakeLists.txt @@ -0,0 +1,5 @@ +add_executable( test_subjective_billing test_subjective_billing.cpp ) +target_link_libraries( test_subjective_billing producer_plugin eosio_testing ) + +add_test(NAME test_subjective_billing COMMAND plugins/producer_plugin/test/test_subjective_billing WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + diff --git a/plugins/producer_plugin/test/test_subjective_billing.cpp b/plugins/producer_plugin/test/test_subjective_billing.cpp new file mode 100644 index 00000000000..dfccb54973d --- /dev/null +++ b/plugins/producer_plugin/test/test_subjective_billing.cpp @@ -0,0 +1,165 @@ +#define BOOST_TEST_MODULE subjective_billing +#include + +#include + +#include + +namespace { + +using namespace eosio; +using namespace eosio::chain; + +BOOST_AUTO_TEST_SUITE( subjective_billing_test ) + +BOOST_AUTO_TEST_CASE( subjective_bill_test ) { + + fc::logger log; + + transaction_id_type id1 = sha256::hash( "1" ); + transaction_id_type id2 = sha256::hash( "2" ); + transaction_id_type id3 = sha256::hash( "3" ); + transaction_id_type id4 = sha256::hash( "4" ); + transaction_id_type id5 = sha256::hash( "5" ); + transaction_id_type id6 = sha256::hash( "6" ); + account_name a = N("a"); + account_name b = N("b"); + account_name c = N("c"); + + const auto now = time_point::now(); + const auto halftime = now + fc::milliseconds(subjective_billing::expired_accumulator_average_window * subjective_billing::subjective_time_interval_ms / 2); + const auto endtime = now + fc::milliseconds(subjective_billing::expired_accumulator_average_window * subjective_billing::subjective_time_interval_ms); + + + { // Failed transactions remain until expired in subjective billing. + subjective_billing sub_bill; + + sub_bill.subjective_bill( id1, now, a, fc::microseconds( 13 ), false ); + sub_bill.subjective_bill( id2, now, a, fc::microseconds( 11 ), false ); + sub_bill.subjective_bill( id3, now, b, fc::microseconds( 9 ), false ); + + BOOST_CHECK_EQUAL( 13+11, sub_bill.get_subjective_bill(a, now) ); + BOOST_CHECK_EQUAL( 9, sub_bill.get_subjective_bill(b, now) ); + + sub_bill.on_block({}, now); + sub_bill.abort_block(); // they all failed so nothing in aborted block + + BOOST_CHECK_EQUAL( 13+11, sub_bill.get_subjective_bill(a, now) ); + BOOST_CHECK_EQUAL( 9, sub_bill.get_subjective_bill(b, now) ); + + // expires transactions but leaves them in the decay at full value + sub_bill.remove_expired( log, now + fc::microseconds(1), now, fc::time_point::maximum() ); + + BOOST_CHECK_EQUAL( 13+11, sub_bill.get_subjective_bill(a, now) ); + BOOST_CHECK_EQUAL( 9, sub_bill.get_subjective_bill(b, now) ); + BOOST_CHECK_EQUAL( 0, sub_bill.get_subjective_bill(c, now) ); + + // ensure that the value decays away at the window + BOOST_CHECK_EQUAL( 0, sub_bill.get_subjective_bill(a, endtime) ); + BOOST_CHECK_EQUAL( 0, sub_bill.get_subjective_bill(b, endtime) ); + BOOST_CHECK_EQUAL( 0, sub_bill.get_subjective_bill(c, endtime) ); + } + { // db_read_mode HEAD mode, so transactions are immediately reverted + subjective_billing sub_bill; + + sub_bill.subjective_bill( id1, now, a, fc::microseconds( 23 ), false ); + sub_bill.subjective_bill( id2, now, a, fc::microseconds( 19 ), false ); + sub_bill.subjective_bill( id3, now, b, fc::microseconds( 7 ), false ); + + BOOST_CHECK_EQUAL( 23+19, sub_bill.get_subjective_bill(a, now) ); + BOOST_CHECK_EQUAL( 7, sub_bill.get_subjective_bill(b, now) ); + + sub_bill.on_block({}, now); // have not seen any of the transactions come back yet + sub_bill.abort_block(); + + BOOST_CHECK_EQUAL( 23+19, sub_bill.get_subjective_bill(a, now) ); + BOOST_CHECK_EQUAL( 7, sub_bill.get_subjective_bill(b, now) ); + + sub_bill.on_block({}, now); + sub_bill.remove_subjective_billing( id1, 0 ); // simulate seeing id1 come back in block (this is what on_block would do) + sub_bill.abort_block(); + + BOOST_CHECK_EQUAL( 19, sub_bill.get_subjective_bill(a, now) ); + BOOST_CHECK_EQUAL( 7, sub_bill.get_subjective_bill(b, now) ); + } + { // db_read_mode SPECULATIVE mode, so transactions are in pending block until aborted + subjective_billing sub_bill; + + sub_bill.subjective_bill( id1, now, a, fc::microseconds( 23 ), true ); + sub_bill.subjective_bill( id2, now, a, fc::microseconds( 19 ), false ); // trx outside of block + sub_bill.subjective_bill( id3, now, a, fc::microseconds( 55 ), true ); + sub_bill.subjective_bill( id4, now, b, fc::microseconds( 3 ), true ); + sub_bill.subjective_bill( id5, now, b, fc::microseconds( 7 ), true ); + sub_bill.subjective_bill( id6, now, a, fc::microseconds( 11 ), false ); // trx outside of block + + BOOST_CHECK_EQUAL( 19+11, sub_bill.get_subjective_bill(a, now) ); // should not include what is in the pending block + BOOST_CHECK_EQUAL( 0, sub_bill.get_subjective_bill(b, now) ); // should not include what is in the pending block + BOOST_CHECK_EQUAL( 0, sub_bill.get_subjective_bill(c, now) ); + + sub_bill.on_block({}, now); // have not seen any of the transactions come back yet + sub_bill.abort_block(); // aborts the pending block, so subjective billing needs to include the reverted trxs + + BOOST_CHECK_EQUAL( 23+19+55+11, sub_bill.get_subjective_bill(a, now) ); + BOOST_CHECK_EQUAL( 3+7, sub_bill.get_subjective_bill(b, now) ); + + sub_bill.on_block({}, now); + sub_bill.remove_subjective_billing( id3, 0 ); // simulate seeing id3 come back in block (this is what on_block would do) + sub_bill.remove_subjective_billing( id4, 0 ); // simulate seeing id4 come back in block (this is what on_block would do) + sub_bill.abort_block(); + + BOOST_CHECK_EQUAL( 23+19+11, sub_bill.get_subjective_bill(a, now) ); + BOOST_CHECK_EQUAL( 7, sub_bill.get_subjective_bill(b, now) ); + } + { // failed handling logic, decay with repeated failures should be exponential, single failures should be linear + subjective_billing sub_bill; + + sub_bill.subjective_bill_failure(a, fc::microseconds(1024), now); + sub_bill.subjective_bill_failure(b, fc::microseconds(1024), now); + BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(a, now) ); + BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(b, now) ); + + sub_bill.subjective_bill_failure(a, fc::microseconds(1024), halftime); + BOOST_CHECK_EQUAL( 512 + 1024, sub_bill.get_subjective_bill(a, halftime) ); + BOOST_CHECK_EQUAL( 512, sub_bill.get_subjective_bill(b, halftime) ); + + sub_bill.subjective_bill_failure(a, fc::microseconds(1024), endtime); + BOOST_CHECK_EQUAL( 256 + 512 + 1024, sub_bill.get_subjective_bill(a, endtime) ); + BOOST_CHECK_EQUAL( 0, sub_bill.get_subjective_bill(b, endtime) ); + } + + { // expired handling logic, full billing until expiration then failed/decay logic + subjective_billing sub_bill; + constexpr uint32_t window_size = subjective_billing::expired_accumulator_average_window; + + sub_bill.subjective_bill( id1, now, a, fc::microseconds( 1024 ), false ); + sub_bill.subjective_bill( id2, now + fc::microseconds(1), a, fc::microseconds( 1024 ), false ); + sub_bill.subjective_bill( id3, now, b, fc::microseconds( 1024 ), false ); + BOOST_CHECK_EQUAL( 1024 + 1024, sub_bill.get_subjective_bill(a, now) ); + BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(b, now) ); + + sub_bill.remove_expired( log, now, now, fc::time_point::maximum() ); + BOOST_CHECK_EQUAL( 1024 + 1024, sub_bill.get_subjective_bill(a, now) ); + BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(b, now) ); + + BOOST_CHECK_EQUAL( 512 + 1024, sub_bill.get_subjective_bill(a, halftime) ); + BOOST_CHECK_EQUAL( 512, sub_bill.get_subjective_bill(b, halftime) ); + + BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(a, endtime) ); + BOOST_CHECK_EQUAL( 0, sub_bill.get_subjective_bill(b, endtime) ); + + sub_bill.remove_expired( log, now + fc::microseconds(1), now, fc::time_point::maximum() ); + BOOST_CHECK_EQUAL( 1024 + 1024, sub_bill.get_subjective_bill(a, now) ); + BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(b, now) ); + + BOOST_CHECK_EQUAL( 512 + 512, sub_bill.get_subjective_bill(a, halftime) ); + BOOST_CHECK_EQUAL( 512, sub_bill.get_subjective_bill(b, halftime) ); + + BOOST_CHECK_EQUAL( 0, sub_bill.get_subjective_bill(a, endtime) ); + BOOST_CHECK_EQUAL( 0, sub_bill.get_subjective_bill(b, endtime) ); + } + +} + +BOOST_AUTO_TEST_SUITE_END() + +} diff --git a/unittests/wasm_tests.cpp b/unittests/wasm_tests.cpp index b36b0bc05a4..655971bd64d 100644 --- a/unittests/wasm_tests.cpp +++ b/unittests/wasm_tests.cpp @@ -1921,8 +1921,8 @@ BOOST_AUTO_TEST_CASE( billed_cpu_test ) try { }; auto push_trx = [&]( const transaction_metadata_ptr& trx, fc::time_point deadline, - uint32_t billed_cpu_time_us, bool explicit_billed_cpu_time ) { - auto r = chain.control->push_transaction( trx, deadline, billed_cpu_time_us, explicit_billed_cpu_time ); + uint32_t billed_cpu_time_us, bool explicit_billed_cpu_time, uint32_t subjective_cpu_bill_us ) { + auto r = chain.control->push_transaction( trx, deadline, billed_cpu_time_us, explicit_billed_cpu_time, subjective_cpu_bill_us ); if( r->except_ptr ) std::rethrow_exception( r->except_ptr ); if( r->except ) throw *r->except; return r; @@ -1930,7 +1930,7 @@ BOOST_AUTO_TEST_CASE( billed_cpu_test ) try { auto ptrx = create_trx(0); // no limits, just verifying trx works - push_trx( ptrx, fc::time_point::maximum(), 0, false ); // non-explicit billing + push_trx( ptrx, fc::time_point::maximum(), 0, false, 0 ); // non-explicit billing // setup account acc with large limits chain.push_action( config::system_account_name, N(setalimits), config::system_account_name, fc::mutable_variant_object() @@ -1956,7 +1956,7 @@ BOOST_AUTO_TEST_CASE( billed_cpu_test ) try { ptrx = create_trx(0); BOOST_CHECK_LT( max_cpu_time_us, cpu_limit ); // max_cpu_time_us has to be less than cpu_limit to actually test max and not account // indicate explicit billing at transaction max, max_cpu_time_us has to be greater than account cpu time - push_trx( ptrx, fc::time_point::maximum(), max_cpu_time_us, true ); + push_trx( ptrx, fc::time_point::maximum(), max_cpu_time_us, true, 0 ); chain.produce_block(); cpu_limit = mgr.get_account_cpu_limit(acc).first; @@ -1965,14 +1965,14 @@ BOOST_AUTO_TEST_CASE( billed_cpu_test ) try { ptrx = create_trx(0); BOOST_CHECK_LT( max_cpu_time_us + 1, cpu_limit ); // max_cpu_time_us+1 has to be less than cpu_limit to actually test max and not account // indicate explicit billing at max + 1 - BOOST_CHECK_EXCEPTION( push_trx( ptrx, fc::time_point::maximum(), max_cpu_time_us + 1, true ), tx_cpu_usage_exceeded, + BOOST_CHECK_EXCEPTION( push_trx( ptrx, fc::time_point::maximum(), max_cpu_time_us + 1, true, 0 ), tx_cpu_usage_exceeded, fc_exception_message_starts_with( "billed") ); // allow to bill at trx configured max ptrx = create_trx(5); // set trx max at 5ms BOOST_CHECK_LT( 5 * 1000, cpu_limit ); // 5ms has to be less than cpu_limit to actually test trx max and not account // indicate explicit billing at max - push_trx( ptrx, fc::time_point::maximum(), 5 * 1000, true ); + push_trx( ptrx, fc::time_point::maximum(), 5 * 1000, true, 0 ); chain.produce_block(); cpu_limit = mgr.get_account_cpu_limit(acc).first; // update after last trx @@ -1981,19 +1981,19 @@ BOOST_AUTO_TEST_CASE( billed_cpu_test ) try { ptrx = create_trx(5); // set trx max at 5ms BOOST_CHECK_LT( 5 * 1000 + 1, cpu_limit ); // 5ms has to be less than cpu_limit to actually test trx max and not account // indicate explicit billing at max + 1 - BOOST_CHECK_EXCEPTION( push_trx( ptrx, fc::time_point::maximum(), 5 * 1000 + 1, true ), tx_cpu_usage_exceeded, + BOOST_CHECK_EXCEPTION( push_trx( ptrx, fc::time_point::maximum(), 5 * 1000 + 1, true, 0 ), tx_cpu_usage_exceeded, fc_exception_message_starts_with("billed") ); // bill at minimum ptrx = create_trx(0); // indicate explicit billing at transaction minimum - push_trx( ptrx, fc::time_point::maximum(), min_cpu_time_us, true ); + push_trx( ptrx, fc::time_point::maximum(), min_cpu_time_us, true, 0 ); chain.produce_block(); // do not allow to bill less than minimum ptrx = create_trx(0); // indicate explicit billing at minimum-1, objective failure even with explicit billing for under min - BOOST_CHECK_EXCEPTION( push_trx( ptrx, fc::time_point::maximum(), min_cpu_time_us - 1, true ), transaction_exception, + BOOST_CHECK_EXCEPTION( push_trx( ptrx, fc::time_point::maximum(), min_cpu_time_us - 1, true, 0 ), transaction_exception, fc_exception_message_starts_with("cannot bill CPU time less than the minimum") ); chain.push_action( config::system_account_name, N(setalimits), config::system_account_name, fc::mutable_variant_object() @@ -2012,13 +2012,16 @@ BOOST_AUTO_TEST_CASE( billed_cpu_test ) try { ptrx = create_trx(0); BOOST_CHECK_LT( cpu_limit+1, max_cpu_time_us ); // needs to be less or this just tests the same thing as max_cpu_time_us test above // indicate non-explicit billing with 1 more than our account cpu limit, triggers optimization check #8638 and fails trx - BOOST_CHECK_EXCEPTION( push_trx( ptrx, fc::time_point::maximum(), cpu_limit+1, false ), tx_cpu_usage_exceeded, + BOOST_CHECK_EXCEPTION( push_trx( ptrx, fc::time_point::maximum(), cpu_limit+1, false, 0 ), tx_cpu_usage_exceeded, + fc_exception_message_starts_with("estimated") ); + // indicate non-explicit billing with 1 more (subjective) than our account cpu limit, triggers optimization check #8638 and fails trx + BOOST_CHECK_EXCEPTION( push_trx( ptrx, fc::time_point::maximum(), cpu_limit, false, 1 ), tx_cpu_usage_exceeded, fc_exception_message_starts_with("estimated") ); ptrx = create_trx(0); BOOST_CHECK_LT( cpu_limit, max_cpu_time_us ); // indicate non-explicit billing at our account cpu limit, will allow this trx to run, but only bills for actual use - auto r = push_trx( ptrx, fc::time_point::maximum(), cpu_limit, false ); + auto r = push_trx( ptrx, fc::time_point::maximum(), cpu_limit, false, 0 ); BOOST_CHECK_LT( r->receipt->cpu_usage_us, cpu_limit ); // verify not billed at provided bill amount when explicit_billed_cpu_time=false chain.produce_block(); @@ -2028,7 +2031,7 @@ BOOST_AUTO_TEST_CASE( billed_cpu_test ) try { BOOST_CHECK_LT( cpu_limit+1, max_cpu_time_us ); // needs to be less or this just tests the same thing as max_cpu_time_us test above // indicate explicit billing at over our account cpu limit, not allowed cpu_limit = mgr.get_account_cpu_limit_ex(acc).first.max; - BOOST_CHECK_EXCEPTION( push_trx( ptrx, fc::time_point::maximum(), cpu_limit+1, true ), tx_cpu_usage_exceeded, + BOOST_CHECK_EXCEPTION( push_trx( ptrx, fc::time_point::maximum(), cpu_limit+1, true, 0 ), tx_cpu_usage_exceeded, fc_exception_message_starts_with("billed") ); } FC_LOG_AND_RETHROW()