From a0b6ad93159402149507baf2e1ab27bc48e02891 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Fri, 26 Feb 2021 14:20:39 -0600 Subject: [PATCH 001/157] extend block_header_state --- libraries/chain/fork_database.cpp | 9 +- .../eosio/chain/block_header_state.hpp | 79 ++++++++++++- .../block_header_state_unpack_stream.hpp | 30 +++++ .../chain/include/eosio/chain/block_state.hpp | 18 ++- .../chain/include/eosio/chain/snapshot.hpp | 11 +- libraries/chain/snapshot.cpp | 20 ++-- tests/validate-reflection.py | 2 +- unittests/block_state_tests.cpp | 111 ++++++++++++++++++ 8 files changed, 260 insertions(+), 20 deletions(-) create mode 100644 libraries/chain/include/eosio/chain/block_header_state_unpack_stream.hpp create mode 100644 unittests/block_state_tests.cpp diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp index e14975cb37c..19582b6c679 100644 --- a/libraries/chain/fork_database.cpp +++ b/libraries/chain/fork_database.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace eosio { namespace chain { using boost::multi_index_container; @@ -17,7 +18,7 @@ namespace eosio { namespace chain { const uint32_t fork_database::magic_number = 0x30510FDB; const uint32_t fork_database::min_supported_version = 1; - const uint32_t fork_database::max_supported_version = 1; + const uint32_t fork_database::max_supported_version = 2; // work around block_state::is_valid being private inline bool block_state_is_valid( const block_state& bs ) { @@ -122,14 +123,16 @@ namespace eosio { namespace chain { ("max", max_supported_version) ); + block_header_state_unpack_stream unpack_strm(ds, version - min_supported_version); + block_header_state bhs; - fc::raw::unpack( ds, bhs ); + fc::raw::unpack( unpack_strm, bhs ); reset( bhs ); unsigned_int size; fc::raw::unpack( ds, size ); for( uint32_t i = 0, n = size.value; i < n; ++i ) { block_state s; - fc::raw::unpack( ds, s ); + fc::raw::unpack( unpack_strm, s ); // do not populate transaction_metadatas, they will be created as needed in apply_block with appropriate key recovery s.header_exts = s.block->validate_and_extract_header_extensions(); my->add( std::make_shared( move( s ) ), false, true, validator ); diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index 259a593fa21..3dfc0fdef55 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace eosio { namespace chain { @@ -110,6 +111,11 @@ struct pending_block_header_state : public detail::block_header_state_common { const vector& )>& validator )&&; }; +struct security_group_info_t { + uint32_t version = 0; + flat_set participants; +}; + /** * @struct block_header_state * @brief defines the minimum state necessary to validate transaction headers @@ -125,6 +131,18 @@ struct block_header_state : public detail::block_header_state_common { /// duplication of work flat_multimap header_exts; + struct state_extension_v1 { + security_group_info_t security_group_info; + }; + + // For future extension, one should use + // + // struct state_extension_v2 : state_extension_v1 { new_field_t new_field }; + // using state_extension_t = std::variant state_extension; + + using state_extension_t = std::variant; + state_extension_t state_extension; + block_header_state() = default; explicit block_header_state( detail::block_header_state_common&& base ) @@ -153,6 +171,15 @@ struct block_header_state : public detail::block_header_state_common { void verify_signee()const; const vector& get_new_protocol_feature_activations()const; + + security_group_info_t& get_security_group_info() { + return std::visit( [](auto& v) -> security_group_info_t& { return v.security_group_info; }, state_extension); + } + + const security_group_info_t& get_security_group_info() const { + return std::visit( [](const auto& v) -> const security_group_info_t& { return v.security_group_info; }, state_extension); + } + }; using block_header_state_ptr = std::shared_ptr; @@ -177,6 +204,15 @@ FC_REFLECT( eosio::chain::detail::schedule_info, (schedule) ) +FC_REFLECT( eosio::chain::security_group_info_t, + (version) + (participants) +) + +FC_REFLECT( eosio::chain::block_header_state::state_extension_v1, + (security_group_info) +) + // @ignore header_exts FC_REFLECT_DERIVED( eosio::chain::block_header_state, (eosio::chain::detail::block_header_state_common), (id) @@ -184,9 +220,9 @@ FC_REFLECT_DERIVED( eosio::chain::block_header_state, (eosio::chain::detail::bl (pending_schedule) (activated_protocol_features) (additional_signatures) + (state_extension) ) - FC_REFLECT( eosio::chain::legacy::snapshot_block_header_state_v2::schedule_info, ( schedule_lib_num ) ( schedule_hash ) @@ -209,3 +245,44 @@ FC_REFLECT( eosio::chain::legacy::snapshot_block_header_state_v2, ( pending_schedule ) ( activated_protocol_features ) ) + +namespace fc { +namespace raw { +namespace detail { + +// The `Stream` class should contain a `block_header_state_version` member; otherwise, the compilation would fail +template +struct unpack_block_header_state_derived_visitor + : fc::reflector_init_visitor { + + unpack_block_header_state_derived_visitor(Class& _c, Stream& _s) + : fc::reflector_init_visitor(_c) + , s(_s) {} + + template + inline void operator()(const char* name) const { + try { + if constexpr (std::is_same_v< eosio::chain::block_header_state::state_extension_t, std::decay_tobj.*p)>>) + // do not unpack `state_extension` when the block_header_state_version is zero + if (s.block_header_state_version == 0) return; + + fc::raw::unpack(s, this->obj.*p); + } + FC_RETHROW_EXCEPTIONS(warn, "Error unpacking field ${field}", ("field", name)) + } + + private: + Stream& s; +}; + + +template +struct unpack_object_visitor + : unpack_block_header_state_derived_visitor { + using Base = unpack_block_header_state_derived_visitor; + using Base::Base; +}; + +} // namespace detail +} // namespace raw +} // namespace fc diff --git a/libraries/chain/include/eosio/chain/block_header_state_unpack_stream.hpp b/libraries/chain/include/eosio/chain/block_header_state_unpack_stream.hpp new file mode 100644 index 00000000000..7f4b1f692c3 --- /dev/null +++ b/libraries/chain/include/eosio/chain/block_header_state_unpack_stream.hpp @@ -0,0 +1,30 @@ +#pragma once + +namespace eosio { namespace chain { + +/// +/// Provide an extra `version` context to a stream to unpack eosio::chain::block_header_state +/// +/// eosio::chain::block_header_state was not designed to be extensible by itself. In order to +/// add new field to eosio::chain::block_header_state which provie backward compatiblity, we +/// need to add version support for eosio::chain::block_header_state. The version is embedded +/// to eosio::chain::block_header_state, it is derived from the version of snapshot and fork +/// database. This class provides the version information so that eosio::chain::block_header_state +/// can be correctly unpacked. +/// +/// For snapshot and fork database version 1, the `block_header_state.state_extension` field +// should be ignored during unpacking. +/// +template +struct block_header_state_unpack_stream { + + block_header_state_unpack_stream(Stream& stream, uint32_t ver) + : strm(stream) + , block_header_state_version(ver) {} + Stream& strm; + uint32_t block_header_state_version; /// 1 if block_header_state has state_extension; otherwise, 0. + inline void read(char* data, std::size_t len) { strm.read(data, len); } + inline auto get(char& c) ->decltype(strm.get(c)) { return strm.get(c); } +}; +}} + diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index a507c373585..dc8f9b286b7 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -7,7 +7,7 @@ namespace eosio { namespace chain { - struct block_state : public block_header_state { + struct block_state final : block_header_state { block_state( const block_header_state& prev, signed_block_ptr b, const protocol_feature_set& pfs, @@ -71,3 +71,19 @@ namespace eosio { namespace chain { // @ignore _pub_keys_recovered _cached_trxs FC_REFLECT_DERIVED( eosio::chain::block_state, (eosio::chain::block_header_state), (block)(validated) ) + +namespace fc { +namespace raw { +namespace detail { + +template +struct unpack_object_visitor + : unpack_block_header_state_derived_visitor { + using Base = unpack_block_header_state_derived_visitor; + using Base::Base; +}; + +} // namespace detail +} // namespace raw +} // namespace fc + diff --git a/libraries/chain/include/eosio/chain/snapshot.hpp b/libraries/chain/include/eosio/chain/snapshot.hpp index e5bfd262439..88d5e697c9f 100644 --- a/libraries/chain/include/eosio/chain/snapshot.hpp +++ b/libraries/chain/include/eosio/chain/snapshot.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -10,8 +11,10 @@ namespace eosio { namespace chain { /** * History: * Version 1: initial version with string identified sections and rows + * Version 2: support block_header_state with state extensions */ - static const uint32_t current_snapshot_version = 1; + static const uint32_t minimum_snapshot_version = 1; + static const uint32_t current_snapshot_version = 2; namespace detail { template @@ -156,7 +159,7 @@ namespace eosio { namespace chain { namespace detail { struct abstract_snapshot_row_reader { - virtual void provide(std::istream& in) const = 0; + virtual void provide(block_header_state_unpack_stream& in) const = 0; virtual void provide(const fc::variant&) const = 0; virtual std::string row_type_name() const = 0; }; @@ -195,8 +198,7 @@ namespace eosio { namespace chain { explicit snapshot_row_reader( T& data ) :data(data) {} - - void provide(std::istream& in) const override { + void provide(block_header_state_unpack_stream& in) const override { row_validation_helper::apply(data, [&in,this](){ fc::raw::unpack(in, data); }); @@ -361,6 +363,7 @@ namespace eosio { namespace chain { bool validate_section() const; std::istream& snapshot; + uint32_t version; std::streampos header_pos; uint64_t num_rows; uint64_t cur_row; diff --git a/libraries/chain/snapshot.cpp b/libraries/chain/snapshot.cpp index 66966ef6be5..90a4323f2fa 100644 --- a/libraries/chain/snapshot.cpp +++ b/libraries/chain/snapshot.cpp @@ -47,9 +47,10 @@ void variant_snapshot_reader::validate() const { EOS_ASSERT(version.is_integer(), snapshot_validation_exception, "Variant snapshot version is not an integer"); - EOS_ASSERT(version.as_uint64() == (uint64_t)current_snapshot_version, snapshot_validation_exception, - "Variant snapshot is an unsuppored version. Expected : ${expected}, Got: ${actual}", - ("expected", current_snapshot_version)("actual",o["version"].as_uint64())); + uint64_t got_version = version.as_uint64(); + EOS_ASSERT(got_version >= minimum_snapshot_version && got_version<= current_snapshot_version, snapshot_validation_exception, + "Variant snapshot is an unsuppored version. Expected : [${min_version}, ${current}] , Got: ${actual}", + ("min_version", minimum_snapshot_version) ("current", current_snapshot_version)("actual",got_version)); EOS_ASSERT(o.contains("sections"), snapshot_validation_exception, "Variant snapshot has no sections"); @@ -217,12 +218,10 @@ void istream_snapshot_reader::validate() const { "Binary snapshot has unexpected magic number!"); // validate version - auto expected_version = current_snapshot_version; - decltype(expected_version) actual_version; - snapshot.read((char*)&actual_version, sizeof(actual_version)); - EOS_ASSERT(actual_version == expected_version, snapshot_exception, - "Binary snapshot is an unsuppored version. Expected : ${expected}, Got: ${actual}", - ("expected", expected_version)("actual", actual_version)); + snapshot.read((char*)&version, sizeof(version)); + EOS_ASSERT(version > 0 && version <= current_snapshot_version, snapshot_exception, + "Binary snapshot is an unsuppored version. version is ${actual} while code supports version(s) [1,${max}]", + ("max", current_snapshot_version)("actual", version)); while (validate_section()) {} } catch( const std::exception& e ) { \ @@ -328,7 +327,8 @@ void istream_snapshot_reader::set_section( const string& section_name ) { } bool istream_snapshot_reader::read_row( detail::abstract_snapshot_row_reader& row_reader ) { - row_reader.provide(snapshot); + block_header_state_unpack_stream unpack_strm(snapshot, version - minimum_snapshot_version); + row_reader.provide(unpack_strm); return ++cur_row < num_rows; } diff --git a/tests/validate-reflection.py b/tests/validate-reflection.py index 263eda54ce7..c846f08c5b7 100755 --- a/tests/validate-reflection.py +++ b/tests/validate-reflection.py @@ -426,7 +426,7 @@ def next_scope(self, end = None): return new_scope[0] class Namespace(ClassStruct): - namespace_class_pattern = re.compile(r'((?]*>)?)?\s*\{' % (EmptyScope.namespace_str, EmptyScope.struct_str, EmptyScope.class_str, EmptyScope.enum_str), re.MULTILINE | re.DOTALL) + namespace_class_pattern = re.compile(r'((?]*>)?)?\s*\{' % (EmptyScope.namespace_str, EmptyScope.struct_str, EmptyScope.class_str, EmptyScope.enum_str), re.MULTILINE | re.DOTALL) def __init__(self, name, inherit, start, content, parent_scope): assert inherit is None, "namespace %s should not inherit from %s" % (name, inherit) diff --git a/unittests/block_state_tests.cpp b/unittests/block_state_tests.cpp new file mode 100644 index 00000000000..8925a22298c --- /dev/null +++ b/unittests/block_state_tests.cpp @@ -0,0 +1,111 @@ +#include +#include +#include + + +namespace eosio { namespace chain { + +// block_header_state_v0 should has the same layout with block_header_state except +// the lack of the state_extension data member +struct block_header_state_v0 : public detail::block_header_state_common { + block_id_type id; + signed_block_header header; + detail::schedule_info pending_schedule; + protocol_feature_activation_set_ptr activated_protocol_features; + vector additional_signatures; + + /// this data is redundant with the data stored in header, but it acts as a cache that avoids + /// duplication of work + flat_multimap header_exts; +}; + +}} + +FC_REFLECT_DERIVED( eosio::chain::block_header_state_v0, (eosio::chain::detail::block_header_state_common), + (id) + (header) + (pending_schedule) + (activated_protocol_features) + (additional_signatures) +) + + +BOOST_AUTO_TEST_SUITE(block_state_tests) + +BOOST_AUTO_TEST_CASE(test_unpack_legacy_block_state) { + eosio::testing::tester main; + + using namespace eosio::chain::literals; + + // First we create a valid block with valid transaction + main.create_account("newacc"_n); + auto b = main.produce_block(); + + auto bs = main.control->head_block_state(); + + // pack block_header_state as the legacy format + fc::datastream> out_strm; + fc::raw::pack(out_strm, reinterpret_cast(*bs)); + + BOOST_CHECK_NE(out_strm.storage().size(), 0); + + // make sure we can unpack block_header_state + { + fc::datastream in_strm(out_strm.storage().data(), out_strm.storage().size()); + eosio::chain::block_header_state_unpack_stream unpack_strm(in_strm, 0); + eosio::chain::block_header_state tmp; + BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, tmp)); + BOOST_CHECK_EQUAL(bs->id, tmp.id); + BOOST_CHECK_EQUAL(in_strm.remaining(), 0); + } + + // manual pack legacy block_state + fc::raw::pack(out_strm, bs->block); + fc::raw::pack(out_strm, false); + + // make sure we can unpack block_state + { + fc::datastream in_strm(out_strm.storage().data(), out_strm.storage().size()); + eosio::chain::block_header_state_unpack_stream unpack_strm(in_strm, 0); + eosio::chain::block_state tmp; + BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, tmp)); + BOOST_CHECK_EQUAL(bs->id, tmp.id); + BOOST_CHECK_EQUAL(in_strm.remaining(), 0); + } +} + +BOOST_AUTO_TEST_CASE(test_unpack_new_block_state) { +eosio::testing::tester main; + + using namespace eosio::chain::literals; + + // First we create a valid block with valid transaction + main.create_account("newacc"_n); + auto b = main.produce_block(); + + auto bs = main.control->head_block_state(); + auto& sgi = bs->get_security_group_info(); + sgi.version = 1; + sgi.participants.insert("adam"_n); + + // pack block_header_state as the legacy format + fc::datastream> out_strm; + fc::raw::pack(out_strm, *bs); + + BOOST_CHECK_NE(out_strm.storage().size(), 0); + + + // make sure we can unpack block_state + { + fc::datastream in_strm(out_strm.storage().data(), out_strm.storage().size()); + eosio::chain::block_header_state_unpack_stream unpack_strm(in_strm, 1); + eosio::chain::block_state tmp; + BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, tmp)); + BOOST_CHECK_EQUAL(bs->id, tmp.id); + BOOST_CHECK_EQUAL(bs->get_security_group_info().version, 1); + BOOST_TEST(bs->get_security_group_info().participants == sgi.participants); + BOOST_CHECK_EQUAL(in_strm.remaining(), 0); + } +} + +BOOST_AUTO_TEST_SUITE_END() From c4a449056b732d6498c8cf1a1223a4b8ee498158 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 1 Mar 2021 09:51:30 -0600 Subject: [PATCH 002/157] Changes for PR comments --- libraries/chain/fork_database.cpp | 2 +- .../eosio/chain/block_header_state.hpp | 50 +++++++++---------- .../block_header_state_unpack_stream.hpp | 8 +-- libraries/chain/snapshot.cpp | 2 +- unittests/block_state_tests.cpp | 2 + 5 files changed, 32 insertions(+), 32 deletions(-) diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp index 19582b6c679..a599bc83c57 100644 --- a/libraries/chain/fork_database.cpp +++ b/libraries/chain/fork_database.cpp @@ -123,7 +123,7 @@ namespace eosio { namespace chain { ("max", max_supported_version) ); - block_header_state_unpack_stream unpack_strm(ds, version - min_supported_version); + block_header_state_unpack_stream unpack_strm(ds, version > min_supported_version); block_header_state bhs; fc::raw::unpack( unpack_strm, bhs ); diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index 3dfc0fdef55..12e5c3adf40 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -131,23 +131,22 @@ struct block_header_state : public detail::block_header_state_common { /// duplication of work flat_multimap header_exts; - struct state_extension_v1 { + struct state_extension_v0 { security_group_info_t security_group_info; }; // For future extension, one should use // - // struct state_extension_v2 : state_extension_v1 { new_field_t new_field }; - // using state_extension_t = std::variant state_extension; + // struct state_extension_v1 : state_extension_v0 { new_field_t new_field }; + // using state_extension_t = std::variant state_extension; + + using state_extension_t = std::variant; + state_extension_t state_extension; - using state_extension_t = std::variant; - state_extension_t state_extension; - block_header_state() = default; - explicit block_header_state( detail::block_header_state_common&& base ) - :detail::block_header_state_common( std::move(base) ) - {} + explicit block_header_state(detail::block_header_state_common&& base) + : detail::block_header_state_common(std::move(base)) {} explicit block_header_state( legacy::snapshot_block_header_state_v2&& snapshot ); @@ -170,16 +169,16 @@ struct block_header_state : public detail::block_header_state_common { void sign( const signer_callback_type& signer ); void verify_signee()const; - const vector& get_new_protocol_feature_activations()const; + const vector& get_new_protocol_feature_activations() const; - security_group_info_t& get_security_group_info() { - return std::visit( [](auto& v) -> security_group_info_t& { return v.security_group_info; }, state_extension); + security_group_info_t& get_security_group_info() { + return std::visit([](auto& v) -> security_group_info_t& { return v.security_group_info; }, state_extension); } - const security_group_info_t& get_security_group_info() const { - return std::visit( [](const auto& v) -> const security_group_info_t& { return v.security_group_info; }, state_extension); + const security_group_info_t& get_security_group_info() const { + return std::visit([](const auto& v) -> const security_group_info_t& { return v.security_group_info; }, + state_extension); } - }; using block_header_state_ptr = std::shared_ptr; @@ -209,7 +208,7 @@ FC_REFLECT( eosio::chain::security_group_info_t, (participants) ) -FC_REFLECT( eosio::chain::block_header_state::state_extension_v1, +FC_REFLECT( eosio::chain::block_header_state::state_extension_v0, (security_group_info) ) @@ -250,11 +249,10 @@ namespace fc { namespace raw { namespace detail { -// The `Stream` class should contain a `block_header_state_version` member; otherwise, the compilation would fail +// The `Stream` class should contain a `has_block_header_state_extension` member; otherwise, the compilation would fail template -struct unpack_block_header_state_derived_visitor - : fc::reflector_init_visitor { - +struct unpack_block_header_state_derived_visitor : fc::reflector_init_visitor { + unpack_block_header_state_derived_visitor(Class& _c, Stream& _s) : fc::reflector_init_visitor(_c) , s(_s) {} @@ -262,9 +260,10 @@ struct unpack_block_header_state_derived_visitor template inline void operator()(const char* name) const { try { - if constexpr (std::is_same_v< eosio::chain::block_header_state::state_extension_t, std::decay_tobj.*p)>>) - // do not unpack `state_extension` when the block_header_state_version is zero - if (s.block_header_state_version == 0) return; + if constexpr (std::is_same_vobj.*p)>>) + if (!s.has_block_header_state_extension) + return; fc::raw::unpack(s, this->obj.*p); } @@ -275,10 +274,9 @@ struct unpack_block_header_state_derived_visitor Stream& s; }; - template -struct unpack_object_visitor - : unpack_block_header_state_derived_visitor { +struct unpack_object_visitor + : unpack_block_header_state_derived_visitor { using Base = unpack_block_header_state_derived_visitor; using Base::Base; }; diff --git a/libraries/chain/include/eosio/chain/block_header_state_unpack_stream.hpp b/libraries/chain/include/eosio/chain/block_header_state_unpack_stream.hpp index 7f4b1f692c3..c24b4713789 100644 --- a/libraries/chain/include/eosio/chain/block_header_state_unpack_stream.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state_unpack_stream.hpp @@ -7,7 +7,7 @@ namespace eosio { namespace chain { /// /// eosio::chain::block_header_state was not designed to be extensible by itself. In order to /// add new field to eosio::chain::block_header_state which provie backward compatiblity, we -/// need to add version support for eosio::chain::block_header_state. The version is embedded +/// need to add version support for eosio::chain::block_header_state. The version is not embedded /// to eosio::chain::block_header_state, it is derived from the version of snapshot and fork /// database. This class provides the version information so that eosio::chain::block_header_state /// can be correctly unpacked. @@ -18,11 +18,11 @@ namespace eosio { namespace chain { template struct block_header_state_unpack_stream { - block_header_state_unpack_stream(Stream& stream, uint32_t ver) + block_header_state_unpack_stream(Stream& stream, bool has_state_extension) : strm(stream) - , block_header_state_version(ver) {} + , has_block_header_state_extension(has_state_extension) {} Stream& strm; - uint32_t block_header_state_version; /// 1 if block_header_state has state_extension; otherwise, 0. + uint32_t has_block_header_state_extension; inline void read(char* data, std::size_t len) { strm.read(data, len); } inline auto get(char& c) ->decltype(strm.get(c)) { return strm.get(c); } }; diff --git a/libraries/chain/snapshot.cpp b/libraries/chain/snapshot.cpp index 90a4323f2fa..6606e97adf0 100644 --- a/libraries/chain/snapshot.cpp +++ b/libraries/chain/snapshot.cpp @@ -327,7 +327,7 @@ void istream_snapshot_reader::set_section( const string& section_name ) { } bool istream_snapshot_reader::read_row( detail::abstract_snapshot_row_reader& row_reader ) { - block_header_state_unpack_stream unpack_strm(snapshot, version - minimum_snapshot_version); + block_header_state_unpack_stream unpack_strm(snapshot, version > minimum_snapshot_version); row_reader.provide(unpack_strm); return ++cur_row < num_rows; } diff --git a/unittests/block_state_tests.cpp b/unittests/block_state_tests.cpp index 8925a22298c..287d47b604c 100644 --- a/unittests/block_state_tests.cpp +++ b/unittests/block_state_tests.cpp @@ -70,6 +70,7 @@ BOOST_AUTO_TEST_CASE(test_unpack_legacy_block_state) { eosio::chain::block_state tmp; BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, tmp)); BOOST_CHECK_EQUAL(bs->id, tmp.id); + BOOST_CHECK_EQUAL(bs->block->previous, tmp.block->previous); BOOST_CHECK_EQUAL(in_strm.remaining(), 0); } } @@ -102,6 +103,7 @@ eosio::testing::tester main; eosio::chain::block_state tmp; BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, tmp)); BOOST_CHECK_EQUAL(bs->id, tmp.id); + BOOST_CHECK_EQUAL(bs->block->previous, tmp.block->previous); BOOST_CHECK_EQUAL(bs->get_security_group_info().version, 1); BOOST_TEST(bs->get_security_group_info().participants == sgi.participants); BOOST_CHECK_EQUAL(in_strm.remaining(), 0); From 97d3925704a8f57bde4985e8d646a9a42d8d73ca Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 1 Mar 2021 15:39:55 -0600 Subject: [PATCH 003/157] Changed to use the version from chain_snapshot_header --- libraries/chain/combined_database.cpp | 2 ++ libraries/chain/fork_database.cpp | 4 ++-- .../include/eosio/chain/block_header_state.hpp | 8 ++++++-- .../chain/include/eosio/chain/chain_snapshot.hpp | 8 +++++--- libraries/chain/include/eosio/chain/snapshot.hpp | 12 ++++++------ ...ack_stream.hpp => versioned_unpack_stream.hpp} | 15 +++++++-------- libraries/chain/snapshot.cpp | 3 ++- unittests/block_state_tests.cpp | 6 +++--- 8 files changed, 33 insertions(+), 25 deletions(-) rename libraries/chain/include/eosio/chain/{block_header_state_unpack_stream.hpp => versioned_unpack_stream.hpp} (68%) diff --git a/libraries/chain/combined_database.cpp b/libraries/chain/combined_database.cpp index 4795ac15edd..dbb89b775e2 100644 --- a/libraries/chain/combined_database.cpp +++ b/libraries/chain/combined_database.cpp @@ -389,6 +389,8 @@ namespace eosio { namespace chain { header.validate(); }); + snapshot->chain_snapshot_version = header.version; + db.create([](auto&) {}); check_backing_store_setting(true); diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp index a599bc83c57..446bd1141ad 100644 --- a/libraries/chain/fork_database.cpp +++ b/libraries/chain/fork_database.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include namespace eosio { namespace chain { using boost::multi_index_container; @@ -123,7 +123,7 @@ namespace eosio { namespace chain { ("max", max_supported_version) ); - block_header_state_unpack_stream unpack_strm(ds, version > min_supported_version); + versioned_unpack_stream unpack_strm(ds, version - min_supported_version + block_header_state::minimum_version_with_state_extension -1); block_header_state bhs; fc::raw::unpack( unpack_strm, bhs ); diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index 12e5c3adf40..cc35d780fa2 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include namespace eosio { namespace chain { @@ -121,6 +121,10 @@ struct security_group_info_t { * @brief defines the minimum state necessary to validate transaction headers */ struct block_header_state : public detail::block_header_state_common { + + /// this version is coming from chain_snapshot_header.version + static constexpr uint32_t minimum_version_with_state_extension = 6; + block_id_type id; signed_block_header header; detail::schedule_info pending_schedule; @@ -262,7 +266,7 @@ struct unpack_block_header_state_derived_visitor : fc::reflector_init_visitorobj.*p)>>) - if (!s.has_block_header_state_extension) + if (s.version < eosio::chain::block_header_state::minimum_version_with_state_extension) return; fc::raw::unpack(s, this->obj.*p); diff --git a/libraries/chain/include/eosio/chain/chain_snapshot.hpp b/libraries/chain/include/eosio/chain/chain_snapshot.hpp index d2bd01492f1..b46cc88af1d 100644 --- a/libraries/chain/include/eosio/chain/chain_snapshot.hpp +++ b/libraries/chain/include/eosio/chain/chain_snapshot.hpp @@ -16,16 +16,18 @@ struct chain_snapshot_header { * - WebAuthn keys * - wtmsig block siganatures: the block header state changed to include producer authorities and additional signatures * - removed genesis_state and added chain ID to global_property_object - * 4: Updated for v3.0.0 protocol features: + * 4: Updated for v2.1.0 protocol features: * - forwards compatible with versions 2 and 3 * - kv database * - Configurable wasm limits - * 5: Updated for v3.0.0 eos features: + * 5: Updated for v2.1.0 eos features: * - chain_config update + * 6: Updated for v2.2.0 eos features: + * - block_header_state::state_extension */ static constexpr uint32_t minimum_compatible_version = 2; - static constexpr uint32_t current_version = 5; + static constexpr uint32_t current_version = 6; uint32_t version = current_version; diff --git a/libraries/chain/include/eosio/chain/snapshot.hpp b/libraries/chain/include/eosio/chain/snapshot.hpp index 88d5e697c9f..2b0f2fc6663 100644 --- a/libraries/chain/include/eosio/chain/snapshot.hpp +++ b/libraries/chain/include/eosio/chain/snapshot.hpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include @@ -159,7 +159,7 @@ namespace eosio { namespace chain { namespace detail { struct abstract_snapshot_row_reader { - virtual void provide(block_header_state_unpack_stream& in) const = 0; + virtual void provide(versioned_unpack_stream& in) const = 0; virtual void provide(const fc::variant&) const = 0; virtual std::string row_type_name() const = 0; }; @@ -198,7 +198,7 @@ namespace eosio { namespace chain { explicit snapshot_row_reader( T& data ) :data(data) {} - void provide(block_header_state_unpack_stream& in) const override { + void provide(versioned_unpack_stream& in) const override { row_validation_helper::apply(data, [&in,this](){ fc::raw::unpack(in, data); }); @@ -285,6 +285,8 @@ namespace eosio { namespace chain { virtual ~snapshot_reader(){}; + uint32_t chain_snapshot_version = 0; + protected: virtual bool has_section( const std::string& section_name ) = 0; virtual void set_section( const std::string& section_name ) = 0; @@ -361,9 +363,7 @@ namespace eosio { namespace chain { private: bool validate_section() const; - - std::istream& snapshot; - uint32_t version; + std::istream& snapshot; std::streampos header_pos; uint64_t num_rows; uint64_t cur_row; diff --git a/libraries/chain/include/eosio/chain/block_header_state_unpack_stream.hpp b/libraries/chain/include/eosio/chain/versioned_unpack_stream.hpp similarity index 68% rename from libraries/chain/include/eosio/chain/block_header_state_unpack_stream.hpp rename to libraries/chain/include/eosio/chain/versioned_unpack_stream.hpp index c24b4713789..c4c0ea19787 100644 --- a/libraries/chain/include/eosio/chain/block_header_state_unpack_stream.hpp +++ b/libraries/chain/include/eosio/chain/versioned_unpack_stream.hpp @@ -3,7 +3,8 @@ namespace eosio { namespace chain { /// -/// Provide an extra `version` context to a stream to unpack eosio::chain::block_header_state +/// Provide an extra `version` context to a stream to unpack eosio::chain::block_header_state and +/// eosio::chain::global_property_object /// /// eosio::chain::block_header_state was not designed to be extensible by itself. In order to /// add new field to eosio::chain::block_header_state which provie backward compatiblity, we @@ -11,18 +12,16 @@ namespace eosio { namespace chain { /// to eosio::chain::block_header_state, it is derived from the version of snapshot and fork /// database. This class provides the version information so that eosio::chain::block_header_state /// can be correctly unpacked. -/// -/// For snapshot and fork database version 1, the `block_header_state.state_extension` field -// should be ignored during unpacking. +/// /// template -struct block_header_state_unpack_stream { +struct versioned_unpack_stream { - block_header_state_unpack_stream(Stream& stream, bool has_state_extension) + versioned_unpack_stream(Stream& stream, uint32_t ver) : strm(stream) - , has_block_header_state_extension(has_state_extension) {} + , version(ver) {} Stream& strm; - uint32_t has_block_header_state_extension; + uint32_t version; inline void read(char* data, std::size_t len) { strm.read(data, len); } inline auto get(char& c) ->decltype(strm.get(c)) { return strm.get(c); } }; diff --git a/libraries/chain/snapshot.cpp b/libraries/chain/snapshot.cpp index 6606e97adf0..302b0cee984 100644 --- a/libraries/chain/snapshot.cpp +++ b/libraries/chain/snapshot.cpp @@ -218,6 +218,7 @@ void istream_snapshot_reader::validate() const { "Binary snapshot has unexpected magic number!"); // validate version + uint32_t version; snapshot.read((char*)&version, sizeof(version)); EOS_ASSERT(version > 0 && version <= current_snapshot_version, snapshot_exception, "Binary snapshot is an unsuppored version. version is ${actual} while code supports version(s) [1,${max}]", @@ -327,7 +328,7 @@ void istream_snapshot_reader::set_section( const string& section_name ) { } bool istream_snapshot_reader::read_row( detail::abstract_snapshot_row_reader& row_reader ) { - block_header_state_unpack_stream unpack_strm(snapshot, version > minimum_snapshot_version); + versioned_unpack_stream unpack_strm(snapshot, chain_snapshot_version); row_reader.provide(unpack_strm); return ++cur_row < num_rows; } diff --git a/unittests/block_state_tests.cpp b/unittests/block_state_tests.cpp index 287d47b604c..22bbce713ac 100644 --- a/unittests/block_state_tests.cpp +++ b/unittests/block_state_tests.cpp @@ -52,7 +52,7 @@ BOOST_AUTO_TEST_CASE(test_unpack_legacy_block_state) { // make sure we can unpack block_header_state { fc::datastream in_strm(out_strm.storage().data(), out_strm.storage().size()); - eosio::chain::block_header_state_unpack_stream unpack_strm(in_strm, 0); + eosio::chain::versioned_unpack_stream unpack_strm(in_strm, eosio::chain::block_header_state::minimum_version_with_state_extension-1); eosio::chain::block_header_state tmp; BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, tmp)); BOOST_CHECK_EQUAL(bs->id, tmp.id); @@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(test_unpack_legacy_block_state) { // make sure we can unpack block_state { fc::datastream in_strm(out_strm.storage().data(), out_strm.storage().size()); - eosio::chain::block_header_state_unpack_stream unpack_strm(in_strm, 0); + eosio::chain::versioned_unpack_stream unpack_strm(in_strm, eosio::chain::block_header_state::minimum_version_with_state_extension-1); eosio::chain::block_state tmp; BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, tmp)); BOOST_CHECK_EQUAL(bs->id, tmp.id); @@ -99,7 +99,7 @@ eosio::testing::tester main; // make sure we can unpack block_state { fc::datastream in_strm(out_strm.storage().data(), out_strm.storage().size()); - eosio::chain::block_header_state_unpack_stream unpack_strm(in_strm, 1); + eosio::chain::versioned_unpack_stream unpack_strm(in_strm, eosio::chain::block_header_state::minimum_version_with_state_extension); eosio::chain::block_state tmp; BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, tmp)); BOOST_CHECK_EQUAL(bs->id, tmp.id); From 37cfa92b324dd4b5539e4b0d3a0836759f256950 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 2 Mar 2021 14:39:03 -0600 Subject: [PATCH 004/157] add global_property_object extension --- .../eosio/chain/block_header_state.hpp | 23 +++-- .../include/eosio/chain/chain_snapshot.hpp | 1 + .../eosio/chain/global_property_object.hpp | 96 +++++++++++++++++-- unittests/block_state_tests.cpp | 86 ++++++++++++++++- 4 files changed, 189 insertions(+), 17 deletions(-) diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index cc35d780fa2..2914be3d409 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -253,11 +253,18 @@ namespace fc { namespace raw { namespace detail { -// The `Stream` class should contain a `has_block_header_state_extension` member; otherwise, the compilation would fail -template +// C++20 Concept +// +// template +// concept VersionedStream = requires (T t) { +// t.version; +// } +// + +template struct unpack_block_header_state_derived_visitor : fc::reflector_init_visitor { - unpack_block_header_state_derived_visitor(Class& _c, Stream& _s) + unpack_block_header_state_derived_visitor(Class& _c, VersionedStream& _s) : fc::reflector_init_visitor(_c) , s(_s) {} @@ -275,13 +282,13 @@ struct unpack_block_header_state_derived_visitor : fc::reflector_init_visitor -struct unpack_object_visitor - : unpack_block_header_state_derived_visitor { - using Base = unpack_block_header_state_derived_visitor; +template +struct unpack_object_visitor + : unpack_block_header_state_derived_visitor { + using Base = unpack_block_header_state_derived_visitor; using Base::Base; }; diff --git a/libraries/chain/include/eosio/chain/chain_snapshot.hpp b/libraries/chain/include/eosio/chain/chain_snapshot.hpp index b46cc88af1d..957c091db45 100644 --- a/libraries/chain/include/eosio/chain/chain_snapshot.hpp +++ b/libraries/chain/include/eosio/chain/chain_snapshot.hpp @@ -24,6 +24,7 @@ struct chain_snapshot_header { * - chain_config update * 6: Updated for v2.2.0 eos features: * - block_header_state::state_extension + * - global_proper_state::extension */ static constexpr uint32_t minimum_compatible_version = 2; diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index 7b7497e34ec..f0c70cda7cb 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -54,6 +54,14 @@ namespace eosio { namespace chain { }; } + + struct pending_security_group_t { + block_num_type block_num = 0; + uint32_t version = 0; + flat_set participants; + }; + + /** * @class global_property_object * @brief Maintains global state information about block producer schedules and chain configuration parameters @@ -73,6 +81,8 @@ namespace eosio { namespace chain { kv_database_config kv_configuration; wasm_config wasm_configuration; + std::optional pending_security_group; + void initalize_from( const legacy::snapshot_global_property_object_v2& legacy, const chain_id_type& chain_id_val, const kv_database_config& kv_config_val, const wasm_config& wasm_config_val ) { proposed_schedule_block_num = legacy.proposed_schedule_block_num; proposed_schedule = producer_authority_schedule(legacy.proposed_schedule).to_shared(proposed_schedule.producers.get_allocator()); @@ -118,6 +128,21 @@ namespace eosio { namespace chain { chain_id_type chain_id; kv_database_config kv_configuration; wasm_config wasm_configuration; + + static constexpr uint32_t minimum_version_with_extension = 6; + + struct extension_v0 { + std::optional pending_security_group; + }; + + // for future extensions, please use the following pattern: + // + // struct extension_v1 : extension_v0 { new_field_t new_field; }; + // using extension_t = std::variant; + // + + using extension_t = std::variant; + extension_t extension; }; namespace detail { @@ -126,17 +151,33 @@ namespace eosio { namespace chain { using value_type = global_property_object; using snapshot_type = snapshot_global_property_object; - static snapshot_global_property_object to_snapshot_row( const global_property_object& value, const chainbase::database& ) { - return {value.proposed_schedule_block_num, producer_authority_schedule::from_shared(value.proposed_schedule), value.configuration, value.chain_id, value.kv_configuration, value.wasm_configuration}; + static_assert(std::is_same_v>, + "Please update to_snapshot_row()/from_snapshot_row() accordingly when " + "snapshot_global_property_object::extension_t is changed"); + + static snapshot_global_property_object to_snapshot_row(const global_property_object& value, + const chainbase::database&) { + return {value.proposed_schedule_block_num, + producer_authority_schedule::from_shared(value.proposed_schedule), + value.configuration, + value.chain_id, + value.kv_configuration, + value.wasm_configuration, + snapshot_global_property_object::extension_v0{value.pending_security_group}}; } - static void from_snapshot_row( snapshot_global_property_object&& row, global_property_object& value, chainbase::database& ) { + static void from_snapshot_row(snapshot_global_property_object&& row, global_property_object& value, + chainbase::database&) { value.proposed_schedule_block_num = row.proposed_schedule_block_num; - value.proposed_schedule = row.proposed_schedule.to_shared(value.proposed_schedule.producers.get_allocator()); - value.configuration = row.configuration; - value.chain_id = row.chain_id; - value.kv_configuration = row.kv_configuration; + value.proposed_schedule = + row.proposed_schedule.to_shared(value.proposed_schedule.producers.get_allocator()); + value.configuration = row.configuration; + value.chain_id = row.chain_id; + value.kv_configuration = row.kv_configuration; value.wasm_configuration = row.wasm_configuration; + std::visit([&value](auto& ext) { value.pending_security_group = std::move(ext.pending_security_group); }, + row.extension); } }; } @@ -186,10 +227,49 @@ FC_REFLECT(eosio::chain::legacy::snapshot_global_property_object_v4, (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id)(kv_configuration)(wasm_configuration) ) +FC_REFLECT(eosio::chain::pending_security_group_t, + (block_num)(version)(participants) + ) + +FC_REFLECT(eosio::chain::snapshot_global_property_object::extension_v0, + (pending_security_group) + ) + FC_REFLECT(eosio::chain::snapshot_global_property_object, - (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id)(kv_configuration)(wasm_configuration) + (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id)(kv_configuration)(wasm_configuration)(extension) ) FC_REFLECT(eosio::chain::dynamic_global_property_object, (global_action_sequence) ) + +namespace fc { +namespace raw { +namespace detail { + +template +struct unpack_object_visitor + : fc::reflector_init_visitor { + unpack_object_visitor(eosio::chain::snapshot_global_property_object& _c, VersionedStream& _s) + : fc::reflector_init_visitor(_c) + , s(_s) {} + + template + inline void operator()(const char* name) const { + try { + if constexpr (std::is_same_vobj.*p)>>) + if (s.version < eosio::chain::snapshot_global_property_object::minimum_version_with_extension) + return; + + fc::raw::unpack(s, this->obj.*p); + } + FC_RETHROW_EXCEPTIONS(warn, "Error unpacking field ${field}", ("field", name)) + } + + private: + VersionedStream& s; +}; +} // namespace detail +} // namespace raw +} // namespace fc diff --git a/unittests/block_state_tests.cpp b/unittests/block_state_tests.cpp index 22bbce713ac..b254badf808 100644 --- a/unittests/block_state_tests.cpp +++ b/unittests/block_state_tests.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace eosio { namespace chain { @@ -19,6 +20,22 @@ struct block_header_state_v0 : public detail::block_header_state_common { flat_multimap header_exts; }; +struct snapshot_global_property_object_v5 { + std::optional proposed_schedule_block_num; + producer_authority_schedule proposed_schedule; + chain_config configuration; + chain_id_type chain_id; + kv_database_config kv_configuration; + wasm_config wasm_configuration; + + snapshot_global_property_object_v5(const global_property_object& value) + : proposed_schedule_block_num(value.proposed_schedule_block_num) + , proposed_schedule(producer_authority_schedule::from_shared(value.proposed_schedule)) + , configuration(value.configuration) + , chain_id(value.chain_id) + , kv_configuration(value.kv_configuration) + , wasm_configuration(value.wasm_configuration) {} +}; }} FC_REFLECT_DERIVED( eosio::chain::block_header_state_v0, (eosio::chain::detail::block_header_state_common), @@ -29,6 +46,10 @@ FC_REFLECT_DERIVED( eosio::chain::block_header_state_v0, (eosio::chain::detail: (additional_signatures) ) +FC_REFLECT(eosio::chain::snapshot_global_property_object_v5, + (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id)(kv_configuration)(wasm_configuration) + ) + BOOST_AUTO_TEST_SUITE(block_state_tests) @@ -76,7 +97,7 @@ BOOST_AUTO_TEST_CASE(test_unpack_legacy_block_state) { } BOOST_AUTO_TEST_CASE(test_unpack_new_block_state) { -eosio::testing::tester main; + eosio::testing::tester main; using namespace eosio::chain::literals; @@ -110,4 +131,67 @@ eosio::testing::tester main; } } +BOOST_AUTO_TEST_CASE(test_snapshot_global_property_object) { + eosio::testing::tester main; + + using namespace eosio::chain::literals; + + // First we create a valid block with valid transaction + main.create_account("newacc"_n); + auto b = main.produce_block(); + + auto gpo = main.control->get_global_properties(); + + using row_traits = eosio::chain::detail::snapshot_row_traits; + + { + // pack snapshot_global_property_object as the legacy format + fc::datastream> out_strm; + fc::raw::pack(out_strm, eosio::chain::snapshot_global_property_object_v5(gpo)); + + // make sure we can unpack snapshot_global_property_object + { + fc::datastream in_strm(out_strm.storage().data(), out_strm.storage().size()); + uint32_t version = 5; + eosio::chain::versioned_unpack_stream unpack_strm(in_strm, version); + + eosio::chain::snapshot_global_property_object row; + BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, row)); + BOOST_CHECK_EQUAL(in_strm.remaining(), 0); + BOOST_CHECK_EQUAL(row.chain_id, gpo.chain_id); + } + } + + { + // pack snapshot_global_property_object as the new format + gpo.pending_security_group = eosio::chain::pending_security_group_t { + .block_num = 10, + .version = 1, + .participants = { "adam"_n } + }; + + fc::datastream> out_strm; + fc::raw::pack(out_strm, row_traits::to_snapshot_row(gpo, main.control->db())); + + // make sure we can unpack snapshot_global_property_object + { + fc::datastream in_strm(out_strm.storage().data(), out_strm.storage().size()); + eosio::chain::versioned_unpack_stream unpack_strm(in_strm, eosio::chain::snapshot_global_property_object::minimum_version_with_extension); + + eosio::chain::snapshot_global_property_object row; + BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, row)); + BOOST_CHECK_EQUAL(in_strm.remaining(), 0); + + BOOST_CHECK_EQUAL(row.chain_id, gpo.chain_id); + auto pending_security_group = std::visit([](auto& ext) { return ext.pending_security_group; }, row.extension); + BOOST_CHECK(pending_security_group.has_value()); + if (pending_security_group.has_value()) { + BOOST_CHECK_EQUAL(pending_security_group->block_num, gpo.pending_security_group->block_num); + BOOST_CHECK_EQUAL(pending_security_group->version, gpo.pending_security_group->version); + BOOST_TEST(pending_security_group->participants == gpo.pending_security_group->participants); + } + } + } +} + BOOST_AUTO_TEST_SUITE_END() From f2fc7b5bfe52c3caf0dbd30037cb62727cdaae5e Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Fri, 5 Mar 2021 11:55:58 -0600 Subject: [PATCH 005/157] EPE-672: Added unit test framework --- plugins/net_plugin/CMakeLists.txt | 4 +++ plugins/net_plugin/test/CMakeLists.txt | 24 +++++++++++++++ plugins/net_plugin/test/main.cpp | 41 ++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 plugins/net_plugin/test/CMakeLists.txt create mode 100644 plugins/net_plugin/test/main.cpp diff --git a/plugins/net_plugin/CMakeLists.txt b/plugins/net_plugin/CMakeLists.txt index 3b8c1cd7b71..36b2b7bcdb0 100644 --- a/plugins/net_plugin/CMakeLists.txt +++ b/plugins/net_plugin/CMakeLists.txt @@ -1,7 +1,11 @@ file(GLOB HEADERS "include/eosio/net_plugin/*.hpp" ) add_library( net_plugin net_plugin.cpp + security_group.cpp ${HEADERS} ) target_link_libraries( net_plugin chain_plugin producer_plugin appbase fc ) target_include_directories( net_plugin PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/../chain_interface/include "${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/appbase/include") + +add_subdirectory(test) + diff --git a/plugins/net_plugin/test/CMakeLists.txt b/plugins/net_plugin/test/CMakeLists.txt new file mode 100644 index 00000000000..9b19540f673 --- /dev/null +++ b/plugins/net_plugin/test/CMakeLists.txt @@ -0,0 +1,24 @@ +file(GLOB UNIT_TESTS "*.cpp") + +add_executable( net_plugin_tests ${UNIT_TESTS} ) + +target_link_libraries( net_plugin_tests + net_plugin + chain_plugin + producer_plugin + appbase + fc + ${PLATFORM_SPECIFIC_LIBS} + eosio_testing + eosio_chain + ) +target_include_directories( net_plugin_tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/plugins/chain_interface/include + ${CMAKE_SOURCE_DIR}/libraries/appbase/include + ${CMAKE_SOURCE_DIR}/plugins/net_plugin/include + ${CMAKE_SOURCE_DIR}/plugins/chain_plugin/include + ${CMAKE_SOURCE_DIR}/libraries/testing/include + ${CMAKE_SOURCE_DIR}/cmake-build-debug/unittests/include + ) + +add_test(NAME net_plugin_tests COMMAND plugins/net_plugin/test/net_plugin_tests WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/plugins/net_plugin/test/main.cpp b/plugins/net_plugin/test/main.cpp new file mode 100644 index 00000000000..eb7cbf7cae7 --- /dev/null +++ b/plugins/net_plugin/test/main.cpp @@ -0,0 +1,41 @@ +#include +#include +#include +#include +#include + +//extern uint32_t EOS_TESTING_GENESIS_TIMESTAMP; + +void translate_fc_exception(const fc::exception &e) { + std::cerr << "\033[33m" << e.to_detail_string() << "\033[0m" << std::endl; + BOOST_TEST_FAIL("Caught Unexpected Exception"); +} + +boost::unit_test::test_suite* init_unit_test_suite(int argc, char* argv[]) { + // Turn off blockchain logging if no --verbose parameter is not added + // To have verbose enabled, call "tests/chain_test -- --verbose" + bool is_verbose = false; + std::string verbose_arg = "--verbose"; + for (int i = 0; i < argc; i++) { + if (verbose_arg == argv[i]) { + is_verbose = true; + break; + } + } + if(!is_verbose) fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::off); + + // Register fc::exception translator + boost::unit_test::unit_test_monitor.register_exception_translator(&translate_fc_exception); + + std::srand(time(NULL)); + std::cout << "Random number generator seeded to " << time(NULL) << std::endl; + /* + const char* genesis_timestamp_str = getenv("EOS_TESTING_GENESIS_TIMESTAMP"); + if( genesis_timestamp_str != nullptr ) + { + EOS_TESTING_GENESIS_TIMESTAMP = std::stoul( genesis_timestamp_str ); + } + std::cout << "EOS_TESTING_GENESIS_TIMESTAMP is " << EOS_TESTING_GENESIS_TIMESTAMP << std::endl; + */ + return nullptr; +} From 5fd0176fad9d0af13894e6c3bb9b27e183c5d235 Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Fri, 5 Mar 2021 14:05:40 -0600 Subject: [PATCH 006/157] EPE-672: unittest Needed to remove a line --- plugins/net_plugin/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/net_plugin/CMakeLists.txt b/plugins/net_plugin/CMakeLists.txt index 36b2b7bcdb0..86404f6ddef 100644 --- a/plugins/net_plugin/CMakeLists.txt +++ b/plugins/net_plugin/CMakeLists.txt @@ -1,7 +1,6 @@ file(GLOB HEADERS "include/eosio/net_plugin/*.hpp" ) add_library( net_plugin net_plugin.cpp - security_group.cpp ${HEADERS} ) target_link_libraries( net_plugin chain_plugin producer_plugin appbase fc ) From 9cbb2bfede4a81409a3eaaac299c09cc8181138a Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 4 Mar 2021 09:38:13 -0600 Subject: [PATCH 007/157] add/remove participants --- libraries/chain/block_header_state.cpp | 4 + libraries/chain/controller.cpp | 87 +++++++++++++++++++ .../eosio/chain/block_header_state.hpp | 15 ++-- .../chain/include/eosio/chain/controller.hpp | 8 ++ .../eosio/chain/global_property_object.hpp | 38 ++++---- .../eosio/chain/protocol_feature_manager.hpp | 3 +- .../eosio/chain/security_group_info.hpp | 18 ++++ libraries/chain/protocol_feature_manager.cpp | 11 +++ ...ate_tests.cpp => security_group_tests.cpp} | 77 ++++++++++++---- 9 files changed, 210 insertions(+), 51 deletions(-) create mode 100644 libraries/chain/include/eosio/chain/security_group_info.hpp rename unittests/{block_state_tests.cpp => security_group_tests.cpp} (71%) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index b7cd3036ac3..ace383921a5 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -167,6 +167,8 @@ namespace eosio { namespace chain { result.producer_to_last_implied_irb[proauth.producer_name] = dpos_proposed_irreversible_blocknum; } + result.security_group = get_security_group_info(); + return result; } @@ -314,6 +316,8 @@ namespace eosio { namespace chain { result.activated_protocol_features = std::move( new_activated_protocol_features ); + result.set_security_group_info(std::move(security_group)); + return result; } diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 48a88901032..c3a1b1bbf5f 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1429,6 +1429,27 @@ struct controller_impl { }); } + if (gpo.proposed_security_group_block_num) { + if (gpo.proposed_security_group_block_num <= pbhs.dpos_irreversible_blocknum) { + + // Promote proposed security group to pending. + if( !replay_head_time ) { + ilog( "promoting proposed security group (set in block ${proposed_num}) to pending; current block: ${n} lib: ${lib} participants: ${participants} ", + ("proposed_num", gpo.proposed_security_group_block_num)("n", pbhs.block_num) + ("lib", pbhs.dpos_irreversible_blocknum) + ("participants", gpo.proposed_security_group_participants ) ); + } + + ++bb._pending_block_header_state.security_group.version; + bb._pending_block_header_state.security_group.participants = std::move(gpo.proposed_security_group_participants); + + db.modify(gpo, [&](auto& gp) { + gp.proposed_security_group_block_num = 0; + gp.proposed_security_group_participants.clear(); + }); + } + } + try { transaction_metadata_ptr onbtrx = transaction_metadata::create_no_recover_keys( std::make_shared( get_on_block_transaction(), true ), @@ -2229,6 +2250,35 @@ struct controller_impl { return deep_mind_logger; } + int64_t propose_security_group(std::function&)> && modify_participants) { + const auto& gpo = self.get_global_properties(); + auto cur_block_num = head->block_num + 1; + + if (!self.is_builtin_activated(builtin_protocol_feature_t::security_group)) { + return -1; + } + + flat_set proposed_participants = gpo.proposed_security_group_block_num == 0 + ? self.active_security_group().participants + : gpo.proposed_security_group_participants; + + auto orig_participants_size = proposed_participants.size(); + + modify_participants(proposed_participants); + + if (orig_participants_size == proposed_participants.size()) { + // no changes in the participants + return -1; + } + + db.modify(gpo, [&proposed_participants, cur_block_num](auto& gp) { + gp.proposed_security_group_block_num = cur_block_num; + gp.proposed_security_group_participants = std::move(proposed_participants); + }); + + return 0; + } + }; /// controller_impl const resource_limits_manager& controller::get_resource_limits_manager()const @@ -2808,6 +2858,43 @@ int64_t controller::set_proposed_producers( vector producers return version; } +const security_group_info_t& controller::active_security_group() const { + if( !(my->pending) ) + return my->head->get_security_group_info(); + + return std::visit( + overloaded{ + [](const building_block& bb) -> const security_group_info_t& { return bb._pending_block_header_state.security_group; }, + [](const assembled_block& ab) -> const security_group_info_t& { return ab._pending_block_header_state.security_group; }, + [](const completed_block& cb) -> const security_group_info_t& { return cb._block_state->get_security_group_info(); }}, + my->pending->_block_stage); +} + +const flat_set& controller::proposed_security_group_participants() const { + return get_global_properties().proposed_security_group_participants; +} + +int64_t controller::propose_security_group_participants_add(const flat_set& participants) { + return participants.size() == 0 ? -1 : my->propose_security_group([&participants](auto& pending_participants) { + pending_participants.insert(participants.begin(), participants.end()); + }); +} + +int64_t controller::propose_security_group_participants_remove(const flat_set& participants) { + return participants.size() == 0 ? -1 : my->propose_security_group([&participants](auto& pending_participants) { + flat_set::sequence_type tmp; + tmp.reserve(pending_participants.size()); + std::set_difference(pending_participants.begin(), pending_participants.end(), participants.begin(), + participants.end(), std::back_inserter(tmp)); + pending_participants.adopt_sequence(std::move(tmp)); + }); +} + +bool controller::in_active_security_group(const flat_set& participants) const { + const auto& active = active_security_group().participants; + return std::includes(active.begin(), active.end(), participants.begin(), participants.end()); +} + const producer_authority_schedule& controller::active_producers()const { if( !(my->pending) ) return my->head->active_schedule; diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index 2914be3d409..668136fb7b7 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace eosio { namespace chain { @@ -81,6 +82,7 @@ struct pending_block_header_state : public detail::block_header_state_common { block_timestamp_type timestamp; uint32_t active_schedule_version = 0; uint16_t confirmed = 1; + security_group_info_t security_group; signed_block_header make_block_header( const checksum256_type& transaction_mroot, const checksum256_type& action_mroot, @@ -111,10 +113,7 @@ struct pending_block_header_state : public detail::block_header_state_common { const vector& )>& validator )&&; }; -struct security_group_info_t { - uint32_t version = 0; - flat_set participants; -}; + /** * @struct block_header_state @@ -175,8 +174,8 @@ struct block_header_state : public detail::block_header_state_common { const vector& get_new_protocol_feature_activations() const; - security_group_info_t& get_security_group_info() { - return std::visit([](auto& v) -> security_group_info_t& { return v.security_group_info; }, state_extension); + void set_security_group_info(security_group_info_t&& new_info) { + std::visit([&new_info](auto& v) { v.security_group_info = std::move(new_info); }, state_extension); } const security_group_info_t& get_security_group_info() const { @@ -207,10 +206,6 @@ FC_REFLECT( eosio::chain::detail::schedule_info, (schedule) ) -FC_REFLECT( eosio::chain::security_group_info_t, - (version) - (participants) -) FC_REFLECT( eosio::chain::block_header_state::state_extension_v0, (security_group_info) diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 571b7287c77..c408a1cea13 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -292,6 +292,14 @@ namespace eosio { namespace chain { int64_t set_proposed_producers( vector producers ); + const security_group_info_t& active_security_group() const; + const flat_set& proposed_security_group_participants() const; + + int64_t propose_security_group_participants_add(const flat_set& participants); + int64_t propose_security_group_participants_remove(const flat_set& participants); + + bool in_active_security_group(const flat_set& participants) const; + bool light_validation_allowed() const; bool skip_auth_check()const; bool skip_trx_checks()const; diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index f0c70cda7cb..cddfa11c5b4 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "multi_index_includes.hpp" @@ -54,14 +55,6 @@ namespace eosio { namespace chain { }; } - - struct pending_security_group_t { - block_num_type block_num = 0; - uint32_t version = 0; - flat_set participants; - }; - - /** * @class global_property_object * @brief Maintains global state information about block producer schedules and chain configuration parameters @@ -80,10 +73,11 @@ namespace eosio { namespace chain { chain_id_type chain_id; kv_database_config kv_configuration; wasm_config wasm_configuration; + block_num_type proposed_security_group_block_num = 0; + flat_set proposed_security_group_participants; - std::optional pending_security_group; - - void initalize_from( const legacy::snapshot_global_property_object_v2& legacy, const chain_id_type& chain_id_val, const kv_database_config& kv_config_val, const wasm_config& wasm_config_val ) { + void initalize_from(const legacy::snapshot_global_property_object_v2& legacy, const chain_id_type& chain_id_val, + const kv_database_config& kv_config_val, const wasm_config& wasm_config_val) { proposed_schedule_block_num = legacy.proposed_schedule_block_num; proposed_schedule = producer_authority_schedule(legacy.proposed_schedule).to_shared(proposed_schedule.producers.get_allocator()); configuration = legacy.configuration; @@ -129,10 +123,11 @@ namespace eosio { namespace chain { kv_database_config kv_configuration; wasm_config wasm_configuration; - static constexpr uint32_t minimum_version_with_extension = 6; + static constexpr uint32_t minimum_version_with_extension = 6; struct extension_v0 { - std::optional pending_security_group; + block_num_type proposed_security_group_block_num = 0; + flat_set proposed_security_group_participants; }; // for future extensions, please use the following pattern: @@ -164,7 +159,8 @@ namespace eosio { namespace chain { value.chain_id, value.kv_configuration, value.wasm_configuration, - snapshot_global_property_object::extension_v0{value.pending_security_group}}; + snapshot_global_property_object::extension_v0{value.proposed_security_group_block_num, + value.proposed_security_group_participants}}; } static void from_snapshot_row(snapshot_global_property_object&& row, global_property_object& value, @@ -176,8 +172,12 @@ namespace eosio { namespace chain { value.chain_id = row.chain_id; value.kv_configuration = row.kv_configuration; value.wasm_configuration = row.wasm_configuration; - std::visit([&value](auto& ext) { value.pending_security_group = std::move(ext.pending_security_group); }, - row.extension); + std::visit( + [&value](auto& ext) { + value.proposed_security_group_block_num = ext.proposed_security_group_block_num; + value.proposed_security_group_participants = std::move(ext.proposed_security_group_participants); + }, + row.extension); } }; } @@ -227,12 +227,8 @@ FC_REFLECT(eosio::chain::legacy::snapshot_global_property_object_v4, (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id)(kv_configuration)(wasm_configuration) ) -FC_REFLECT(eosio::chain::pending_security_group_t, - (block_num)(version)(participants) - ) - FC_REFLECT(eosio::chain::snapshot_global_property_object::extension_v0, - (pending_security_group) + (proposed_security_group_block_num)(proposed_security_group_participants) ) FC_REFLECT(eosio::chain::snapshot_global_property_object, diff --git a/libraries/chain/include/eosio/chain/protocol_feature_manager.hpp b/libraries/chain/include/eosio/chain/protocol_feature_manager.hpp index 22b0df48905..bf3ecb7cfa8 100644 --- a/libraries/chain/include/eosio/chain/protocol_feature_manager.hpp +++ b/libraries/chain/include/eosio/chain/protocol_feature_manager.hpp @@ -26,7 +26,8 @@ enum class builtin_protocol_feature_t : uint32_t { action_return_value, kv_database, configurable_wasm_limits, - blockchain_parameters + blockchain_parameters, + security_group }; struct protocol_feature_subjective_restrictions { diff --git a/libraries/chain/include/eosio/chain/security_group_info.hpp b/libraries/chain/include/eosio/chain/security_group_info.hpp new file mode 100644 index 00000000000..c53bbfe59f6 --- /dev/null +++ b/libraries/chain/include/eosio/chain/security_group_info.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include +#include + +namespace eosio { +namespace chain { + +struct security_group_info_t { + uint32_t version = 0; + boost::container::flat_set participants; +}; + +} // namespace chain +} // namespace eosio + +FC_REFLECT(eosio::chain::security_group_info_t, (version)(participants)) \ No newline at end of file diff --git a/libraries/chain/protocol_feature_manager.cpp b/libraries/chain/protocol_feature_manager.cpp index 82da9deb542..9ef467cd415 100644 --- a/libraries/chain/protocol_feature_manager.cpp +++ b/libraries/chain/protocol_feature_manager.cpp @@ -230,6 +230,17 @@ Builtin protocol feature: BLOCKCHAIN_PARAMETERS Allows privileged contracts to get and set subsets of blockchain parameters. */ + ( builtin_protocol_feature_t::security_group, builtin_protocol_feature_spec{ + "SECURITY_GROUP", + fc::variant("410976abd4d5caed3308134f03f1eb5983e91eb26ac20f3a8fd34fd1b49faaea").as(), + // SHA256 hash of the raw message below within the comment delimiters (do not modify message below). +/* +Builtin protocol feature: SECURITY_GROUP + +Allows privileged contracts to add/remove participants for mutual TLS enforcement. +*/ + {} + } ) ; diff --git a/unittests/block_state_tests.cpp b/unittests/security_group_tests.cpp similarity index 71% rename from unittests/block_state_tests.cpp rename to unittests/security_group_tests.cpp index b254badf808..7630bf8484d 100644 --- a/unittests/block_state_tests.cpp +++ b/unittests/security_group_tests.cpp @@ -36,6 +36,11 @@ struct snapshot_global_property_object_v5 { , kv_configuration(value.kv_configuration) , wasm_configuration(value.wasm_configuration) {} }; + +bool operator == (const security_group_info_t& lhs, const security_group_info_t& rhs) { + return lhs.version == rhs.version && std::equal(lhs.participants.begin(), lhs.participants.end(), + rhs.participants.begin(), rhs.participants.end()); +} }} FC_REFLECT_DERIVED( eosio::chain::block_header_state_v0, (eosio::chain::detail::block_header_state_common), @@ -51,7 +56,7 @@ FC_REFLECT(eosio::chain::snapshot_global_property_object_v5, ) -BOOST_AUTO_TEST_SUITE(block_state_tests) +BOOST_AUTO_TEST_SUITE(security_group_tests) BOOST_AUTO_TEST_CASE(test_unpack_legacy_block_state) { eosio::testing::tester main; @@ -106,9 +111,8 @@ BOOST_AUTO_TEST_CASE(test_unpack_new_block_state) { auto b = main.produce_block(); auto bs = main.control->head_block_state(); - auto& sgi = bs->get_security_group_info(); - sgi.version = 1; - sgi.participants.insert("adam"_n); + bs->set_security_group_info( { .version =1, .participants={"adam"_n }} ); + // pack block_header_state as the legacy format fc::datastream> out_strm; @@ -125,8 +129,8 @@ BOOST_AUTO_TEST_CASE(test_unpack_new_block_state) { BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, tmp)); BOOST_CHECK_EQUAL(bs->id, tmp.id); BOOST_CHECK_EQUAL(bs->block->previous, tmp.block->previous); - BOOST_CHECK_EQUAL(bs->get_security_group_info().version, 1); - BOOST_TEST(bs->get_security_group_info().participants == sgi.participants); + BOOST_CHECK_EQUAL(bs->get_security_group_info().version, tmp.get_security_group_info().version); + BOOST_TEST(bs->get_security_group_info().participants == tmp.get_security_group_info().participants); BOOST_CHECK_EQUAL(in_strm.remaining(), 0); } } @@ -164,12 +168,9 @@ BOOST_AUTO_TEST_CASE(test_snapshot_global_property_object) { { // pack snapshot_global_property_object as the new format - gpo.pending_security_group = eosio::chain::pending_security_group_t { - .block_num = 10, - .version = 1, - .participants = { "adam"_n } - }; - + gpo.proposed_security_group_block_num = 10; + gpo.proposed_security_group_participants = {"adam"_n}; + fc::datastream> out_strm; fc::raw::pack(out_strm, row_traits::to_snapshot_row(gpo, main.control->db())); @@ -183,15 +184,53 @@ BOOST_AUTO_TEST_CASE(test_snapshot_global_property_object) { BOOST_CHECK_EQUAL(in_strm.remaining(), 0); BOOST_CHECK_EQUAL(row.chain_id, gpo.chain_id); - auto pending_security_group = std::visit([](auto& ext) { return ext.pending_security_group; }, row.extension); - BOOST_CHECK(pending_security_group.has_value()); - if (pending_security_group.has_value()) { - BOOST_CHECK_EQUAL(pending_security_group->block_num, gpo.pending_security_group->block_num); - BOOST_CHECK_EQUAL(pending_security_group->version, gpo.pending_security_group->version); - BOOST_TEST(pending_security_group->participants == gpo.pending_security_group->participants); - } + std::visit( + [&gpo](const auto& ext) { + BOOST_CHECK_EQUAL(ext.proposed_security_group_block_num, gpo.proposed_security_group_block_num); + BOOST_TEST(ext.proposed_security_group_participants == gpo.proposed_security_group_participants ); + }, + row.extension); } } } +BOOST_AUTO_TEST_CASE(test_participants_change) { + eosio::testing::tester chain; + using namespace eosio::chain::literals; + + chain.create_accounts( {"alice"_n,"bob"_n,"charlie"_n} ); + chain.produce_block(); + + { + const auto& cur_security_group = chain.control->active_security_group(); + BOOST_REQUIRE_EQUAL(cur_security_group.version, 0); + BOOST_REQUIRE_EQUAL(cur_security_group.participants.size(), 0); + } + + using participants_t = boost::container::flat_set; + + participants_t new_participants({"alice"_n, "bob"_n}); + chain.control->propose_security_group_participants_add(new_participants); + + BOOST_TEST(chain.control->proposed_security_group_participants() == new_participants); + BOOST_CHECK_EQUAL(chain.control->active_security_group().participants.size() , 0); + + chain.produce_block(); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 0); + BOOST_TEST(chain.control->active_security_group().participants == new_participants); + BOOST_CHECK(chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + BOOST_CHECK(!chain.control->in_active_security_group(participants_t{"bob"_n, "charlie"_n})); + + chain.control->propose_security_group_participants_remove({"alice"_n}); + BOOST_TEST(chain.control->proposed_security_group_participants() == participants_t{"bob"_n}); + + chain.produce_block(); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 0); + BOOST_TEST(chain.control->active_security_group().participants == participants_t{"bob"_n}); + BOOST_CHECK(chain.control->in_active_security_group(participants_t{"bob"_n})); + +} + BOOST_AUTO_TEST_SUITE_END() From ef5233637d17acc1f1d674cbee4851f51ada1d6b Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Fri, 5 Mar 2021 15:25:20 -0600 Subject: [PATCH 008/157] EPE-672: Feedback changes 1) Removed unused code 2) Clarified comments 3) Disabled cmake processing until a test is added --- plugins/net_plugin/CMakeLists.txt | 3 ++- plugins/net_plugin/test/main.cpp | 20 ++------------------ 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/plugins/net_plugin/CMakeLists.txt b/plugins/net_plugin/CMakeLists.txt index 86404f6ddef..b30c2370878 100644 --- a/plugins/net_plugin/CMakeLists.txt +++ b/plugins/net_plugin/CMakeLists.txt @@ -6,5 +6,6 @@ add_library( net_plugin target_link_libraries( net_plugin chain_plugin producer_plugin appbase fc ) target_include_directories( net_plugin PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/../chain_interface/include "${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/appbase/include") -add_subdirectory(test) +# once tests are added to the test subdirectory, the following line should be uncommented +# add_subdirectory(test) diff --git a/plugins/net_plugin/test/main.cpp b/plugins/net_plugin/test/main.cpp index eb7cbf7cae7..810ec77d511 100644 --- a/plugins/net_plugin/test/main.cpp +++ b/plugins/net_plugin/test/main.cpp @@ -1,19 +1,14 @@ -#include -#include #include -#include #include -//extern uint32_t EOS_TESTING_GENESIS_TIMESTAMP; - void translate_fc_exception(const fc::exception &e) { std::cerr << "\033[33m" << e.to_detail_string() << "\033[0m" << std::endl; BOOST_TEST_FAIL("Caught Unexpected Exception"); } boost::unit_test::test_suite* init_unit_test_suite(int argc, char* argv[]) { - // Turn off blockchain logging if no --verbose parameter is not added - // To have verbose enabled, call "tests/chain_test -- --verbose" + // Turn off logging if --verbose parameter not provided + // To enable, add --verbose to the command line bool is_verbose = false; std::string verbose_arg = "--verbose"; for (int i = 0; i < argc; i++) { @@ -26,16 +21,5 @@ boost::unit_test::test_suite* init_unit_test_suite(int argc, char* argv[]) { // Register fc::exception translator boost::unit_test::unit_test_monitor.register_exception_translator(&translate_fc_exception); - - std::srand(time(NULL)); - std::cout << "Random number generator seeded to " << time(NULL) << std::endl; - /* - const char* genesis_timestamp_str = getenv("EOS_TESTING_GENESIS_TIMESTAMP"); - if( genesis_timestamp_str != nullptr ) - { - EOS_TESTING_GENESIS_TIMESTAMP = std::stoul( genesis_timestamp_str ); - } - std::cout << "EOS_TESTING_GENESIS_TIMESTAMP is " << EOS_TESTING_GENESIS_TIMESTAMP << std::endl; - */ return nullptr; } From cf36ebf447f1abb4b0a4174fae26349e2369c0c2 Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Thu, 4 Mar 2021 16:12:00 -0600 Subject: [PATCH 009/157] EPE-721: Add connection management --- plugins/net_plugin/CMakeLists.txt | 4 + .../eosio/net_plugin/security_group.hpp | 83 ++++++++++++++ plugins/net_plugin/net_plugin.cpp | 104 +++++++++++++++++ plugins/net_plugin/security_group.cpp | 64 +++++++++++ plugins/net_plugin/test/CMakeLists.txt | 24 ++++ plugins/net_plugin/test/main.cpp | 41 +++++++ .../net_plugin/test/security_group_tests.cpp | 108 ++++++++++++++++++ tests/CMakeLists.txt | 2 +- 8 files changed, 429 insertions(+), 1 deletion(-) create mode 100644 plugins/net_plugin/include/eosio/net_plugin/security_group.hpp create mode 100644 plugins/net_plugin/security_group.cpp create mode 100644 plugins/net_plugin/test/CMakeLists.txt create mode 100644 plugins/net_plugin/test/main.cpp create mode 100644 plugins/net_plugin/test/security_group_tests.cpp diff --git a/plugins/net_plugin/CMakeLists.txt b/plugins/net_plugin/CMakeLists.txt index 3b8c1cd7b71..36b2b7bcdb0 100644 --- a/plugins/net_plugin/CMakeLists.txt +++ b/plugins/net_plugin/CMakeLists.txt @@ -1,7 +1,11 @@ file(GLOB HEADERS "include/eosio/net_plugin/*.hpp" ) add_library( net_plugin net_plugin.cpp + security_group.cpp ${HEADERS} ) target_link_libraries( net_plugin chain_plugin producer_plugin appbase fc ) target_include_directories( net_plugin PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/../chain_interface/include "${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/appbase/include") + +add_subdirectory(test) + diff --git a/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp b/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp new file mode 100644 index 00000000000..e01ab61595f --- /dev/null +++ b/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp @@ -0,0 +1,83 @@ +#pragma once +#include + +#include + +#include +#include + +namespace eosio { + /** \brief Describes a security group participant */ + class security_group_participant { + public: + /** \brief Simple initialization + * + * @param account_name The recognized name of the participant + * @param participating Indicates the participant's participation (default = true) + * true - participating + * false - sync'ing only + */ + explicit security_group_participant(chain::account_name account_name, bool participating = true) : + participant_(account_name), participating_(participating) {} + security_group_participant(const security_group_participant& that) : + participant_(that.participant_), participating_(that.is_participating()) {} + ~security_group_participant() = default; + + auto name() const { return participant_; } + + [[maybe_unused]] + bool is_participating() const { return participating_.load(std::memory_order_relaxed); } + bool is_syncing() const { return !participating_.load(std::memory_order_relaxed); } + void now_syncing() { participating_.store(false, std::memory_order_relaxed); } + void now_participating() { participating_.store(true, std::memory_order_relaxed); } + + security_group_participant& operator=(const security_group_participant& that) { + participant_ = that.participant_; + participating_.store(that.is_participating(), std::memory_order_relaxed); + return *this; + } + + bool operator==(const security_group_participant& that) const { return that.participant_ == participant_; } + bool operator!=(const security_group_participant& that) const { return !operator==(that); } + bool operator<(const security_group_participant& that) const { return participant_ < that.participant_; } + bool operator>(const security_group_participant& that) const { return participant_ > that.participant_; } + + bool operator==(chain::account_name account_name) const { return account_name == participant_; } + + private: + chain::account_name participant_; //!< Identifier for this participant + std::atomic participating_ {true}; //!< Indicates if participant producing blocks (true) or + //!< only used for sync'ing + }; + + /** \brief Describes the security group cache */ + class security_group_manager { + public: + security_group_manager() = default; + security_group_manager(const security_group_manager&) = delete; + security_group_manager(security_group_manager&&) = delete; + + ~security_group_manager() = default; + + using account_name_list_t = boost::container::flat_set; + using callback_t = std::function; + /** Update the cache using new information from a block + * + * @param version The version number for this update + * @param participant_list The list of accounts for the security group + * @param on_participating Callback when an account is added to the cache + * @param on_syncing Callback when an account is "removed" from the cache + */ + void update_cache(const uint32_t version, const account_name_list_t& participant_list, callback_t on_participating, + callback_t on_syncing); + + using cache_t = boost::container::flat_set; + const cache_t& security_group() const { return cache_; } + + security_group_manager& operator==(const security_group_manager&) = delete; + security_group_manager& operator==(security_group_manager&&) = delete; + private: + uint32_t version_ {std::numeric_limits::max()}; ///! The security group version + cache_t cache_; ///! Cache of accounts in the security group + }; +} diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index d3fa117d29c..3d6e7736808 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -27,6 +28,7 @@ #include #include +#include using namespace eosio::chain::plugin_interface; @@ -336,6 +338,17 @@ namespace eosio { constexpr static uint16_t to_protocol_version(uint16_t v); connection_ptr find_connection(const string& host)const; // must call with held mutex + /** private security group management */ + security_group_manager security_group; + /** \brief Update security group information + * + * Retrieves the security group information from the indicated block and + * updates the cached security group information. If a participant is + * removed from the cached security group, the TLS connection is closed. + * + * \param bs The block_state containing the security group update + */ + void update_security_group(const block_state_ptr& bs); }; const fc::string logger_name("net_plugin_impl"); @@ -575,7 +588,20 @@ namespace eosio { class connection : public std::enable_shared_from_this { public: + /** @brief For non TLS connection setup + * + * @param endpoint The connection endpoint + */ explicit connection( string endpoint ); + /** @brief For TLS connection setup + * + * @param endpoint The connection endpoint + * @param participant The name of the participant + * @param participaing Indicates if the participant is participating in block creation (default = true) + * true - block creation + * false - syncing only + */ + explicit connection( string endpoint, chain::account_name participant, bool participaing = true ); connection(); ~connection() {} @@ -789,6 +815,32 @@ namespace eosio { ( "_lport", local_endpoint_port ); return mvo; } + + // security group management + // + private: + std::optional security_group_info_; + + public: + /** @brief Set/change the participant name for the connection + * + * @param name The name of the participant + */ + void participant_name(chain::account_name name) { + security_group_info_ = security_group_participant(name); + } + /** @brief Returns the optional name of the participant */ + std::optional participant_name(); + bool is_participating() const { + return security_group_info_ && security_group_info_->is_participating(); + } + bool is_syncing() const { + return security_group_info_ && security_group_info_->is_syncing(); + } + /** @brief Flag the tls connection as participating in block production */ + void now_participating(); + /** @brief Flag the connection as only publishing sync information */ + void now_syncing(); }; const string connection::unknown = ""; @@ -877,6 +929,12 @@ namespace eosio { fc_ilog( logger, "creating connection to ${n}", ("n", endpoint) ); } + connection::connection( string endpoint, chain::account_name participant, bool participating ) + : connection(endpoint) + { + security_group_info_ = security_group_participant(participant, participating); + } + connection::connection() : peer_addr(), strand( my_impl->thread_pool->get_executor() ), @@ -1288,6 +1346,22 @@ namespace eosio { return true; } + std::optional connection::participant_name() { + if(security_group_info_) + return security_group_info_.value().name(); + return std::optional(); + } + + void connection::now_participating() { + if(security_group_info_) + security_group_info_.value().now_participating(); + } + + void connection::now_syncing() { + if(security_group_info_) + security_group_info_.value().now_syncing(); + } + //------------------------------------------------------------------------ using send_buffer_type = std::shared_ptr>; @@ -3428,10 +3502,40 @@ namespace eosio { } } + // called from any thread + void net_plugin_impl::update_security_group(const block_state_ptr& bs) { + // update connections + // + auto updater = [](chain::account_name name, auto handler) { + for_each_connection([name, handler](auto& cp) { + auto participant_name = cp->participant_name(); + if(participant_name && participant_name.value() == name) { + handler(*cp); + } + return true; + }); + }; + // add a new participant to the security group + // + auto on_add = [updater](chain::account_name name) { + updater(name, [](connection& con) { con.now_participating(); }); + }; + // remove a participant from the security group + // + auto on_remove = [updater](chain::account_name name) { + updater(name, [](connection& con) { con.now_syncing(); }); + }; + // update cache + // + auto& update = bs->get_security_group_info(); + security_group.update_cache(update.version, update.participants, on_add, on_remove); + } + // called from application thread void net_plugin_impl::on_accepted_block(const block_state_ptr& bs) { update_chain_info(); controller& cc = chain_plug->chain(); + update_security_group(bs); dispatcher->strand.post( [this, bs]() { fc_dlog( logger, "signaled accepted_block, blk num = ${num}, id = ${id}", ("num", bs->block_num)("id", bs->id) ); diff --git a/plugins/net_plugin/security_group.cpp b/plugins/net_plugin/security_group.cpp new file mode 100644 index 00000000000..af31e19cb6f --- /dev/null +++ b/plugins/net_plugin/security_group.cpp @@ -0,0 +1,64 @@ +#include + +namespace eosio { + // Cache update looks at 3 scenarios: + // 1) The cache is empty and is being populated + // 2) The update is empty so all accounts are being removed from the cache + // 3) The cache is populated and accouts are being added and/or removed + void security_group_manager::update_cache(const uint32_t version, const account_name_list_t& participant_list, + callback_t on_participating, callback_t on_syncing) { + // update on version change + if(version == version_) + return; + version_ = version; + + // Scenario 1: The cache is empty and being populated + // + if(cache_.empty()) { + if(participant_list.empty()) { + return; + } + for(auto participant : participant_list) { + cache_.emplace(participant); + on_participating(participant); + } + return; + } + + // Scenaio 2: The update is empty so all accounts are being removed from the cache + // + if(participant_list.empty()) { + for(auto& participant : cache_) { + if(participant.is_participating()) { + participant.now_syncing(); + on_syncing(participant.name()); + } + } + return; + } + + // Scenario 3: Accounts are added and/or removed from cache + // + // 1) Update status of existing accounts + // 2) Add new accounts + // + for(auto& participant : cache_) { + if(participant_list.find(participant.name()) != participant_list.end()) { + if(participant.is_syncing()) { + participant.now_participating(); + on_participating(participant.name()); + } + } + else { + participant.now_syncing(); + on_syncing(participant.name()); + } + } + + for(const auto& participant : participant_list) { + if(cache_.emplace(participant).second) { + on_participating(participant); + } + } + } +} diff --git a/plugins/net_plugin/test/CMakeLists.txt b/plugins/net_plugin/test/CMakeLists.txt new file mode 100644 index 00000000000..9b19540f673 --- /dev/null +++ b/plugins/net_plugin/test/CMakeLists.txt @@ -0,0 +1,24 @@ +file(GLOB UNIT_TESTS "*.cpp") + +add_executable( net_plugin_tests ${UNIT_TESTS} ) + +target_link_libraries( net_plugin_tests + net_plugin + chain_plugin + producer_plugin + appbase + fc + ${PLATFORM_SPECIFIC_LIBS} + eosio_testing + eosio_chain + ) +target_include_directories( net_plugin_tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/plugins/chain_interface/include + ${CMAKE_SOURCE_DIR}/libraries/appbase/include + ${CMAKE_SOURCE_DIR}/plugins/net_plugin/include + ${CMAKE_SOURCE_DIR}/plugins/chain_plugin/include + ${CMAKE_SOURCE_DIR}/libraries/testing/include + ${CMAKE_SOURCE_DIR}/cmake-build-debug/unittests/include + ) + +add_test(NAME net_plugin_tests COMMAND plugins/net_plugin/test/net_plugin_tests WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/plugins/net_plugin/test/main.cpp b/plugins/net_plugin/test/main.cpp new file mode 100644 index 00000000000..eb7cbf7cae7 --- /dev/null +++ b/plugins/net_plugin/test/main.cpp @@ -0,0 +1,41 @@ +#include +#include +#include +#include +#include + +//extern uint32_t EOS_TESTING_GENESIS_TIMESTAMP; + +void translate_fc_exception(const fc::exception &e) { + std::cerr << "\033[33m" << e.to_detail_string() << "\033[0m" << std::endl; + BOOST_TEST_FAIL("Caught Unexpected Exception"); +} + +boost::unit_test::test_suite* init_unit_test_suite(int argc, char* argv[]) { + // Turn off blockchain logging if no --verbose parameter is not added + // To have verbose enabled, call "tests/chain_test -- --verbose" + bool is_verbose = false; + std::string verbose_arg = "--verbose"; + for (int i = 0; i < argc; i++) { + if (verbose_arg == argv[i]) { + is_verbose = true; + break; + } + } + if(!is_verbose) fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::off); + + // Register fc::exception translator + boost::unit_test::unit_test_monitor.register_exception_translator(&translate_fc_exception); + + std::srand(time(NULL)); + std::cout << "Random number generator seeded to " << time(NULL) << std::endl; + /* + const char* genesis_timestamp_str = getenv("EOS_TESTING_GENESIS_TIMESTAMP"); + if( genesis_timestamp_str != nullptr ) + { + EOS_TESTING_GENESIS_TIMESTAMP = std::stoul( genesis_timestamp_str ); + } + std::cout << "EOS_TESTING_GENESIS_TIMESTAMP is " << EOS_TESTING_GENESIS_TIMESTAMP << std::endl; + */ + return nullptr; +} diff --git a/plugins/net_plugin/test/security_group_tests.cpp b/plugins/net_plugin/test/security_group_tests.cpp new file mode 100644 index 00000000000..77fe212b84e --- /dev/null +++ b/plugins/net_plugin/test/security_group_tests.cpp @@ -0,0 +1,108 @@ +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace { + using participant_list_t = boost::container::flat_set; + participant_list_t create_list(std::vector participants) { + participant_list_t participant_list; + for(auto participant : participants) { + participant_list.emplace(participant); + } + return participant_list; + } + auto dummy_add = [](eosio::chain::account_name) {}; + auto dummy_remove = [](eosio::chain::account_name) {}; +} + +BOOST_AUTO_TEST_SUITE(security_group_tests) +using namespace eosio::testing; +BOOST_AUTO_TEST_CASE(test_initial_population) { + auto populate = create_list({ 1, 2, 3, 4, 5, 6}); + size_t add_count = 0; + auto on_add = [&](eosio::chain::account_name name) { + BOOST_REQUIRE(populate.find(name) != populate.end()); + ++add_count; + }; + + auto on_rem = [&](eosio::chain::account_name name) { + BOOST_FAIL("Name removed from security group"); + }; + + eosio::security_group_manager manager; + manager.update_cache(1, populate, on_add, on_rem); + + BOOST_REQUIRE_EQUAL(add_count, populate.size()); +} + +BOOST_AUTO_TEST_CASE(test_remove_all) { + auto populate = create_list({1, 2, 3, 4, 5, 6}); + eosio::security_group_manager manager; + manager.update_cache(1, populate, dummy_add, dummy_remove); + + auto check_add = [](eosio::chain::account_name) { + BOOST_FAIL("Tried to add name from empty list"); + }; + + size_t remove_count = 0; + auto check_remove = [&remove_count](eosio::chain::account_name) { ++remove_count; }; + + participant_list_t clear; + manager.update_cache(2, clear, check_add, check_remove); + BOOST_REQUIRE_EQUAL(populate.size(), remove_count); +} + +BOOST_AUTO_TEST_CASE(test_add_only) { + auto populate = create_list({1, 2, 3, 4, 5, 6}); + eosio::security_group_manager manager; + manager.update_cache(1, populate, dummy_add, dummy_remove); + + size_t add_count = 0; + auto check_add = [&add_count](eosio::chain::account_name) { ++add_count; }; + auto check_remove = [](eosio::chain::account_name) { BOOST_FAIL("Nothing should be removed");}; + + auto update = create_list({1, 2, 3, 4, 5, 6, 7, 8, 9}); + manager.update_cache(2, update, check_add, check_remove); + BOOST_REQUIRE_EQUAL((update.size() - populate.size()), add_count); +} + +BOOST_AUTO_TEST_CASE(test_remove_only) { + auto populate = create_list({1, 2, 3, 4, 5, 6}); + eosio::security_group_manager manager; + manager.update_cache(1, populate, dummy_add, dummy_remove); + + auto check_add = [](eosio::chain::account_name) { BOOST_FAIL("Nothing should be added");}; + size_t remove_count = 0; + auto check_remove = [&remove_count](eosio::chain::account_name) { ++remove_count; }; + + auto update = create_list({2, 4, 6}); + manager.update_cache(2, update, check_add, check_remove); + BOOST_REQUIRE_EQUAL(populate.size() - update.size(), remove_count); +} + +BOOST_AUTO_TEST_CASE(test_update) { + auto populate = create_list({1, 2, 3, 4, 5, 6}); + eosio::security_group_manager manager; + manager.update_cache(1, populate, dummy_add, dummy_remove); + + size_t add_count = 0; + auto check_add = [&add_count](eosio::chain::account_name) { ++add_count;}; + size_t remove_count = 0; + auto check_remove = [&remove_count](eosio::chain::account_name) { ++remove_count; }; + + auto update = create_list({2, 4, 6, 7, 8, 9}); + manager.update_cache(2, update, check_add, check_remove); + BOOST_REQUIRE_EQUAL(add_count + remove_count, update.size()); +} +BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0d1cce56fe7..63b05f6974d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,7 +5,7 @@ include_directories( "${CMAKE_SOURCE_DIR}/plugins/wallet_plugin/include" ) file(GLOB UNIT_TESTS "*.cpp") add_executable( plugin_test ${UNIT_TESTS} ) -target_link_libraries( plugin_test eosio_testing eosio_chain chainbase chain_plugin wallet_plugin fc ${PLATFORM_SPECIFIC_LIBS} ) +target_link_libraries( plugin_test eosio_testing eosio_chain chainbase chain_plugin wallet_plugin fc net_plugin ${PLATFORM_SPECIFIC_LIBS} ) target_include_directories( plugin_test PUBLIC ${CMAKE_SOURCE_DIR}/plugins/net_plugin/include From 992ef2b95be91c46c00f3886ea5d5a0f77e81e84 Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Mon, 8 Mar 2021 12:06:08 -0600 Subject: [PATCH 010/157] EPE-672: add security group Adding security group cache manager and unittest --- plugins/net_plugin/CMakeLists.txt | 4 +- .../eosio/net_plugin/security_group.hpp | 83 ++++++++++++++ plugins/net_plugin/security_group.cpp | 64 +++++++++++ .../net_plugin/test/security_group_tests.cpp | 108 ++++++++++++++++++ 4 files changed, 257 insertions(+), 2 deletions(-) create mode 100644 plugins/net_plugin/include/eosio/net_plugin/security_group.hpp create mode 100644 plugins/net_plugin/security_group.cpp create mode 100644 plugins/net_plugin/test/security_group_tests.cpp diff --git a/plugins/net_plugin/CMakeLists.txt b/plugins/net_plugin/CMakeLists.txt index b30c2370878..36b2b7bcdb0 100644 --- a/plugins/net_plugin/CMakeLists.txt +++ b/plugins/net_plugin/CMakeLists.txt @@ -1,11 +1,11 @@ file(GLOB HEADERS "include/eosio/net_plugin/*.hpp" ) add_library( net_plugin net_plugin.cpp + security_group.cpp ${HEADERS} ) target_link_libraries( net_plugin chain_plugin producer_plugin appbase fc ) target_include_directories( net_plugin PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/../chain_interface/include "${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/appbase/include") -# once tests are added to the test subdirectory, the following line should be uncommented -# add_subdirectory(test) +add_subdirectory(test) diff --git a/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp b/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp new file mode 100644 index 00000000000..e01ab61595f --- /dev/null +++ b/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp @@ -0,0 +1,83 @@ +#pragma once +#include + +#include + +#include +#include + +namespace eosio { + /** \brief Describes a security group participant */ + class security_group_participant { + public: + /** \brief Simple initialization + * + * @param account_name The recognized name of the participant + * @param participating Indicates the participant's participation (default = true) + * true - participating + * false - sync'ing only + */ + explicit security_group_participant(chain::account_name account_name, bool participating = true) : + participant_(account_name), participating_(participating) {} + security_group_participant(const security_group_participant& that) : + participant_(that.participant_), participating_(that.is_participating()) {} + ~security_group_participant() = default; + + auto name() const { return participant_; } + + [[maybe_unused]] + bool is_participating() const { return participating_.load(std::memory_order_relaxed); } + bool is_syncing() const { return !participating_.load(std::memory_order_relaxed); } + void now_syncing() { participating_.store(false, std::memory_order_relaxed); } + void now_participating() { participating_.store(true, std::memory_order_relaxed); } + + security_group_participant& operator=(const security_group_participant& that) { + participant_ = that.participant_; + participating_.store(that.is_participating(), std::memory_order_relaxed); + return *this; + } + + bool operator==(const security_group_participant& that) const { return that.participant_ == participant_; } + bool operator!=(const security_group_participant& that) const { return !operator==(that); } + bool operator<(const security_group_participant& that) const { return participant_ < that.participant_; } + bool operator>(const security_group_participant& that) const { return participant_ > that.participant_; } + + bool operator==(chain::account_name account_name) const { return account_name == participant_; } + + private: + chain::account_name participant_; //!< Identifier for this participant + std::atomic participating_ {true}; //!< Indicates if participant producing blocks (true) or + //!< only used for sync'ing + }; + + /** \brief Describes the security group cache */ + class security_group_manager { + public: + security_group_manager() = default; + security_group_manager(const security_group_manager&) = delete; + security_group_manager(security_group_manager&&) = delete; + + ~security_group_manager() = default; + + using account_name_list_t = boost::container::flat_set; + using callback_t = std::function; + /** Update the cache using new information from a block + * + * @param version The version number for this update + * @param participant_list The list of accounts for the security group + * @param on_participating Callback when an account is added to the cache + * @param on_syncing Callback when an account is "removed" from the cache + */ + void update_cache(const uint32_t version, const account_name_list_t& participant_list, callback_t on_participating, + callback_t on_syncing); + + using cache_t = boost::container::flat_set; + const cache_t& security_group() const { return cache_; } + + security_group_manager& operator==(const security_group_manager&) = delete; + security_group_manager& operator==(security_group_manager&&) = delete; + private: + uint32_t version_ {std::numeric_limits::max()}; ///! The security group version + cache_t cache_; ///! Cache of accounts in the security group + }; +} diff --git a/plugins/net_plugin/security_group.cpp b/plugins/net_plugin/security_group.cpp new file mode 100644 index 00000000000..af31e19cb6f --- /dev/null +++ b/plugins/net_plugin/security_group.cpp @@ -0,0 +1,64 @@ +#include + +namespace eosio { + // Cache update looks at 3 scenarios: + // 1) The cache is empty and is being populated + // 2) The update is empty so all accounts are being removed from the cache + // 3) The cache is populated and accouts are being added and/or removed + void security_group_manager::update_cache(const uint32_t version, const account_name_list_t& participant_list, + callback_t on_participating, callback_t on_syncing) { + // update on version change + if(version == version_) + return; + version_ = version; + + // Scenario 1: The cache is empty and being populated + // + if(cache_.empty()) { + if(participant_list.empty()) { + return; + } + for(auto participant : participant_list) { + cache_.emplace(participant); + on_participating(participant); + } + return; + } + + // Scenaio 2: The update is empty so all accounts are being removed from the cache + // + if(participant_list.empty()) { + for(auto& participant : cache_) { + if(participant.is_participating()) { + participant.now_syncing(); + on_syncing(participant.name()); + } + } + return; + } + + // Scenario 3: Accounts are added and/or removed from cache + // + // 1) Update status of existing accounts + // 2) Add new accounts + // + for(auto& participant : cache_) { + if(participant_list.find(participant.name()) != participant_list.end()) { + if(participant.is_syncing()) { + participant.now_participating(); + on_participating(participant.name()); + } + } + else { + participant.now_syncing(); + on_syncing(participant.name()); + } + } + + for(const auto& participant : participant_list) { + if(cache_.emplace(participant).second) { + on_participating(participant); + } + } + } +} diff --git a/plugins/net_plugin/test/security_group_tests.cpp b/plugins/net_plugin/test/security_group_tests.cpp new file mode 100644 index 00000000000..77fe212b84e --- /dev/null +++ b/plugins/net_plugin/test/security_group_tests.cpp @@ -0,0 +1,108 @@ +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace { + using participant_list_t = boost::container::flat_set; + participant_list_t create_list(std::vector participants) { + participant_list_t participant_list; + for(auto participant : participants) { + participant_list.emplace(participant); + } + return participant_list; + } + auto dummy_add = [](eosio::chain::account_name) {}; + auto dummy_remove = [](eosio::chain::account_name) {}; +} + +BOOST_AUTO_TEST_SUITE(security_group_tests) +using namespace eosio::testing; +BOOST_AUTO_TEST_CASE(test_initial_population) { + auto populate = create_list({ 1, 2, 3, 4, 5, 6}); + size_t add_count = 0; + auto on_add = [&](eosio::chain::account_name name) { + BOOST_REQUIRE(populate.find(name) != populate.end()); + ++add_count; + }; + + auto on_rem = [&](eosio::chain::account_name name) { + BOOST_FAIL("Name removed from security group"); + }; + + eosio::security_group_manager manager; + manager.update_cache(1, populate, on_add, on_rem); + + BOOST_REQUIRE_EQUAL(add_count, populate.size()); +} + +BOOST_AUTO_TEST_CASE(test_remove_all) { + auto populate = create_list({1, 2, 3, 4, 5, 6}); + eosio::security_group_manager manager; + manager.update_cache(1, populate, dummy_add, dummy_remove); + + auto check_add = [](eosio::chain::account_name) { + BOOST_FAIL("Tried to add name from empty list"); + }; + + size_t remove_count = 0; + auto check_remove = [&remove_count](eosio::chain::account_name) { ++remove_count; }; + + participant_list_t clear; + manager.update_cache(2, clear, check_add, check_remove); + BOOST_REQUIRE_EQUAL(populate.size(), remove_count); +} + +BOOST_AUTO_TEST_CASE(test_add_only) { + auto populate = create_list({1, 2, 3, 4, 5, 6}); + eosio::security_group_manager manager; + manager.update_cache(1, populate, dummy_add, dummy_remove); + + size_t add_count = 0; + auto check_add = [&add_count](eosio::chain::account_name) { ++add_count; }; + auto check_remove = [](eosio::chain::account_name) { BOOST_FAIL("Nothing should be removed");}; + + auto update = create_list({1, 2, 3, 4, 5, 6, 7, 8, 9}); + manager.update_cache(2, update, check_add, check_remove); + BOOST_REQUIRE_EQUAL((update.size() - populate.size()), add_count); +} + +BOOST_AUTO_TEST_CASE(test_remove_only) { + auto populate = create_list({1, 2, 3, 4, 5, 6}); + eosio::security_group_manager manager; + manager.update_cache(1, populate, dummy_add, dummy_remove); + + auto check_add = [](eosio::chain::account_name) { BOOST_FAIL("Nothing should be added");}; + size_t remove_count = 0; + auto check_remove = [&remove_count](eosio::chain::account_name) { ++remove_count; }; + + auto update = create_list({2, 4, 6}); + manager.update_cache(2, update, check_add, check_remove); + BOOST_REQUIRE_EQUAL(populate.size() - update.size(), remove_count); +} + +BOOST_AUTO_TEST_CASE(test_update) { + auto populate = create_list({1, 2, 3, 4, 5, 6}); + eosio::security_group_manager manager; + manager.update_cache(1, populate, dummy_add, dummy_remove); + + size_t add_count = 0; + auto check_add = [&add_count](eosio::chain::account_name) { ++add_count;}; + size_t remove_count = 0; + auto check_remove = [&remove_count](eosio::chain::account_name) { ++remove_count; }; + + auto update = create_list({2, 4, 6, 7, 8, 9}); + manager.update_cache(2, update, check_add, check_remove); + BOOST_REQUIRE_EQUAL(add_count + remove_count, update.size()); +} +BOOST_AUTO_TEST_SUITE_END() From efeef44aa5f5c475a4dfb9ed2f994d36e000686a Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 8 Mar 2021 12:17:20 -0600 Subject: [PATCH 011/157] implements security group intrinsics --- libraries/chain/CMakeLists.txt | 1 + libraries/chain/controller.cpp | 12 ++ libraries/chain/fork_database.cpp | 9 +- .../eosio/chain/block_header_state.hpp | 11 +- .../eosio/chain/global_property_object.hpp | 2 +- .../eosio/chain/security_group_info.hpp | 18 --- .../eos-vm-oc/intrinsic_mapping.hpp | 6 +- .../eosio/chain/webassembly/interface.hpp | 42 +++++ .../chain/webassembly/runtimes/eos-vm.cpp | 6 + .../chain/webassembly/security_group.cpp | 43 +++++ unittests/security_group_tests.cpp | 149 +++++++++++++++++- 11 files changed, 270 insertions(+), 29 deletions(-) delete mode 100644 libraries/chain/include/eosio/chain/security_group_info.hpp create mode 100644 libraries/chain/webassembly/security_group.cpp diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index d4c1bbb6e43..7613d66422b 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -71,6 +71,7 @@ set(CHAIN_WEBASSEMBLY_SOURCES webassembly/softfloat.cpp webassembly/system.cpp webassembly/transaction.cpp + webassembly/security_group.cpp ) ## SORT .cpp by most likely to change / break compile diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index c3a1b1bbf5f..445638ef2a8 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -252,6 +252,7 @@ struct controller_impl { set_activation_handler(); set_activation_handler(); set_activation_handler(); + set_activation_handler(); self.irreversible_block.connect([this](const block_state_ptr& bsp) { wasmif.current_lib(bsp->block_num); @@ -3402,6 +3403,17 @@ void controller_impl::on_activation +void controller_impl::on_activation() { + db.modify( db.get(), [&]( auto& ps ) { + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "propose_security_group_participants_add" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "propose_security_group_participants_remove" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "in_active_security_group" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "get_active_security_group" ); + } ); +} + /// End of protocol feature activation handlers } } /// eosio::chain diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp index 446bd1141ad..b74414d2a12 100644 --- a/libraries/chain/fork_database.cpp +++ b/libraries/chain/fork_database.cpp @@ -123,7 +123,14 @@ namespace eosio { namespace chain { ("max", max_supported_version) ); - versioned_unpack_stream unpack_strm(ds, version - min_supported_version + block_header_state::minimum_version_with_state_extension -1); + // The unpack_strm here is used only to unpack `block_header_state` and `block_state`. However, those two + // classes are written to unpack based on the snapshot version; therefore, we orient it to the snapshot version. + + const bool has_block_header_state_extension = version > min_supported_version; + versioned_unpack_stream unpack_strm( + ds, has_block_header_state_extension + ? block_header_state::minimum_snapshot_version_with_state_extension + : block_header_state::minimum_snapshot_version_with_state_extension - 1); block_header_state bhs; fc::raw::unpack( unpack_strm, bhs ); diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index 668136fb7b7..5426b251ff2 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -4,7 +4,6 @@ #include #include #include -#include #include namespace eosio { namespace chain { @@ -49,6 +48,11 @@ using signer_callback_type = std::function(const dig struct block_header_state; +struct security_group_info_t { + uint32_t version = 0; + boost::container::flat_set participants; +}; + namespace detail { struct block_header_state_common { uint32_t block_num = 0; @@ -122,7 +126,7 @@ struct pending_block_header_state : public detail::block_header_state_common { struct block_header_state : public detail::block_header_state_common { /// this version is coming from chain_snapshot_header.version - static constexpr uint32_t minimum_version_with_state_extension = 6; + static constexpr uint32_t minimum_snapshot_version_with_state_extension = 6; block_id_type id; signed_block_header header; @@ -206,6 +210,7 @@ FC_REFLECT( eosio::chain::detail::schedule_info, (schedule) ) +FC_REFLECT(eosio::chain::security_group_info_t, (version)(participants)) FC_REFLECT( eosio::chain::block_header_state::state_extension_v0, (security_group_info) @@ -268,7 +273,7 @@ struct unpack_block_header_state_derived_visitor : fc::reflector_init_visitorobj.*p)>>) - if (s.version < eosio::chain::block_header_state::minimum_version_with_state_extension) + if (s.version < eosio::chain::block_header_state::minimum_snapshot_version_with_state_extension) return; fc::raw::unpack(s, this->obj.*p); diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index cddfa11c5b4..8110f9e736f 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include "multi_index_includes.hpp" @@ -213,6 +212,7 @@ CHAINBASE_SET_INDEX_TYPE(eosio::chain::dynamic_global_property_object, FC_REFLECT(eosio::chain::global_property_object, (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id)(kv_configuration)(wasm_configuration) + (proposed_security_group_block_num)(proposed_security_group_participants) ) FC_REFLECT(eosio::chain::legacy::snapshot_global_property_object_v2, diff --git a/libraries/chain/include/eosio/chain/security_group_info.hpp b/libraries/chain/include/eosio/chain/security_group_info.hpp deleted file mode 100644 index c53bbfe59f6..00000000000 --- a/libraries/chain/include/eosio/chain/security_group_info.hpp +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace eosio { -namespace chain { - -struct security_group_info_t { - uint32_t version = 0; - boost::container::flat_set participants; -}; - -} // namespace chain -} // namespace eosio - -FC_REFLECT(eosio::chain::security_group_info_t, (version)(participants)) \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp index fdf5d162ce8..9cec37f1a97 100644 --- a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp @@ -277,7 +277,11 @@ inline constexpr auto get_intrinsic_table() { "env.get_wasm_parameters_packed", "env.set_wasm_parameters_packed", "env.get_parameters_packed", - "env.set_parameters_packed" + "env.set_parameters_packed", + "env.propose_security_group_participants_add", + "env.propose_security_group_participants_remove", + "env.in_active_security_group", + "env.get_active_security_group" ); } inline constexpr std::size_t find_intrinsic_index(std::string_view hf) { diff --git a/libraries/chain/include/eosio/chain/webassembly/interface.hpp b/libraries/chain/include/eosio/chain/webassembly/interface.hpp index bc53b9ee72d..ba0093650f0 100644 --- a/libraries/chain/include/eosio/chain/webassembly/interface.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/interface.hpp @@ -1959,6 +1959,48 @@ namespace webassembly { int32_t __lttf2(uint64_t, uint64_t, uint64_t, uint64_t) const; int32_t __unordtf2(uint64_t, uint64_t, uint64_t, uint64_t) const; + // security group api + /** + * Propose new participants to the security group. + * + * @ingroup security-group + * @param packed_participants - the buffer containing the packed participants. + * + * @return -1 if proposing a new security group was unsuccessful, otherwise returns 0. + */ + int64_t propose_security_group_participants_add(span packed_participants); + + /** + * Propose to remove participants from the security group. + * + * @ingroup security-group + * @param packed_participants - the buffer containing the packed participants. + * + * @return -1 if proposing a new security group was unsuccessful, otherwise returns 0. + */ + int64_t propose_security_group_participants_remove(span packed_participants); + + /** + * Check if the specified accounts are all in the active security group. + * + * @ingroup security-group + * @param packed_participants - the buffer containing the packed participants. + * + * @return Returns true if the specified accounts are all in the active security group. + */ + bool in_active_security_group(span packed_participants) const; + + /** + * Gets the active security group + * + * @ingroup security-group + * @param[out] packed_security_group - the buffer containing the packed security_group. + * + * @return Returns the size required in the buffer (if the buffer is too small, nothing is written). + * + */ + uint32_t get_active_security_group(span packed_security_group) const; + private: apply_context& context; }; diff --git a/libraries/chain/webassembly/runtimes/eos-vm.cpp b/libraries/chain/webassembly/runtimes/eos-vm.cpp index 1c53b62d654..7654b844ded 100644 --- a/libraries/chain/webassembly/runtimes/eos-vm.cpp +++ b/libraries/chain/webassembly/runtimes/eos-vm.cpp @@ -275,6 +275,12 @@ REGISTER_HOST_FUNCTION(set_kv_parameters_packed, privileged_check); REGISTER_HOST_FUNCTION(is_privileged, privileged_check); REGISTER_HOST_FUNCTION(set_privileged, privileged_check); +// security group api +REGISTER_HOST_FUNCTION(propose_security_group_participants_add, privileged_check); +REGISTER_HOST_FUNCTION(propose_security_group_participants_remove, privileged_check); +REGISTER_HOST_FUNCTION(in_active_security_group); +REGISTER_HOST_FUNCTION(get_active_security_group); + // softfloat api REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_add); REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_sub); diff --git a/libraries/chain/webassembly/security_group.cpp b/libraries/chain/webassembly/security_group.cpp new file mode 100644 index 00000000000..d86dc9b0eae --- /dev/null +++ b/libraries/chain/webassembly/security_group.cpp @@ -0,0 +1,43 @@ + +#include +#include + +namespace eosio { +namespace chain { +namespace webassembly { + +int64_t interface::propose_security_group_participants_add(span packed_participants) { + datastream ds(packed_participants.data(), packed_participants.size()); + flat_set participants; + fc::raw::unpack(ds, participants); + return context.control.propose_security_group_participants_add(participants); +} + +int64_t interface::propose_security_group_participants_remove(span packed_participants) { + datastream ds(packed_participants.data(), packed_participants.size()); + flat_set participants; + fc::raw::unpack(ds, participants); + return context.control.propose_security_group_participants_remove(participants); +} + +bool interface::in_active_security_group(span packed_participants) const { + datastream ds(packed_participants.data(), packed_participants.size()); + flat_set participants; + fc::raw::unpack(ds, participants); + return context.control.in_active_security_group(participants); +} + +uint32_t interface::get_active_security_group(span packed_security_group) const { + datastream size_strm; + const auto& active_security_group = context.control.active_security_group(); + fc::raw::pack(size_strm, active_security_group); + if (size_strm.tellp() <= packed_security_group.size()) { + datastream ds(packed_security_group.data(), packed_security_group.size()); + fc::raw::pack(ds, active_security_group); + } + return size_strm.tellp(); +} + +} // namespace webassembly +} // namespace chain +} // namespace eosio diff --git a/unittests/security_group_tests.cpp b/unittests/security_group_tests.cpp index 7630bf8484d..3fa61992999 100644 --- a/unittests/security_group_tests.cpp +++ b/unittests/security_group_tests.cpp @@ -58,6 +58,8 @@ FC_REFLECT(eosio::chain::snapshot_global_property_object_v5, BOOST_AUTO_TEST_SUITE(security_group_tests) +using participants_t = boost::container::flat_set; + BOOST_AUTO_TEST_CASE(test_unpack_legacy_block_state) { eosio::testing::tester main; @@ -78,7 +80,7 @@ BOOST_AUTO_TEST_CASE(test_unpack_legacy_block_state) { // make sure we can unpack block_header_state { fc::datastream in_strm(out_strm.storage().data(), out_strm.storage().size()); - eosio::chain::versioned_unpack_stream unpack_strm(in_strm, eosio::chain::block_header_state::minimum_version_with_state_extension-1); + eosio::chain::versioned_unpack_stream unpack_strm(in_strm, eosio::chain::block_header_state::minimum_snapshot_version_with_state_extension-1); eosio::chain::block_header_state tmp; BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, tmp)); BOOST_CHECK_EQUAL(bs->id, tmp.id); @@ -92,7 +94,7 @@ BOOST_AUTO_TEST_CASE(test_unpack_legacy_block_state) { // make sure we can unpack block_state { fc::datastream in_strm(out_strm.storage().data(), out_strm.storage().size()); - eosio::chain::versioned_unpack_stream unpack_strm(in_strm, eosio::chain::block_header_state::minimum_version_with_state_extension-1); + eosio::chain::versioned_unpack_stream unpack_strm(in_strm, eosio::chain::block_header_state::minimum_snapshot_version_with_state_extension-1); eosio::chain::block_state tmp; BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, tmp)); BOOST_CHECK_EQUAL(bs->id, tmp.id); @@ -124,7 +126,7 @@ BOOST_AUTO_TEST_CASE(test_unpack_new_block_state) { // make sure we can unpack block_state { fc::datastream in_strm(out_strm.storage().data(), out_strm.storage().size()); - eosio::chain::versioned_unpack_stream unpack_strm(in_strm, eosio::chain::block_header_state::minimum_version_with_state_extension); + eosio::chain::versioned_unpack_stream unpack_strm(in_strm, eosio::chain::block_header_state::minimum_snapshot_version_with_state_extension); eosio::chain::block_state tmp; BOOST_CHECK_NO_THROW(fc::raw::unpack(unpack_strm, tmp)); BOOST_CHECK_EQUAL(bs->id, tmp.id); @@ -207,8 +209,6 @@ BOOST_AUTO_TEST_CASE(test_participants_change) { BOOST_REQUIRE_EQUAL(cur_security_group.participants.size(), 0); } - using participants_t = boost::container::flat_set; - participants_t new_participants({"alice"_n, "bob"_n}); chain.control->propose_security_group_participants_add(new_participants); @@ -233,4 +233,143 @@ BOOST_AUTO_TEST_CASE(test_participants_change) { } +void push_blocks( eosio::testing::tester& from, eosio::testing::tester& to ) { + while( to.control->fork_db_pending_head_block_num() + < from.control->fork_db_pending_head_block_num() ) + { + auto fb = from.control->fetch_block_by_number( to.control->fork_db_pending_head_block_num()+1 ); + to.push_block( fb ); + } +} + +static const char propose_security_group_participants_add_wast[] = R"=====( +(module + (func $action_data_size (import "env" "action_data_size") (result i32)) + (func $read_action_data (import "env" "read_action_data") (param i32 i32) (result i32)) + (func $propose_security_group_participants_add (import "env" "propose_security_group_participants_add") (param i32 i32)(result i64)) + (memory 1) + (func (export "apply") (param i64 i64 i64) + (local $bytes_remaining i32) + (set_local $bytes_remaining (call $action_data_size)) + (drop (call $read_action_data (i32.const 0) (get_local $bytes_remaining))) + (drop (call $propose_security_group_participants_add (i32.const 0) (get_local $bytes_remaining))) + ) +) +)====="; + +static const char propose_security_group_participants_remove_wast[] = R"=====( +(module + (func $action_data_size (import "env" "action_data_size") (result i32)) + (func $read_action_data (import "env" "read_action_data") (param i32 i32) (result i32)) + (func $propose_security_group_participants_remove (import "env" "propose_security_group_participants_remove") (param i32 i32)(result i64)) + (memory 1) + (func (export "apply") (param i64 i64 i64) + (local $bytes_remaining i32) + (set_local $bytes_remaining (call $action_data_size)) + (drop (call $read_action_data (i32.const 0) (get_local $bytes_remaining))) + (drop (call $propose_security_group_participants_remove (i32.const 0) (get_local $bytes_remaining))) + ) +) +)====="; + +static const char assert_in_security_group_wast[] = R"=====( +(module + (func $action_data_size (import "env" "action_data_size") (result i32)) + (func $read_action_data (import "env" "read_action_data") (param i32 i32) (result i32)) + (func $in_active_security_group (import "env" "in_active_security_group") (param i32 i32)(result i32)) + (func $eosio_assert (import "env" "eosio_assert") (param i32 i32)) + (memory 1) + (func (export "apply") (param i64 i64 i64) + (local $bytes_remaining i32) + (local $in_group i32) + (set_local $bytes_remaining (call $action_data_size)) + (drop (call $read_action_data (i32.const 0) (get_local $bytes_remaining))) + (set_local $in_group (call $in_active_security_group (i32.const 0) (get_local $bytes_remaining))) + (call $eosio_assert (get_local $in_group)(i32.const 512)) + ) + (data (i32.const 512) "in_active_security_group should return true") +) +)====="; + + +static const char assert_get_security_group_wast[] = R"=====( +(module + (func $action_data_size (import "env" "action_data_size") (result i32)) + (func $read_action_data (import "env" "read_action_data") (param i32 i32) (result i32)) + (func $get_active_security_group (import "env" "get_active_security_group") (param i32 i32)(result i32)) + (func $eosio_assert (import "env" "eosio_assert") (param i32 i32)) + (func $memcmp (import "env" "memcmp") (param i32 i32 i32) (result i32)) + (memory 1) + (func (export "apply") (param i64 i64 i64) + (local $bytes_remaining i32) + (local $required_bytes i32) + (set_local $bytes_remaining (call $action_data_size)) + (drop (call $read_action_data (i32.const 0) (get_local $bytes_remaining))) + (set_local $required_bytes (call $get_active_security_group (i32.const 0) (i32.const 0))) + (call $eosio_assert (i32.eq (i32.const 13) (get_local $required_bytes))(i32.const 512)) + (drop (call $get_active_security_group (i32.const 256) (get_local $required_bytes))) + (call $eosio_assert (i32.eq (call $memcmp (i32.const 0) (i32.const 256) (get_local $required_bytes))(i32.const 0)) (i32.const 768)) + ) + (data (i32.const 512) "get_active_security_group should return the right size") + (data (i32.const 768) "get_active_security_group output buffer must match the input") +) +)====="; + +std::vector participants_payload( participants_t names ) { + fc::datastream> ds; + fc::raw::pack(ds, names); + return ds.storage(); +} + + +BOOST_AUTO_TEST_CASE(test_security_group_intrinsic) { + + eosio::testing::tester chain1; + using namespace eosio::chain::literals; + + chain1.create_accounts( {"alice"_n,"bob"_n,"charlie"_n} ); + chain1.produce_blocks(3); + + chain1.create_accounts({ "addmember"_n, "rmmember"_n, "ingroup"_n, "getgroup"_n }); + chain1.produce_block(); + + chain1.set_code( "addmember"_n, propose_security_group_participants_add_wast ); + chain1.set_code( "rmmember"_n, propose_security_group_participants_remove_wast ); + chain1.set_code( "ingroup"_n, assert_in_security_group_wast ); + chain1.set_code( "getgroup"_n, assert_get_security_group_wast ); + chain1.produce_block(); + + chain1.push_action( "eosio"_n, "setpriv"_n, "eosio"_n, fc::mutable_variant_object()("account", "addmember"_n)("is_priv", 1)); + chain1.push_action( "eosio"_n, "setpriv"_n, "eosio"_n, fc::mutable_variant_object()("account", "rmmember"_n)("is_priv", 1)); + chain1.produce_blocks(24); + + chain1.set_producers( {"alice"_n,"bob"_n} ); + chain1.produce_blocks(3); // Starts new blocks which promotes the proposed schedule to pending + BOOST_REQUIRE_EQUAL( chain1.control->active_producers().version, 1u ); + + BOOST_TEST_REQUIRE(chain1.push_action( eosio::chain::action({}, "addmember"_n, {}, participants_payload({"alice"_n, "bob"_n})), "addmember"_n.to_uint64_t() ) == ""); + chain1.produce_blocks(11); + BOOST_CHECK_EQUAL(chain1.control->proposed_security_group_participants().size() , 2); + chain1.produce_blocks(12); + BOOST_CHECK_EQUAL(chain1.control->proposed_security_group_participants().size() , 0); + BOOST_CHECK(chain1.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + + BOOST_TEST_REQUIRE(chain1.push_action( eosio::chain::action({}, "rmmember"_n, {}, participants_payload({"alice"_n})), "rmmember"_n.to_uint64_t() ) == ""); + BOOST_CHECK_EQUAL(chain1.control->proposed_security_group_participants().size() , 1); + chain1.produce_blocks(11+12); + BOOST_CHECK(!chain1.control->in_active_security_group(participants_t({"alice"_n}))); + + BOOST_TEST_REQUIRE(chain1.push_action( eosio::chain::action({}, "ingroup"_n, {}, participants_payload({"bob"_n})), "ingroup"_n.to_uint64_t() ) == ""); + + eosio::chain::security_group_info_t grp{ + .version = 2, + .participants = {"bob"_n} + }; + + fc::datastream> strm; + fc::raw::pack(strm, grp); + + BOOST_TEST_REQUIRE(chain1.push_action(eosio::chain::action({}, "getgroup"_n, {}, strm.storage()), "getgroup"_n.to_uint64_t()) == ""); +} + BOOST_AUTO_TEST_SUITE_END() From ddb63101359e11fcac6c170c257bc653b8967b2b Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 8 Mar 2021 15:20:36 -0600 Subject: [PATCH 012/157] add snapshot recovery test --- .../testing/include/eosio/testing/tester.hpp | 4 ++++ unittests/security_group_tests.cpp | 23 ++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index dfec8b90f10..724a831cc9b 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -480,6 +480,10 @@ namespace eosio { namespace testing { tester(const std::function& control_setup, setup_policy policy = setup_policy::full, db_read_mode read_mode = db_read_mode::SPECULATIVE); + tester(const std::function& lambda) { + lambda(*this); + } + using base_tester::produce_block; signed_block_ptr produce_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override { diff --git a/unittests/security_group_tests.cpp b/unittests/security_group_tests.cpp index 3fa61992999..5ced1b91101 100644 --- a/unittests/security_group_tests.cpp +++ b/unittests/security_group_tests.cpp @@ -321,7 +321,6 @@ std::vector participants_payload( participants_t names ) { return ds.storage(); } - BOOST_AUTO_TEST_CASE(test_security_group_intrinsic) { eosio::testing::tester chain1; @@ -370,6 +369,28 @@ BOOST_AUTO_TEST_CASE(test_security_group_intrinsic) { fc::raw::pack(strm, grp); BOOST_TEST_REQUIRE(chain1.push_action(eosio::chain::action({}, "getgroup"_n, {}, strm.storage()), "getgroup"_n.to_uint64_t()) == ""); + chain1.produce_blocks(11); + chain1.control->abort_block(); + + /// Test snapshot recovery + + std::stringstream snapshot_strm; + auto writer = std::make_shared(snapshot_strm); + chain1.control->write_snapshot(writer); + writer->finalize(); + + auto cfg = chain1.get_config(); + fc::temp_directory tmp_dir; + cfg.blog.log_dir = tmp_dir.path() / "blocks"; + cfg.state_dir = tmp_dir.path() / "state"; + + auto reader = std::make_shared(snapshot_strm); + eosio::testing::tester chain2([&cfg, &reader](eosio::testing::tester& self) { self.init(cfg, reader); }); + { + const auto& active_security_group = chain2.control->active_security_group(); + BOOST_CHECK_EQUAL(2, active_security_group.version); + BOOST_TEST(active_security_group.participants == participants_t{"bob"_n}); + } } BOOST_AUTO_TEST_SUITE_END() From 68a3f79d897ddbc23dc7e193fc077d1d6d03a2cf Mon Sep 17 00:00:00 2001 From: Victor Camacho Date: Mon, 8 Mar 2021 17:07:38 -0500 Subject: [PATCH 013/157] add contract/action callback field to global properties --- .../eosio/chain/global_property_object.hpp | 27 ++++++++++++++----- libraries/chain/include/eosio/chain/types.hpp | 11 ++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index cddfa11c5b4..d6984080f01 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -75,6 +75,7 @@ namespace eosio { namespace chain { wasm_config wasm_configuration; block_num_type proposed_security_group_block_num = 0; flat_set proposed_security_group_participants; + vector transaction_hooks; void initalize_from(const legacy::snapshot_global_property_object_v2& legacy, const chain_id_type& chain_id_val, const kv_database_config& kv_config_val, const wasm_config& wasm_config_val) { @@ -130,13 +131,17 @@ namespace eosio { namespace chain { flat_set proposed_security_group_participants; }; + struct extension_v1 : extension_v0 { + vector transaction_hooks; + }; + // for future extensions, please use the following pattern: // - // struct extension_v1 : extension_v0 { new_field_t new_field; }; - // using extension_t = std::variant; + // struct extension_v2 : extension_v1 { new_field_t new_field; }; + // using extension_t = std::variant; // - using extension_t = std::variant; + using extension_t = std::variant; extension_t extension; }; @@ -147,7 +152,8 @@ namespace eosio { namespace chain { using snapshot_type = snapshot_global_property_object; static_assert(std::is_same_v>, + std::variant>, "Please update to_snapshot_row()/from_snapshot_row() accordingly when " "snapshot_global_property_object::extension_t is changed"); @@ -159,8 +165,10 @@ namespace eosio { namespace chain { value.chain_id, value.kv_configuration, value.wasm_configuration, - snapshot_global_property_object::extension_v0{value.proposed_security_group_block_num, - value.proposed_security_group_participants}}; + snapshot_global_property_object::extension_v1{{value.proposed_security_group_block_num, + value.proposed_security_group_participants}, + value.transaction_hooks + }}; } static void from_snapshot_row(snapshot_global_property_object&& row, global_property_object& value, @@ -176,6 +184,9 @@ namespace eosio { namespace chain { [&value](auto& ext) { value.proposed_security_group_block_num = ext.proposed_security_group_block_num; value.proposed_security_group_participants = std::move(ext.proposed_security_group_participants); + if constexpr( std::is_base_of::type>::value ) { + value.transaction_hooks = std::move(ext.transaction_hooks); + } }, row.extension); } @@ -231,6 +242,10 @@ FC_REFLECT(eosio::chain::snapshot_global_property_object::extension_v0, (proposed_security_group_block_num)(proposed_security_group_participants) ) +FC_REFLECT_DERIVED(eosio::chain::snapshot_global_property_object::extension_v1, + (eosio::chain::snapshot_global_property_object::extension_v0), (transaction_hooks) + ) + FC_REFLECT(eosio::chain::snapshot_global_property_object, (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id)(kv_configuration)(wasm_configuration)(extension) ) diff --git a/libraries/chain/include/eosio/chain/types.hpp b/libraries/chain/include/eosio/chain/types.hpp index d8184035a29..39498bfc194 100644 --- a/libraries/chain/include/eosio/chain/types.hpp +++ b/libraries/chain/include/eosio/chain/types.hpp @@ -330,6 +330,16 @@ namespace eosio { namespace chain { } }; + /** + * Defines a type used for defining transaction hooks + * Transaction hooks are a way for system contracts to inspect / make modifications to the transaction + * at a certain point, for example prior to its execution. + */ + struct transaction_hook{ + uint32_t type; + account_name contract; + action_name action; + }; /** * Extentions are prefixed with type and are a buffer that can be @@ -479,4 +489,5 @@ namespace chainbase { } } +FC_REFLECT(eosio::chain::transaction_hook, (type)(contract)(action)) FC_REFLECT_EMPTY( eosio::chain::void_t ) From 51f6d91d392ece7807458fb45ffd16f41d72ed4c Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Mon, 8 Mar 2021 16:29:55 -0600 Subject: [PATCH 014/157] wip --- plugins/net_plugin/net_plugin.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 3d6e7736808..e26ae96e4fe 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -258,6 +258,7 @@ namespace eosio { mutable std::shared_mutex connections_mtx; std::set< connection_ptr > connections; // todo: switch to a thread safe container to avoid big mutex over complete collection + std::unordered_multimap particiant_connections; std::mutex connector_check_timer_mtx; unique_ptr connector_check_timer; @@ -4023,6 +4024,16 @@ namespace eosio { if( my->find_connection( host ) ) return "already connected"; + /** @todo when making a shared tls connection, need to use the ctor that takes + * requires the participant name. + * connection_ptr c = std::make_shared(host, nam); + * + * @todo Need to add the connection to the + * unordered_multimap. + * + * participant_connections.emplace(name, c); + * + */ connection_ptr c = std::make_shared( host ); fc_dlog( logger, "calling active connector: ${h}", ("h", host) ); if( c->resolve_and_connect() ) { From ded04b8edc900b5a96b3b16090000cc81f31b866 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 8 Mar 2021 15:32:03 -0600 Subject: [PATCH 015/157] add default constructor for snapshot_global_property_object::extension_v0 --- libraries/chain/include/eosio/chain/global_property_object.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index 8110f9e736f..62fca730f53 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -125,6 +125,7 @@ namespace eosio { namespace chain { static constexpr uint32_t minimum_version_with_extension = 6; struct extension_v0 { + extension_v0() {}; block_num_type proposed_security_group_block_num = 0; flat_set proposed_security_group_participants; }; From 23a50248a5f6af1fdd70c5144a652ef21a7154f7 Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Tue, 9 Mar 2021 08:01:43 -0600 Subject: [PATCH 016/157] EPE-672: Review feedback --- .../eosio/net_plugin/security_group.hpp | 70 +++----------- plugins/net_plugin/security_group.cpp | 61 +----------- .../net_plugin/test/security_group_tests.cpp | 92 +++++++++---------- 3 files changed, 60 insertions(+), 163 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp b/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp index e01ab61595f..2e8a2fed302 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp @@ -7,77 +7,29 @@ #include namespace eosio { - /** \brief Describes a security group participant */ - class security_group_participant { - public: - /** \brief Simple initialization - * - * @param account_name The recognized name of the participant - * @param participating Indicates the participant's participation (default = true) - * true - participating - * false - sync'ing only - */ - explicit security_group_participant(chain::account_name account_name, bool participating = true) : - participant_(account_name), participating_(participating) {} - security_group_participant(const security_group_participant& that) : - participant_(that.participant_), participating_(that.is_participating()) {} - ~security_group_participant() = default; - - auto name() const { return participant_; } - - [[maybe_unused]] - bool is_participating() const { return participating_.load(std::memory_order_relaxed); } - bool is_syncing() const { return !participating_.load(std::memory_order_relaxed); } - void now_syncing() { participating_.store(false, std::memory_order_relaxed); } - void now_participating() { participating_.store(true, std::memory_order_relaxed); } - - security_group_participant& operator=(const security_group_participant& that) { - participant_ = that.participant_; - participating_.store(that.is_participating(), std::memory_order_relaxed); - return *this; - } - - bool operator==(const security_group_participant& that) const { return that.participant_ == participant_; } - bool operator!=(const security_group_participant& that) const { return !operator==(that); } - bool operator<(const security_group_participant& that) const { return participant_ < that.participant_; } - bool operator>(const security_group_participant& that) const { return participant_ > that.participant_; } - - bool operator==(chain::account_name account_name) const { return account_name == participant_; } - - private: - chain::account_name participant_; //!< Identifier for this participant - std::atomic participating_ {true}; //!< Indicates if participant producing blocks (true) or - //!< only used for sync'ing - }; - - /** \brief Describes the security group cache */ + /** \brief Manages the security group cache */ class security_group_manager { public: security_group_manager() = default; - security_group_manager(const security_group_manager&) = delete; + security_group_manager(const security_group_manager&) = default; security_group_manager(security_group_manager&&) = delete; ~security_group_manager() = default; - using account_name_list_t = boost::container::flat_set; - using callback_t = std::function; - /** Update the cache using new information from a block + using participant_list_t = boost::container::flat_set; + /** @brief Update the security group participants * * @param version The version number for this update * @param participant_list The list of accounts for the security group - * @param on_participating Callback when an account is added to the cache - * @param on_syncing Callback when an account is "removed" from the cache + * @return True if an update was performed. */ - void update_cache(const uint32_t version, const account_name_list_t& participant_list, callback_t on_participating, - callback_t on_syncing); - - using cache_t = boost::container::flat_set; - const cache_t& security_group() const { return cache_; } - - security_group_manager& operator==(const security_group_manager&) = delete; - security_group_manager& operator==(security_group_manager&&) = delete; + bool update_cache(const uint32_t version, const participant_list_t& participant_list); + /** @brief Determine if a participant is in the security group */ + bool is_in_security_group(chain::account_name participant) { return cache_.find(participant) != cache_.end(); } + security_group_manager& operator=(const security_group_manager&) = default; + security_group_manager& operator=(security_group_manager&&) = delete; private: uint32_t version_ {std::numeric_limits::max()}; ///! The security group version - cache_t cache_; ///! Cache of accounts in the security group + participant_list_t cache_; ///! Cache of accounts in the security group }; } diff --git a/plugins/net_plugin/security_group.cpp b/plugins/net_plugin/security_group.cpp index af31e19cb6f..57d521ebf00 100644 --- a/plugins/net_plugin/security_group.cpp +++ b/plugins/net_plugin/security_group.cpp @@ -1,64 +1,11 @@ #include namespace eosio { - // Cache update looks at 3 scenarios: - // 1) The cache is empty and is being populated - // 2) The update is empty so all accounts are being removed from the cache - // 3) The cache is populated and accouts are being added and/or removed - void security_group_manager::update_cache(const uint32_t version, const account_name_list_t& participant_list, - callback_t on_participating, callback_t on_syncing) { - // update on version change + bool security_group_manager::update_cache(const uint32_t version, const participant_list_t& participant_list) { if(version == version_) - return; + return false; version_ = version; - - // Scenario 1: The cache is empty and being populated - // - if(cache_.empty()) { - if(participant_list.empty()) { - return; - } - for(auto participant : participant_list) { - cache_.emplace(participant); - on_participating(participant); - } - return; - } - - // Scenaio 2: The update is empty so all accounts are being removed from the cache - // - if(participant_list.empty()) { - for(auto& participant : cache_) { - if(participant.is_participating()) { - participant.now_syncing(); - on_syncing(participant.name()); - } - } - return; - } - - // Scenario 3: Accounts are added and/or removed from cache - // - // 1) Update status of existing accounts - // 2) Add new accounts - // - for(auto& participant : cache_) { - if(participant_list.find(participant.name()) != participant_list.end()) { - if(participant.is_syncing()) { - participant.now_participating(); - on_participating(participant.name()); - } - } - else { - participant.now_syncing(); - on_syncing(participant.name()); - } - } - - for(const auto& participant : participant_list) { - if(cache_.emplace(participant).second) { - on_participating(participant); - } - } + cache_ = participant_list; + return true; } } diff --git a/plugins/net_plugin/test/security_group_tests.cpp b/plugins/net_plugin/test/security_group_tests.cpp index 77fe212b84e..9595b8b42cc 100644 --- a/plugins/net_plugin/test/security_group_tests.cpp +++ b/plugins/net_plugin/test/security_group_tests.cpp @@ -22,87 +22,85 @@ namespace { } return participant_list; } - auto dummy_add = [](eosio::chain::account_name) {}; - auto dummy_remove = [](eosio::chain::account_name) {}; } BOOST_AUTO_TEST_SUITE(security_group_tests) using namespace eosio::testing; + BOOST_AUTO_TEST_CASE(test_initial_population) { auto populate = create_list({ 1, 2, 3, 4, 5, 6}); - size_t add_count = 0; - auto on_add = [&](eosio::chain::account_name name) { - BOOST_REQUIRE(populate.find(name) != populate.end()); - ++add_count; - }; - - auto on_rem = [&](eosio::chain::account_name name) { - BOOST_FAIL("Name removed from security group"); - }; - eosio::security_group_manager manager; - manager.update_cache(1, populate, on_add, on_rem); + BOOST_REQUIRE(manager.update_cache(1, populate)); + BOOST_REQUIRE(!manager.update_cache(1, populate)); - BOOST_REQUIRE_EQUAL(add_count, populate.size()); + for(auto participant : populate) { + BOOST_REQUIRE(manager.is_in_security_group(participant)); + } } - BOOST_AUTO_TEST_CASE(test_remove_all) { auto populate = create_list({1, 2, 3, 4, 5, 6}); eosio::security_group_manager manager; - manager.update_cache(1, populate, dummy_add, dummy_remove); - - auto check_add = [](eosio::chain::account_name) { - BOOST_FAIL("Tried to add name from empty list"); - }; - - size_t remove_count = 0; - auto check_remove = [&remove_count](eosio::chain::account_name) { ++remove_count; }; + manager.update_cache(1, populate); participant_list_t clear; - manager.update_cache(2, clear, check_add, check_remove); - BOOST_REQUIRE_EQUAL(populate.size(), remove_count); + BOOST_REQUIRE(manager.update_cache(2, clear)); + + for(auto participant : populate) { + BOOST_REQUIRE(!manager.is_in_security_group(participant)); + } } BOOST_AUTO_TEST_CASE(test_add_only) { auto populate = create_list({1, 2, 3, 4, 5, 6}); eosio::security_group_manager manager; - manager.update_cache(1, populate, dummy_add, dummy_remove); + manager.update_cache(1, populate); - size_t add_count = 0; - auto check_add = [&add_count](eosio::chain::account_name) { ++add_count; }; - auto check_remove = [](eosio::chain::account_name) { BOOST_FAIL("Nothing should be removed");}; + auto add = create_list({7, 8, 9}); + for(auto participant : add) { + BOOST_REQUIRE(!manager.is_in_security_group(participant)); + } - auto update = create_list({1, 2, 3, 4, 5, 6, 7, 8, 9}); - manager.update_cache(2, update, check_add, check_remove); - BOOST_REQUIRE_EQUAL((update.size() - populate.size()), add_count); + populate.insert(add.begin(), add.end()); + manager.update_cache(2, populate); + for(auto participant : populate) { + BOOST_REQUIRE(manager.is_in_security_group(participant)); + } } BOOST_AUTO_TEST_CASE(test_remove_only) { auto populate = create_list({1, 2, 3, 4, 5, 6}); eosio::security_group_manager manager; - manager.update_cache(1, populate, dummy_add, dummy_remove); - - auto check_add = [](eosio::chain::account_name) { BOOST_FAIL("Nothing should be added");}; - size_t remove_count = 0; - auto check_remove = [&remove_count](eosio::chain::account_name) { ++remove_count; }; + manager.update_cache(1, populate); auto update = create_list({2, 4, 6}); - manager.update_cache(2, update, check_add, check_remove); - BOOST_REQUIRE_EQUAL(populate.size() - update.size(), remove_count); + manager.update_cache(2, update); + + auto removed = create_list({1, 3, 5}); + for(auto participant : removed) { + BOOST_REQUIRE(!manager.is_in_security_group(participant)); + } + + for (auto participant : update) { + BOOST_REQUIRE(manager.is_in_security_group(participant)); + } } BOOST_AUTO_TEST_CASE(test_update) { auto populate = create_list({1, 2, 3, 4, 5, 6}); eosio::security_group_manager manager; - manager.update_cache(1, populate, dummy_add, dummy_remove); - - size_t add_count = 0; - auto check_add = [&add_count](eosio::chain::account_name) { ++add_count;}; - size_t remove_count = 0; - auto check_remove = [&remove_count](eosio::chain::account_name) { ++remove_count; }; + manager.update_cache(1, populate); auto update = create_list({2, 4, 6, 7, 8, 9}); - manager.update_cache(2, update, check_add, check_remove); - BOOST_REQUIRE_EQUAL(add_count + remove_count, update.size()); + manager.update_cache(2, update); + + auto removed = create_list({1, 3, 5}); + for(auto participant : removed) { + BOOST_REQUIRE(!manager.is_in_security_group(participant)); + } + + auto added = create_list({7, 8, 9}); + for (auto participant : added) { + BOOST_REQUIRE(manager.is_in_security_group(participant)); + } } BOOST_AUTO_TEST_SUITE_END() From ca764add1326557dbc0daef8878444334b8f0a13 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 9 Mar 2021 08:29:40 -0600 Subject: [PATCH 017/157] another workaround for libstdc++ bug --- .../chain/include/eosio/chain/global_property_object.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index 62fca730f53..3be4819c558 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -125,7 +125,12 @@ namespace eosio { namespace chain { static constexpr uint32_t minimum_version_with_extension = 6; struct extension_v0 { - extension_v0() {}; + // libstdc++ requires the following two constructors to work. + extension_v0(){}; + extension_v0(block_num_type num, const flat_set& participants) + : proposed_security_group_block_num(num) + , proposed_security_group_participants(participants) {} + block_num_type proposed_security_group_block_num = 0; flat_set proposed_security_group_participants; }; From 55680712aff4ef0f8e3ccb3ea9fe3ca3aeaa1cd1 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Tue, 9 Mar 2021 09:46:20 -0500 Subject: [PATCH 018/157] certificate generation scripts for tests added --- tests/CMakeLists.txt | 1 + tests/generate-certificates.sh | 89 ++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100755 tests/generate-certificates.sh diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0d1cce56fe7..0258d4ee8ef 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -60,6 +60,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_filter.wasm ${CMAKE_CURRENT_BINA configure_file(${CMAKE_CURRENT_SOURCE_DIR}/trace_plugin_test.py ${CMAKE_CURRENT_BINARY_DIR}/trace_plugin_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_contrl_c_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_contrl_c_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/blockvault_tests.py ${CMAKE_CURRENT_BINARY_DIR}/blockvault_tests.py COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/generate-certificates.sh ${CMAKE_CURRENT_BINARY_DIR}/generate-certificates.sh COPYONLY) #To run plugin_test with all log from blockchain displayed, put --verbose after --, i.e. plugin_test -- --verbose add_test(NAME plugin_test COMMAND plugin_test --report_level=detailed --color_output) diff --git a/tests/generate-certificates.sh b/tests/generate-certificates.sh new file mode 100755 index 00000000000..ac593e2dcdc --- /dev/null +++ b/tests/generate-certificates.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +function parse-args() { +while [[ $# > 0 ]] +do + case "$1" in + --days|-d) + DAYS=${2} + shift + ;; + --CA-org|-o) + CA_ORG=${2} + ;; + --CA-CN|-n) + CA_CN=${2} + shift + ;; + --org-mask|-m) + ORG_MASK=${2} + shift + ;; + --cn-mask|-cm) + CN_MASK=${2} + shift + ;; + --group-size|-s) + GROUP_SIZE=${2} + shift + ;; + esac + shift +done +} + +if [[ $1 == "--help" ]] +then + echo "Usage:" + echo "--days: Number of days for certificate to expire" + echo "--CA-org: Certificate Authority organization name" + echo "--CA-CN: Certificate Authority common name" + echo "--org-mask: Paritipant certificates name mask in format of name{number}" + echo "--cn-mask: Paritipant certificates common name mask in format of name{number}" + echo "--group-size: Number of participants signed by generated CA" +fi + +#default arguments: +DAYS=1 +CA_ORG="Block.one" +CA_CN="test-domain" +ORG_MASK="node{NUMBER}" +CN_MASK="test-domain{NUMBER}" +GROUP_SIZE=4 + +#overrides default is set +parse-args "${@}" + +echo "*************************************************" +echo " generating dh param " +echo "*************************************************" +#using low values like 128 here and below as this is for unit tests and our goal to save running time. For real applications 2048 recommended +openssl dhparam -out dh.pem 128 + +echo "*************************************************" +echo " generating CA_cert.pem " +echo "*************************************************" + +openssl req -newkey rsa:512 -nodes -keyout CA_key.pem -x509 -days ${DAYS} -out CA_cert.pem -subj "/C=US/ST=VA/L=Blocksburg/O=${CA_ORG}/CN=${CA_CN}" + +echo "*************************************************" +openssl x509 -in CA_cert.pem -text -noout + +echo "*************************************************" +echo " generating nodes certificates " +echo "*************************************************" + +#client certificate requests + private keys +for n in $(seq 1 $GROUP_SIZE) +do + ORG_NAME=$(sed "s/{NUMBER}/$n/" <<< "$ORG_MASK") + CN_NAME=$(sed "s/{NUMBER}/$n/" <<< "$CN_MASK") + echo "*************************************************" + echo "generating certificate for $ORG_NAME / $CN_NAME " + echo "*************************************************" + openssl req -newkey rsa:512 -nodes -keyout "${ORG_NAME}_key.pem" -out "${ORG_NAME}.csr" -subj "/C=US/ST=VA/L=Blockburg/O=${ORG_NAME}/CN=${CN_NAME}" + openssl x509 -req -in "${ORG_NAME}.csr" -CA CA_cert.pem -CAkey CA_key.pem -CAcreateserial -out "${ORG_NAME}.crt" -days ${DAYS} -sha256 + echo "*************************************************" + openssl x509 -in "${ORG_NAME}.crt" -text -noout + echo "" +done \ No newline at end of file From d52a711c43821d6ea0ff3c03902dde0c3cc5f2fd Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 9 Mar 2021 09:15:56 -0600 Subject: [PATCH 019/157] fix protocol feature digest --- libraries/chain/protocol_feature_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/protocol_feature_manager.cpp b/libraries/chain/protocol_feature_manager.cpp index 9ef467cd415..312366909a5 100644 --- a/libraries/chain/protocol_feature_manager.cpp +++ b/libraries/chain/protocol_feature_manager.cpp @@ -232,7 +232,7 @@ Allows privileged contracts to get and set subsets of blockchain parameters. */ ( builtin_protocol_feature_t::security_group, builtin_protocol_feature_spec{ "SECURITY_GROUP", - fc::variant("410976abd4d5caed3308134f03f1eb5983e91eb26ac20f3a8fd34fd1b49faaea").as(), + fc::variant("72ec6337e369cbb33ef7716d3267db9d5678fe54555c25ca4c9f5b9dfb7739f3").as(), // SHA256 hash of the raw message below within the comment delimiters (do not modify message below). /* Builtin protocol feature: SECURITY_GROUP From 92348a5094ecab39935eca4e8902a37a90af13f8 Mon Sep 17 00:00:00 2001 From: Victor Camacho Date: Tue, 9 Mar 2021 11:39:59 -0500 Subject: [PATCH 020/157] add some constructors for extension_v1 required by libstdc++ --- .../include/eosio/chain/global_property_object.hpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index b1aa7204985..268f8657d86 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -137,6 +137,12 @@ namespace eosio { namespace chain { }; struct extension_v1 : extension_v0 { + // libstdc++ requires the following two constructors to work. + extension_v1(){}; + extension_v1(block_num_type num, const flat_set& participants, const vector& trx_hooks) + : extension_v0(num, participants), + transaction_hooks(trx_hooks) {} + vector transaction_hooks; }; @@ -170,8 +176,8 @@ namespace eosio { namespace chain { value.chain_id, value.kv_configuration, value.wasm_configuration, - snapshot_global_property_object::extension_v1{{value.proposed_security_group_block_num, - value.proposed_security_group_participants}, + snapshot_global_property_object::extension_v1{value.proposed_security_group_block_num, + value.proposed_security_group_participants, value.transaction_hooks }}; } From 299f2bd4da79e114c24742635a3b83fdd84f33c4 Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Tue, 9 Mar 2021 10:48:24 -0600 Subject: [PATCH 021/157] wip --- .../eosio/net_plugin/security_group.hpp | 2 +- plugins/net_plugin/net_plugin.cpp | 83 +++++++------------ 2 files changed, 33 insertions(+), 52 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp b/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp index 2e8a2fed302..fe52afdcd9c 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp @@ -25,7 +25,7 @@ namespace eosio { */ bool update_cache(const uint32_t version, const participant_list_t& participant_list); /** @brief Determine if a participant is in the security group */ - bool is_in_security_group(chain::account_name participant) { return cache_.find(participant) != cache_.end(); } + bool is_in_security_group(chain::account_name participant) const { return cache_.find(participant) != cache_.end(); } security_group_manager& operator=(const security_group_manager&) = default; security_group_manager& operator=(security_group_manager&&) = delete; private: diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index e26ae96e4fe..d084ce2e6d0 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -820,28 +820,25 @@ namespace eosio { // security group management // private: - std::optional security_group_info_; + std::optional participant_name_; + std::atomic participating_{false}; public: /** @brief Set/change the participant name for the connection * * @param name The name of the participant */ - void participant_name(chain::account_name name) { - security_group_info_ = security_group_participant(name); - } + void participant_name(chain::account_name name) { participant_name_ = name; } /** @brief Returns the optional name of the participant */ - std::optional participant_name(); - bool is_participating() const { - return security_group_info_ && security_group_info_->is_participating(); - } - bool is_syncing() const { - return security_group_info_ && security_group_info_->is_syncing(); - } + auto participant_name() { return participant_name_; } + /** returns true if connection is in the security group */ + bool is_participating() const { return participating_.load(std::memory_order_relaxed); } + /** returns false if the connection is not in the security group */ + bool is_syncing() const { return !participating_.load(std::memory_order_relaxed); } /** @brief Flag the tls connection as participating in block production */ - void now_participating(); + void now_participating() { participating_.store(true, std::memory_order_relaxed); } /** @brief Flag the connection as only publishing sync information */ - void now_syncing(); + void now_syncing() { participating_.store(false, std::memory_order_relaxed); } }; const string connection::unknown = ""; @@ -933,7 +930,8 @@ namespace eosio { connection::connection( string endpoint, chain::account_name participant, bool participating ) : connection(endpoint) { - security_group_info_ = security_group_participant(participant, participating); + participant_name_ = participant; + participating_.store(participating, std::memory_order_relaxed); } connection::connection() @@ -1347,22 +1345,6 @@ namespace eosio { return true; } - std::optional connection::participant_name() { - if(security_group_info_) - return security_group_info_.value().name(); - return std::optional(); - } - - void connection::now_participating() { - if(security_group_info_) - security_group_info_.value().now_participating(); - } - - void connection::now_syncing() { - if(security_group_info_) - security_group_info_.value().now_syncing(); - } - //------------------------------------------------------------------------ using send_buffer_type = std::shared_ptr>; @@ -3505,31 +3487,30 @@ namespace eosio { // called from any thread void net_plugin_impl::update_security_group(const block_state_ptr& bs) { + // update cache + // + auto& update = bs->get_security_group_info(); + if(!security_group.update_cache(update.version, update.participants)) { + return; + } + // update connections // - auto updater = [](chain::account_name name, auto handler) { - for_each_connection([name, handler](auto& cp) { - auto participant_name = cp->participant_name(); - if(participant_name && participant_name.value() == name) { - handler(*cp); - } + auto do_update = [&](auto& connection) { + const auto& participant = connection->participant_name(); + if(!participant) { return true; - }); - }; - // add a new participant to the security group - // - auto on_add = [updater](chain::account_name name) { - updater(name, [](connection& con) { con.now_participating(); }); - }; - // remove a participant from the security group - // - auto on_remove = [updater](chain::account_name name) { - updater(name, [](connection& con) { con.now_syncing(); }); + } + if(security_group.is_in_security_group(participant.value())) { + connection->now_participating(); + } + else { + connection->now_syncing(); + } + return true; }; - // update cache - // - auto& update = bs->get_security_group_info(); - security_group.update_cache(update.version, update.participants, on_add, on_remove); + + for_each_connection(do_update); } // called from application thread From b367d71416217e8b8eff539e722a2cb84817752e Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Tue, 9 Mar 2021 12:26:18 -0600 Subject: [PATCH 022/157] EPE-672: Review feedback --- plugins/net_plugin/CMakeLists.txt | 2 +- ...curity_group.hpp => security_group_manager.hpp} | 14 +++----------- ...curity_group.cpp => security_group_manager.cpp} | 2 +- ..._tests.cpp => security_group_manager_tests.cpp} | 2 +- 4 files changed, 6 insertions(+), 14 deletions(-) rename plugins/net_plugin/include/eosio/net_plugin/{security_group.hpp => security_group_manager.hpp} (62%) rename plugins/net_plugin/{security_group.cpp => security_group_manager.cpp} (83%) rename plugins/net_plugin/test/{security_group_tests.cpp => security_group_manager_tests.cpp} (98%) diff --git a/plugins/net_plugin/CMakeLists.txt b/plugins/net_plugin/CMakeLists.txt index 36b2b7bcdb0..81130708d3b 100644 --- a/plugins/net_plugin/CMakeLists.txt +++ b/plugins/net_plugin/CMakeLists.txt @@ -1,7 +1,7 @@ file(GLOB HEADERS "include/eosio/net_plugin/*.hpp" ) add_library( net_plugin net_plugin.cpp - security_group.cpp + security_group_manager.cpp ${HEADERS} ) target_link_libraries( net_plugin chain_plugin producer_plugin appbase fc ) diff --git a/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp b/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp similarity index 62% rename from plugins/net_plugin/include/eosio/net_plugin/security_group.hpp rename to plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp index 2e8a2fed302..d9ebb0dd3fa 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/security_group.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp @@ -10,26 +10,18 @@ namespace eosio { /** \brief Manages the security group cache */ class security_group_manager { public: - security_group_manager() = default; - security_group_manager(const security_group_manager&) = default; - security_group_manager(security_group_manager&&) = delete; - - ~security_group_manager() = default; - using participant_list_t = boost::container::flat_set; /** @brief Update the security group participants * * @param version The version number for this update * @param participant_list The list of accounts for the security group - * @return True if an update was performed. + * @return True if an update was performed. */ bool update_cache(const uint32_t version, const participant_list_t& participant_list); /** @brief Determine if a participant is in the security group */ bool is_in_security_group(chain::account_name participant) { return cache_.find(participant) != cache_.end(); } - security_group_manager& operator=(const security_group_manager&) = default; - security_group_manager& operator=(security_group_manager&&) = delete; private: - uint32_t version_ {std::numeric_limits::max()}; ///! The security group version - participant_list_t cache_; ///! Cache of accounts in the security group + uint32_t version_ {std::numeric_limits::max()}; ///! The security group version + participant_list_t cache_; ///! Cache of accounts in the security group }; } diff --git a/plugins/net_plugin/security_group.cpp b/plugins/net_plugin/security_group_manager.cpp similarity index 83% rename from plugins/net_plugin/security_group.cpp rename to plugins/net_plugin/security_group_manager.cpp index 57d521ebf00..729deb26655 100644 --- a/plugins/net_plugin/security_group.cpp +++ b/plugins/net_plugin/security_group_manager.cpp @@ -1,4 +1,4 @@ -#include +#include namespace eosio { bool security_group_manager::update_cache(const uint32_t version, const participant_list_t& participant_list) { diff --git a/plugins/net_plugin/test/security_group_tests.cpp b/plugins/net_plugin/test/security_group_manager_tests.cpp similarity index 98% rename from plugins/net_plugin/test/security_group_tests.cpp rename to plugins/net_plugin/test/security_group_manager_tests.cpp index 9595b8b42cc..2a9d813d5e4 100644 --- a/plugins/net_plugin/test/security_group_tests.cpp +++ b/plugins/net_plugin/test/security_group_manager_tests.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include From 32c17d3128080cd6875945a45eb6ae3901d5dfe2 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 9 Mar 2021 12:27:40 -0600 Subject: [PATCH 023/157] change recover from state commit --- pipeline.jsonc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline.jsonc b/pipeline.jsonc index 35a96f26ea2..d68cc54df29 100644 --- a/pipeline.jsonc +++ b/pipeline.jsonc @@ -52,7 +52,7 @@ "test": [ { - "commit": "16074742f8cfd481b029073e6f01bb920a1bad38" + "commit": "d52a711c43821d6ea0ff3c03902dde0c3cc5f2fd" } ] } From b3a96279395425890b3b7c9e8564383d7a364305 Mon Sep 17 00:00:00 2001 From: Victor Camacho Date: Tue, 9 Mar 2021 14:24:13 -0500 Subject: [PATCH 024/157] fix a reflection problem with global_property_object --- libraries/chain/include/eosio/chain/global_property_object.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index 268f8657d86..1a64b84d92f 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -236,6 +236,7 @@ CHAINBASE_SET_INDEX_TYPE(eosio::chain::dynamic_global_property_object, FC_REFLECT(eosio::chain::global_property_object, (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id)(kv_configuration)(wasm_configuration) (proposed_security_group_block_num)(proposed_security_group_participants) + (transaction_hooks) ) FC_REFLECT(eosio::chain::legacy::snapshot_global_property_object_v2, From d089a31df0ba2e9428db5fc725a7bc74942a22d0 Mon Sep 17 00:00:00 2001 From: Victor Camacho Date: Tue, 9 Mar 2021 15:40:22 -0500 Subject: [PATCH 025/157] update commit for resume from state test --- pipeline.jsonc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline.jsonc b/pipeline.jsonc index 35a96f26ea2..906131be910 100644 --- a/pipeline.jsonc +++ b/pipeline.jsonc @@ -52,7 +52,7 @@ "test": [ { - "commit": "16074742f8cfd481b029073e6f01bb920a1bad38" + "commit": "b3a96279395425890b3b7c9e8564383d7a364305" } ] } From 66b8e467d57fe17b0e92bab62ece69a85834d411 Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Tue, 9 Mar 2021 16:45:38 -0600 Subject: [PATCH 026/157] wip --- .../eosio/net_plugin/security_group_manager.hpp | 5 ++--- plugins/net_plugin/net_plugin.cpp | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp b/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp index bdf87fc0931..0bae90df7af 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp @@ -3,7 +3,6 @@ #include -#include #include namespace eosio { @@ -21,7 +20,7 @@ namespace eosio { /** @brief Determine if a participant is in the security group */ bool is_in_security_group(chain::account_name participant) const { return cache_.find(participant) != cache_.end(); } private: - uint32_t version_ {std::numeric_limits::max()}; ///! The security group version - participant_list_t cache_; ///! Cache of accounts in the security group + uint32_t version_ {0}; ///! The security group version + participant_list_t cache_; ///! Cache of participants in the security group }; } diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index d084ce2e6d0..9528e40dcce 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include @@ -3494,6 +3494,9 @@ namespace eosio { return; } + std::vector added_connections; + added_connections.reserve(connections.size()); + // update connections // auto do_update = [&](auto& connection) { @@ -3502,7 +3505,10 @@ namespace eosio { return true; } if(security_group.is_in_security_group(participant.value())) { - connection->now_participating(); + if(connection->is_syncing()) { + connection->now_participating(); + added_connections.push_back(connection); + } } else { connection->now_syncing(); @@ -3511,6 +3517,12 @@ namespace eosio { }; for_each_connection(do_update); + + // send handshake when added to group + // + for(auto& connection : added_connections) { + connection->send_handshake(); + } } // called from application thread From 78c5a6b084b5cb9849ac8e4370558460342ec38e Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Wed, 10 Mar 2021 11:39:11 -0500 Subject: [PATCH 027/157] tls options added to net_plugin --- plugins/net_plugin/net_plugin.cpp | 4 +++- tests/generate-certificates.sh | 10 ++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index d3fa117d29c..22cfee48202 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -3653,7 +3653,9 @@ namespace eosio { " _lip \tlocal IP address connected to peer\n\n" " _lport \tlocal port number connected to peer\n\n") ( "p2p-keepalive-interval-ms", bpo::value()->default_value(def_keepalive_interval), "peer heartbeat keepalive message interval in milliseconds") - + ( "tls-ca-certificate-file", bpo::value(), "Certificate Authority's certificate file used for verifying peers TLS connection when security groups feature enabled" ) + ( "tls-own-certificate-file", bpo::value(), "Certificate file that will be used to authenticate running node if TLS is enabled") + ( "tls-private-key-file", bpo::value(), "Private key file that is used in conjunction with tsl-own-certificate-file for server authorization in TLS connection. Together tls-private-key-file + tsl-own-certificate-file automatically enables TLS-only connection for peers.") ; } diff --git a/tests/generate-certificates.sh b/tests/generate-certificates.sh index ac593e2dcdc..5762a30aa49 100755 --- a/tests/generate-certificates.sh +++ b/tests/generate-certificates.sh @@ -54,17 +54,11 @@ GROUP_SIZE=4 #overrides default is set parse-args "${@}" -echo "*************************************************" -echo " generating dh param " -echo "*************************************************" -#using low values like 128 here and below as this is for unit tests and our goal to save running time. For real applications 2048 recommended -openssl dhparam -out dh.pem 128 - echo "*************************************************" echo " generating CA_cert.pem " echo "*************************************************" -openssl req -newkey rsa:512 -nodes -keyout CA_key.pem -x509 -days ${DAYS} -out CA_cert.pem -subj "/C=US/ST=VA/L=Blocksburg/O=${CA_ORG}/CN=${CA_CN}" +openssl req -newkey rsa:2048 -nodes -keyout CA_key.pem -x509 -days ${DAYS} -out CA_cert.pem -subj "/C=US/ST=VA/L=Blocksburg/O=${CA_ORG}/CN=${CA_CN}" echo "*************************************************" openssl x509 -in CA_cert.pem -text -noout @@ -81,7 +75,7 @@ do echo "*************************************************" echo "generating certificate for $ORG_NAME / $CN_NAME " echo "*************************************************" - openssl req -newkey rsa:512 -nodes -keyout "${ORG_NAME}_key.pem" -out "${ORG_NAME}.csr" -subj "/C=US/ST=VA/L=Blockburg/O=${ORG_NAME}/CN=${CN_NAME}" + openssl req -newkey rsa:2048 -nodes -keyout "${ORG_NAME}_key.pem" -out "${ORG_NAME}.csr" -subj "/C=US/ST=VA/L=Blockburg/O=${ORG_NAME}/CN=${CN_NAME}" openssl x509 -req -in "${ORG_NAME}.csr" -CA CA_cert.pem -CAkey CA_key.pem -CAcreateserial -out "${ORG_NAME}.crt" -days ${DAYS} -sha256 echo "*************************************************" openssl x509 -in "${ORG_NAME}.crt" -text -noout From 1ecfaaf99cef581c06a99638cf4d35f442dc00de Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Wed, 10 Mar 2021 13:08:10 -0500 Subject: [PATCH 028/157] renamed command line options --- plugins/net_plugin/net_plugin.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 22cfee48202..a62c62cf353 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -3653,9 +3653,9 @@ namespace eosio { " _lip \tlocal IP address connected to peer\n\n" " _lport \tlocal port number connected to peer\n\n") ( "p2p-keepalive-interval-ms", bpo::value()->default_value(def_keepalive_interval), "peer heartbeat keepalive message interval in milliseconds") - ( "tls-ca-certificate-file", bpo::value(), "Certificate Authority's certificate file used for verifying peers TLS connection when security groups feature enabled" ) - ( "tls-own-certificate-file", bpo::value(), "Certificate file that will be used to authenticate running node if TLS is enabled") - ( "tls-private-key-file", bpo::value(), "Private key file that is used in conjunction with tsl-own-certificate-file for server authorization in TLS connection. Together tls-private-key-file + tsl-own-certificate-file automatically enables TLS-only connection for peers.") + ( "p2p-tls-ca-certificate-file", bpo::value(), "Certificate Authority's certificate file used for verifying peers TLS connection when security groups feature enabled" ) + ( "p2p-tls-own-certificate-file", bpo::value(), "Certificate file that will be used to authenticate running node if TLS is enabled") + ( "p2p-tls-private-key-file", bpo::value(), "Private key file that is used in conjunction with tsl-own-certificate-file for server authorization in TLS connection. Together tls-private-key-file + tsl-own-certificate-file automatically enables TLS-only connection for peers.") ; } From 7974437c5f19535b68fa70690f473d7481b5f409 Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Wed, 10 Mar 2021 12:35:11 -0600 Subject: [PATCH 029/157] EPE-672: Add security group to net_plugin Added the security_group_manager class to net_plugin and added logic to handle connections that are and are not participating in a security group. --- plugins/net_plugin/net_plugin.cpp | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 9528e40dcce..8c4d13babcd 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -594,15 +594,6 @@ namespace eosio { * @param endpoint The connection endpoint */ explicit connection( string endpoint ); - /** @brief For TLS connection setup - * - * @param endpoint The connection endpoint - * @param participant The name of the participant - * @param participaing Indicates if the participant is participating in block creation (default = true) - * true - block creation - * false - syncing only - */ - explicit connection( string endpoint, chain::account_name participant, bool participaing = true ); connection(); ~connection() {} @@ -927,13 +918,6 @@ namespace eosio { fc_ilog( logger, "creating connection to ${n}", ("n", endpoint) ); } - connection::connection( string endpoint, chain::account_name participant, bool participating ) - : connection(endpoint) - { - participant_name_ = participant; - participating_.store(participating, std::memory_order_relaxed); - } - connection::connection() : peer_addr(), strand( my_impl->thread_pool->get_executor() ), @@ -1492,6 +1476,18 @@ namespace eosio { void connection::enqueue( const net_message& m ) { verify_strand_in_this_thread( strand, __func__, __LINE__ ); + // for tls connections, when the connection is not in the security group + // certain message types will not be transmitted + if(participant_name_ && !participating_) { + const bool ignore = std::holds_alternative(m) || + std::holds_alternative(m) || + std::holds_alternative(m) || + std::holds_alternative(m) || + std::holds_alternative(m); + if(ignore) { + return; + } + } go_away_reason close_after_send = no_reason; if (std::holds_alternative(m)) { close_after_send = std::get(m).reason; @@ -3497,7 +3493,7 @@ namespace eosio { std::vector added_connections; added_connections.reserve(connections.size()); - // update connections + // update connection // auto do_update = [&](auto& connection) { const auto& participant = connection->participant_name(); From bca93440c26008793d10e76968431ca346ddf4d4 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Wed, 10 Mar 2021 14:00:49 -0500 Subject: [PATCH 030/157] comments updates after net_plugin options rename --- plugins/net_plugin/net_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index a62c62cf353..274b661ce47 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -3655,7 +3655,7 @@ namespace eosio { ( "p2p-keepalive-interval-ms", bpo::value()->default_value(def_keepalive_interval), "peer heartbeat keepalive message interval in milliseconds") ( "p2p-tls-ca-certificate-file", bpo::value(), "Certificate Authority's certificate file used for verifying peers TLS connection when security groups feature enabled" ) ( "p2p-tls-own-certificate-file", bpo::value(), "Certificate file that will be used to authenticate running node if TLS is enabled") - ( "p2p-tls-private-key-file", bpo::value(), "Private key file that is used in conjunction with tsl-own-certificate-file for server authorization in TLS connection. Together tls-private-key-file + tsl-own-certificate-file automatically enables TLS-only connection for peers.") + ( "p2p-tls-private-key-file", bpo::value(), "Private key file that is used in conjunction with p2p-tls-own-certificate-file for server authorization in TLS connection. Together p2p-tls-private-key-file + p2p-tsl-own-certificate-file automatically enables TLS-only connection for peers.") ; } From 135391cd7811cb297764c8151d397a552fbd79ff Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 10 Mar 2021 15:22:18 -0600 Subject: [PATCH 031/157] address PR comments & add contract test --- libraries/chain/controller.cpp | 8 +- .../chain/include/eosio/chain/controller.hpp | 4 +- .../eos-vm-oc/intrinsic_mapping.hpp | 4 +- .../eosio/chain/webassembly/interface.hpp | 4 +- .../chain/webassembly/runtimes/eos-vm.cpp | 4 +- .../chain/webassembly/security_group.cpp | 8 +- unittests/contracts.hpp.in | 1 + unittests/security_group_tests.cpp | 82 ++++++++++++++---- unittests/test-contracts/CMakeLists.txt | 1 + .../security_group_test/CMakeLists.txt | 6 ++ .../security_group_test.abi | 78 +++++++++++++++++ .../security_group_test.cpp | 38 ++++++++ .../security_group_test.wasm | Bin 0 -> 38982 bytes 13 files changed, 207 insertions(+), 31 deletions(-) create mode 100644 unittests/test-contracts/security_group_test/CMakeLists.txt create mode 100644 unittests/test-contracts/security_group_test/security_group_test.abi create mode 100644 unittests/test-contracts/security_group_test/security_group_test.cpp create mode 100755 unittests/test-contracts/security_group_test/security_group_test.wasm diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 445638ef2a8..18418ba7f04 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -2875,13 +2875,13 @@ const flat_set& controller::proposed_security_group_participants() return get_global_properties().proposed_security_group_participants; } -int64_t controller::propose_security_group_participants_add(const flat_set& participants) { +int64_t controller::add_security_group_participants(const flat_set& participants) { return participants.size() == 0 ? -1 : my->propose_security_group([&participants](auto& pending_participants) { pending_participants.insert(participants.begin(), participants.end()); }); } -int64_t controller::propose_security_group_participants_remove(const flat_set& participants) { +int64_t controller::remove_security_group_participants(const flat_set& participants) { return participants.size() == 0 ? -1 : my->propose_security_group([&participants](auto& pending_participants) { flat_set::sequence_type tmp; tmp.reserve(pending_participants.size()); @@ -3407,8 +3407,8 @@ void controller_impl::on_activation void controller_impl::on_activation() { db.modify( db.get(), [&]( auto& ps ) { - add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "propose_security_group_participants_add" ); - add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "propose_security_group_participants_remove" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "add_security_group_participants" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "remove_security_group_participants" ); add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "in_active_security_group" ); add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "get_active_security_group" ); } ); diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index c408a1cea13..87ac7fe3bf4 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -295,8 +295,8 @@ namespace eosio { namespace chain { const security_group_info_t& active_security_group() const; const flat_set& proposed_security_group_participants() const; - int64_t propose_security_group_participants_add(const flat_set& participants); - int64_t propose_security_group_participants_remove(const flat_set& participants); + int64_t add_security_group_participants(const flat_set& participants); + int64_t remove_security_group_participants(const flat_set& participants); bool in_active_security_group(const flat_set& participants) const; diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp index 9cec37f1a97..db050ca5c50 100644 --- a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp @@ -278,8 +278,8 @@ inline constexpr auto get_intrinsic_table() { "env.set_wasm_parameters_packed", "env.get_parameters_packed", "env.set_parameters_packed", - "env.propose_security_group_participants_add", - "env.propose_security_group_participants_remove", + "env.add_security_group_participants", + "env.remove_security_group_participants", "env.in_active_security_group", "env.get_active_security_group" ); diff --git a/libraries/chain/include/eosio/chain/webassembly/interface.hpp b/libraries/chain/include/eosio/chain/webassembly/interface.hpp index ba0093650f0..2f1d9abd36e 100644 --- a/libraries/chain/include/eosio/chain/webassembly/interface.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/interface.hpp @@ -1968,7 +1968,7 @@ namespace webassembly { * * @return -1 if proposing a new security group was unsuccessful, otherwise returns 0. */ - int64_t propose_security_group_participants_add(span packed_participants); + int64_t add_security_group_participants(span packed_participants); /** * Propose to remove participants from the security group. @@ -1978,7 +1978,7 @@ namespace webassembly { * * @return -1 if proposing a new security group was unsuccessful, otherwise returns 0. */ - int64_t propose_security_group_participants_remove(span packed_participants); + int64_t remove_security_group_participants(span packed_participants); /** * Check if the specified accounts are all in the active security group. diff --git a/libraries/chain/webassembly/runtimes/eos-vm.cpp b/libraries/chain/webassembly/runtimes/eos-vm.cpp index 7654b844ded..66aed48eae5 100644 --- a/libraries/chain/webassembly/runtimes/eos-vm.cpp +++ b/libraries/chain/webassembly/runtimes/eos-vm.cpp @@ -276,8 +276,8 @@ REGISTER_HOST_FUNCTION(is_privileged, privileged_check); REGISTER_HOST_FUNCTION(set_privileged, privileged_check); // security group api -REGISTER_HOST_FUNCTION(propose_security_group_participants_add, privileged_check); -REGISTER_HOST_FUNCTION(propose_security_group_participants_remove, privileged_check); +REGISTER_HOST_FUNCTION(add_security_group_participants, privileged_check); +REGISTER_HOST_FUNCTION(remove_security_group_participants, privileged_check); REGISTER_HOST_FUNCTION(in_active_security_group); REGISTER_HOST_FUNCTION(get_active_security_group); diff --git a/libraries/chain/webassembly/security_group.cpp b/libraries/chain/webassembly/security_group.cpp index d86dc9b0eae..2ebd538ac51 100644 --- a/libraries/chain/webassembly/security_group.cpp +++ b/libraries/chain/webassembly/security_group.cpp @@ -6,18 +6,18 @@ namespace eosio { namespace chain { namespace webassembly { -int64_t interface::propose_security_group_participants_add(span packed_participants) { +int64_t interface::add_security_group_participants(span packed_participants) { datastream ds(packed_participants.data(), packed_participants.size()); flat_set participants; fc::raw::unpack(ds, participants); - return context.control.propose_security_group_participants_add(participants); + return context.control.add_security_group_participants(participants); } -int64_t interface::propose_security_group_participants_remove(span packed_participants) { +int64_t interface::remove_security_group_participants(span packed_participants) { datastream ds(packed_participants.data(), packed_participants.size()); flat_set participants; fc::raw::unpack(ds, participants); - return context.control.propose_security_group_participants_remove(participants); + return context.control.remove_security_group_participants(participants); } bool interface::in_active_security_group(span packed_participants) const { diff --git a/unittests/contracts.hpp.in b/unittests/contracts.hpp.in index 2b675f43b43..274ce0a8d86 100644 --- a/unittests/contracts.hpp.in +++ b/unittests/contracts.hpp.in @@ -59,6 +59,7 @@ namespace eosio { MAKE_READ_WASM_ABI(params_test, params_test, test-contracts) MAKE_READ_WASM_ABI(kv_table_test, kv_table_test, test-contracts) MAKE_READ_WASM_ABI(kv_addr_book, kv_addr_book, test-contracts) + MAKE_READ_WASM_ABI(security_group_test, security_group_test, test-contracts) }; } /// eosio::testing } /// eosio diff --git a/unittests/security_group_tests.cpp b/unittests/security_group_tests.cpp index 5ced1b91101..5aa935478fd 100644 --- a/unittests/security_group_tests.cpp +++ b/unittests/security_group_tests.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace eosio { namespace chain { @@ -210,7 +211,7 @@ BOOST_AUTO_TEST_CASE(test_participants_change) { } participants_t new_participants({"alice"_n, "bob"_n}); - chain.control->propose_security_group_participants_add(new_participants); + chain.control->add_security_group_participants(new_participants); BOOST_TEST(chain.control->proposed_security_group_participants() == new_participants); BOOST_CHECK_EQUAL(chain.control->active_security_group().participants.size() , 0); @@ -222,7 +223,7 @@ BOOST_AUTO_TEST_CASE(test_participants_change) { BOOST_CHECK(chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); BOOST_CHECK(!chain.control->in_active_security_group(participants_t{"bob"_n, "charlie"_n})); - chain.control->propose_security_group_participants_remove({"alice"_n}); + chain.control->remove_security_group_participants({"alice"_n}); BOOST_TEST(chain.control->proposed_security_group_participants() == participants_t{"bob"_n}); chain.produce_block(); @@ -242,36 +243,39 @@ void push_blocks( eosio::testing::tester& from, eosio::testing::tester& to ) { } } -static const char propose_security_group_participants_add_wast[] = R"=====( +// The webassembly in text format to add security group participants +static const char add_security_group_participants_wast[] = R"=====( (module (func $action_data_size (import "env" "action_data_size") (result i32)) (func $read_action_data (import "env" "read_action_data") (param i32 i32) (result i32)) - (func $propose_security_group_participants_add (import "env" "propose_security_group_participants_add") (param i32 i32)(result i64)) + (func $add_security_group_participants (import "env" "add_security_group_participants") (param i32 i32)(result i64)) (memory 1) (func (export "apply") (param i64 i64 i64) (local $bytes_remaining i32) (set_local $bytes_remaining (call $action_data_size)) (drop (call $read_action_data (i32.const 0) (get_local $bytes_remaining))) - (drop (call $propose_security_group_participants_add (i32.const 0) (get_local $bytes_remaining))) + (drop (call $add_security_group_participants (i32.const 0) (get_local $bytes_remaining))) ) ) )====="; -static const char propose_security_group_participants_remove_wast[] = R"=====( +// The webassembly in text format to remove security group participants +static const char remove_security_group_participants_wast[] = R"=====( (module (func $action_data_size (import "env" "action_data_size") (result i32)) (func $read_action_data (import "env" "read_action_data") (param i32 i32) (result i32)) - (func $propose_security_group_participants_remove (import "env" "propose_security_group_participants_remove") (param i32 i32)(result i64)) + (func $remove_security_group_participants (import "env" "remove_security_group_participants") (param i32 i32)(result i64)) (memory 1) (func (export "apply") (param i64 i64 i64) (local $bytes_remaining i32) (set_local $bytes_remaining (call $action_data_size)) (drop (call $read_action_data (i32.const 0) (get_local $bytes_remaining))) - (drop (call $propose_security_group_participants_remove (i32.const 0) (get_local $bytes_remaining))) + (drop (call $remove_security_group_participants (i32.const 0) (get_local $bytes_remaining))) ) ) )====="; +// The webassembly in text format to assert the given participants are all in active security group static const char assert_in_security_group_wast[] = R"=====( (module (func $action_data_size (import "env" "action_data_size") (result i32)) @@ -291,7 +295,7 @@ static const char assert_in_security_group_wast[] = R"=====( ) )====="; - +// The webassembly in text format to assert the given participants are exactly the entire active security group static const char assert_get_security_group_wast[] = R"=====( (module (func $action_data_size (import "env" "action_data_size") (result i32)) @@ -332,8 +336,8 @@ BOOST_AUTO_TEST_CASE(test_security_group_intrinsic) { chain1.create_accounts({ "addmember"_n, "rmmember"_n, "ingroup"_n, "getgroup"_n }); chain1.produce_block(); - chain1.set_code( "addmember"_n, propose_security_group_participants_add_wast ); - chain1.set_code( "rmmember"_n, propose_security_group_participants_remove_wast ); + chain1.set_code( "addmember"_n, add_security_group_participants_wast ); + chain1.set_code( "rmmember"_n, remove_security_group_participants_wast ); chain1.set_code( "ingroup"_n, assert_in_security_group_wast ); chain1.set_code( "getgroup"_n, assert_get_security_group_wast ); chain1.produce_block(); @@ -346,10 +350,13 @@ BOOST_AUTO_TEST_CASE(test_security_group_intrinsic) { chain1.produce_blocks(3); // Starts new blocks which promotes the proposed schedule to pending BOOST_REQUIRE_EQUAL( chain1.control->active_producers().version, 1u ); - BOOST_TEST_REQUIRE(chain1.push_action( eosio::chain::action({}, "addmember"_n, {}, participants_payload({"alice"_n, "bob"_n})), "addmember"_n.to_uint64_t() ) == ""); - chain1.produce_blocks(11); + BOOST_TEST_REQUIRE(chain1.push_action( eosio::chain::action({}, "addmember"_n, {}, participants_payload({"alice"_n})), "addmember"_n.to_uint64_t() ) == ""); + chain1.produce_block(); + BOOST_TEST_REQUIRE(chain1.push_action( eosio::chain::action({}, "addmember"_n, {}, participants_payload({"bob"_n})), "addmember"_n.to_uint64_t() ) == ""); + chain1.produce_blocks(10+11); BOOST_CHECK_EQUAL(chain1.control->proposed_security_group_participants().size() , 2); - chain1.produce_blocks(12); + BOOST_CHECK_EQUAL(chain1.control->active_security_group().participants.size(), 0); + chain1.produce_blocks(1); BOOST_CHECK_EQUAL(chain1.control->proposed_security_group_participants().size() , 0); BOOST_CHECK(chain1.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); @@ -369,7 +376,9 @@ BOOST_AUTO_TEST_CASE(test_security_group_intrinsic) { fc::raw::pack(strm, grp); BOOST_TEST_REQUIRE(chain1.push_action(eosio::chain::action({}, "getgroup"_n, {}, strm.storage()), "getgroup"_n.to_uint64_t()) == ""); - chain1.produce_blocks(11); + BOOST_TEST_REQUIRE(chain1.push_action( eosio::chain::action({}, "addmember"_n, {}, participants_payload({"charlie"_n})), "addmember"_n.to_uint64_t() ) == ""); + chain1.produce_blocks(11); + BOOST_TEST(chain1.control->proposed_security_group_participants() == participants_t({"bob"_n, "charlie"_n})); chain1.control->abort_block(); /// Test snapshot recovery @@ -390,6 +399,49 @@ BOOST_AUTO_TEST_CASE(test_security_group_intrinsic) { const auto& active_security_group = chain2.control->active_security_group(); BOOST_CHECK_EQUAL(2, active_security_group.version); BOOST_TEST(active_security_group.participants == participants_t{"bob"_n}); + BOOST_TEST(chain2.control->proposed_security_group_participants() == participants_t({"bob"_n, "charlie"_n})); + } +} + + +BOOST_AUTO_TEST_CASE(test_security_group_contract) { + eosio::testing::tester chain; + using namespace eosio::chain::literals; + + chain.create_accounts({"secgrptest"_n,"alice"_n,"bob"_n,"charlie"_n}); + chain.produce_block(); + chain.set_code("secgrptest"_n, eosio::testing::contracts::security_group_test_wasm()); + chain.set_abi("secgrptest"_n, eosio::testing::contracts::security_group_test_abi().data()); + chain.produce_block(); + chain.push_action( "eosio"_n, "setpriv"_n, "eosio"_n, fc::mutable_variant_object()("account", "secgrptest"_n)("is_priv", 1)); + chain.produce_block(); + + chain.push_action("secgrptest"_n, "add"_n, "secgrptest"_n, fc::mutable_variant_object() + ( "nm", "alice" ) + ); + chain.push_action("secgrptest"_n, "add"_n, "secgrptest"_n, fc::mutable_variant_object() + ( "nm", "bob" ) + ); + chain.produce_block(); + BOOST_CHECK(chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + + chain.push_action("secgrptest"_n, "remove"_n, "secgrptest"_n, fc::mutable_variant_object() + ( "nm", "alice" ) + ); + chain.produce_block(); + BOOST_CHECK(chain.control->in_active_security_group(participants_t({"bob"_n}))); + + { + auto result = + chain.push_action("secgrptest"_n, "ingroup"_n, "secgrptest"_n, fc::mutable_variant_object()("nm", "alice")); + + BOOST_CHECK_EQUAL(false, fc::raw::unpack(result->action_traces[0].return_value)); + } + + { + auto result = chain.push_action("secgrptest"_n, "activegroup"_n, "secgrptest"_n, fc::mutable_variant_object()); + auto participants = fc::raw::unpack(result->action_traces[0].return_value); + BOOST_TEST(participants == participants_t({"bob"_n})); } } diff --git a/unittests/test-contracts/CMakeLists.txt b/unittests/test-contracts/CMakeLists.txt index 6ac0c78bc51..56006197e82 100644 --- a/unittests/test-contracts/CMakeLists.txt +++ b/unittests/test-contracts/CMakeLists.txt @@ -35,3 +35,4 @@ add_subdirectory( wasm_config_bios ) add_subdirectory( params_test ) add_subdirectory( kv_table_test ) add_subdirectory( kv_addr_book ) +add_subdirectory( security_group_test ) diff --git a/unittests/test-contracts/security_group_test/CMakeLists.txt b/unittests/test-contracts/security_group_test/CMakeLists.txt new file mode 100644 index 00000000000..b5ca6ad849d --- /dev/null +++ b/unittests/test-contracts/security_group_test/CMakeLists.txt @@ -0,0 +1,6 @@ +if( EOSIO_COMPILE_TEST_CONTRACTS ) + add_contract( security_group_test security_group_test security_group_test.cpp ) +else() + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/security_group_test.wasm ${CMAKE_CURRENT_BINARY_DIR}/security_group_test.wasm COPYONLY ) + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/security_group_test.abi ${CMAKE_CURRENT_BINARY_DIR}/security_group_test.abi COPYONLY ) +endif() diff --git a/unittests/test-contracts/security_group_test/security_group_test.abi b/unittests/test-contracts/security_group_test/security_group_test.abi new file mode 100644 index 00000000000..2d01efba645 --- /dev/null +++ b/unittests/test-contracts/security_group_test/security_group_test.abi @@ -0,0 +1,78 @@ +{ + "____comment": "This file was generated with eosio-abigen. DO NOT EDIT ", + "version": "eosio::abi/1.2", + "types": [], + "structs": [ + { + "name": "activegroup", + "base": "", + "fields": [] + }, + { + "name": "add", + "base": "", + "fields": [ + { + "name": "nm", + "type": "name" + } + ] + }, + { + "name": "ingroup", + "base": "", + "fields": [ + { + "name": "nm", + "type": "name" + } + ] + }, + { + "name": "remove", + "base": "", + "fields": [ + { + "name": "nm", + "type": "name" + } + ] + } + ], + "actions": [ + { + "name": "activegroup", + "type": "activegroup", + "ricardian_contract": "" + }, + { + "name": "add", + "type": "add", + "ricardian_contract": "" + }, + { + "name": "ingroup", + "type": "ingroup", + "ricardian_contract": "" + }, + { + "name": "remove", + "type": "remove", + "ricardian_contract": "" + } + ], + "tables": [], + "kv_tables": {}, + "ricardian_clauses": [], + "variants": [], + "action_results": [ + { + "name": "activegroup", + "result_type": "name[]" + }, + { + "name": "ingroup", + "result_type": "bool" + } + ] +} \ No newline at end of file diff --git a/unittests/test-contracts/security_group_test/security_group_test.cpp b/unittests/test-contracts/security_group_test/security_group_test.cpp new file mode 100644 index 00000000000..c7171620621 --- /dev/null +++ b/unittests/test-contracts/security_group_test/security_group_test.cpp @@ -0,0 +1,38 @@ +#include +#include +#include + +using namespace eosio; + +class [[eosio::contract]] security_group_test : public contract { + public: + using contract::contract; + + [[eosio::action]] + void add( name nm ); + [[eosio::action]] + void remove( name nm ); + + [[eosio::action]] + bool ingroup( name nm ) const; + + [[eosio::action]] + std::set activegroup() const; + + using add_action = action_wrapper<"add"_n, &security_group_test::add>; + using remove_action = action_wrapper<"remove"_n, &security_group_test::remove>; + using in_active_group_action = action_wrapper<"ingroup"_n, &security_group_test::ingroup>; + using active_group_action = action_wrapper<"activegroup"_n, &security_group_test::activegroup>; +}; + +[[eosio::action]] void security_group_test::add(name nm) { eosio::add_security_group_participants({nm}); } + +[[eosio::action]] void security_group_test::remove(name nm) { eosio::remove_security_group_participants({nm}); } + +[[eosio::action]] bool security_group_test::ingroup(name nm) const { + return eosio::in_active_security_group({nm}); +} + +[[eosio::action]] std::set security_group_test::activegroup() const { + return eosio::get_active_security_group().participants; +} \ No newline at end of file diff --git a/unittests/test-contracts/security_group_test/security_group_test.wasm b/unittests/test-contracts/security_group_test/security_group_test.wasm new file mode 100755 index 0000000000000000000000000000000000000000..0e0112c621ca562c479a52df3a42d36767d9ffdf GIT binary patch literal 38982 zcmeIb3zS^PdEZ(0-tM06xji#Ig9iZ&DONYhn3PC)@CFTD3%?0 ztt8Mn*&I1u8~gizb^9^EfFxvZ9G{Z`y6?U9`0A_gRbPEox7tlCpNyPy(Qig~J>lFF z$?EEAw94h&YIN?20#B>%iI}u=`VP0NPdJJ=OM2ElQCvO8pR&%nCknbJH|G1SeLqVE zxrvpD1Uv$cAfE$*wFm68t9fy<&vIJ;Xx^LvC|5;k1;kcqQ-9o~b`!pMG=$XzEz%Z% z!7Tol#d^}5KYi!K)YS3i=H$xKbnBVpPc1F1EFNE+SZYmAPA^W(x0YQ&rHT_zE-ba2 z(|xtMuspqRd}4XINm5ZsrIXE*%gvUH^w7t{$%WI+@YGmp>f!a9*fBkSd}6XSeM4DS z64aefHCwNhFBB#hp9ziD1ZZLY_|!ye;`s9PN1CorFltN9i75k;=LOKWfqFZ2so7dt znm>MeVs51w`sS7UXPZ-bG2!=`#jG3K1#NL@VR2!( zIi;Jal}SJaDp%pcQggm_yfuBY>B>TRYfrbkr47`;@#D=it>*mH@`=`oJ+7kS<>SXs zuFSPg>~;NmOphNwF}E<$TAps52xw9OhIa~dU`!c|RF(RUAE!|uR`o;)1F)+mg+m58LbJyt3XxC_|bbE3~@}}fY@=Fo_>_YWBM(>R7Ecubqda+ol508`@{1pqu zXff*RD<+YPO52mwC|g^1>8i`V@s-Y{uM!m}78mE9ag9nfa#@s?9{#|@AOQc~|=@wYqmc`o6-^@wM>xlY4y3qf%=xM8%N&Ghg^WKD*jY|EqcWKm66N ze)B)MZu*N#|KgAS{Ez(1=YFII%wLO&-UUGGpZ(Ntxc3!~X7>D>U-+qi|Iz>GYChFp z-j)^VqV$jc%UA#Je-PTH(sOHTy_GUoJDV;=$!ZM311@tzX~y@x@h5oBYNs2~Od~$P zQ%HUOXe04%CL5Z`7Ed+Yf%y5ORE)D)%e!}}I``3x%KM0$dX$nK9d+62L(0w0ue(~D z`Xv3KD6|=QH`j=+YdBKF@k}Gx&@l9DH(baYPSkKAG@P^>76$cL4ae6v-0dWt6MvFj z-lefi4@A-GUG#UhLHdEXN+vX@-W$7aSJ#dzRV`=k2-i;>O-nBOJfibr9F>fKKwX_V z_27W3eOI>q0F@!9+RRp$ee_;SuJ$I^KpiaZYL|l6bh36=n<65fSCU$uh2T)pM^-w@ z&O#s%Y!tG-Q;nDmmz|k`>}O_3^^}{1d9+6Q*%_~*B-!&&3fMw+=F|+8&&)K602_c~R{iNVx|(X4~tD5YGw^54g~S+P&&g?Yn4) zwrRtNP){TYd!u+26eI6O>rs3(p(X@y15xJc5b)mk%SRjWg9FfE*tx1psSn-JynE83 zt3@t-7)he6+N2yU0>VAuAdHW6DLzEyAdPBgrY<2}h13JC8UwE2i?achpO#R1Cp7er z(utU z^Fw+@7t3??g#N%y=ac14X)qw*1#zb!(Jo{O$|3fsp&aSBi*%ef*T%6RmKIba3+fCO z5(n6+mJ=`HORN($!lN#5DwhNktB*#AuL$Vr(rgy5_!y=2Diya(-&6$<0N4OCuo=*B zI}ooUUAKzE{f5GIsQ2K#0pSLPA;9%A1{FZssR!wIyPOb#bYei%C7sc`3lh?8s!ftx zv`Lp-n~ZolzuNV$)20fv&!p+8-HG^V)n#8{2sn^^9kW>y_6U(nKPM56kJb*U1qavK z$Ud66cZ!F#!@Eq_?v1xfh;Dzk5;Y|0>F(jkwX+KiI(~0lZW!l`ce5{NmUi!OOflsi z!n`0T@llzVZydE2p%M?b(4LCFZfl{5Z~p(;M2a)==hNpRGow=IM;Z}A?W3jdagT&v z-IQjc;eq&ed<%ClsfEIh>IGRO5A`xHu+n33%;ZMR9!wUw&=h5hat#^7PQG{hV;kn#686gW=*RhlHLMdqEMD+V^o?lzwx z!y-Fr2>{d#>dED4nJXDk4g3XP0A${N?P$G_W>LLJ`9kK^P$B#9OjdljUQB%vbzlfB zS?2fS5M(i}@KhX3Tm&{5BP#csbH~kT8F#fKb{FBAlVeKd+xblM4IQ!6*LAuX#N$vwjWDeK2K35{ltp;cMsT>}Y>U~u-utbb1I3P`^( z27COmnViRMcn(NE@>9R_E78M9JFI?%F6IsO*CXFI-hZx9_VLS&3KISU1M#g2l`s3s z*?J#n7SxBL`rrWjqqM1ZcYe?wt(9WC6*(s0l2zV~qm-mQL;+3&U4t7f(pW-=zKtTL z9}0n{I_sl3eEnM_QU{n(D$aiKx4v~qI7hztE~8NljWmMNmI%Zk^-80Et&#q~Kt!{B zKJ?|4@oUk?+Elwm6avj?qJxV%76Uu|!Q97-vfJje;%U6?>?e`o`%&fU>(is!TKZ$t z$R!8a?XeTckvvFF#N1>nH))eLs{yNP2Q$ z{VhvpPj&hvNo&ZPDrFe!@#tfZVbFX6yAiWZwU+cgpU_iol9w{`&r3FwGdRik>RE;! zihyk!cw#A~mmB4e`|`7rrM`4ER^*QH@jUaggz=8C-~i*PT?)94@{hQz9*7^+g`pWf zT$J*L0WAgl;FeW0?&@GbVrSwl43(=!|%UEQ%z-=y?ACH)!QgW;th$FK_oB5i{ zt|sI*$Yr4;m+~=k zxlDXPa>-YDR^J30G?zlC>4!;Z0sJJP#q=LXQ4z7fH-5a4WIw%bB27#HZ0kiniTa82 ziNNcL*=PxjN+MF9h(sRRG~@d+s`h1T(Zi$oQ7Af76;#}&<_3VMgh*)>8YTHj5P<@q zrQT2lFc2s{8fIb4bX|t$q@3KbihK#5!rZiC`cM9JD7`@ezeAG$%%2YGZH@iTuzo51 zQ$o9~;@>ISgqQO00ukr3&1=8c+257bPuHc#QTCi^$XfY5nDrPZllhVBEkuz&yv@Ge zIhuWRk$8kA9;){UHQGg=NZRmzxlbDR+?hvnc$%f;@T?G3uRd*GkBnx2V+eRs4MIeH zGp~&Ad*Rhjr~3h`8JRAH@v#?Q%GTE6Q$DuaYlM=PUKt;IfoE|Q&*5XwA56;WyO@I! z*NZx-_28vNooXmKwV8-y%#oNbMOmCNwt4sbW3q5|!<>bo>+7XLfuSo}O*DS4Va|$o zLoIVOumj`rt9NP~7qWO?Qfm+bBt0w7 z+?TipIXN~g6^8@QRB z!Qo&X?XRH0I^1?!(%cj{C24Xcf7DsH_y8WvAn4*o9PLJ9Ry65S8x><7dSyA!vSDtMQjIVw_ zJmKlty4o-MeuOaz?8Bhu5!L~pK0of>XUO@D#JIOh3lA-rucqTiUaVLAz_8+X6&Bx!jW28B4MMO+;UHfUeFuP$B9grrjcvr}P#2@IoI73W z{D68FFiNZi$X^g*pkNq+R)>H%mf*^8g-+Sjx@>|1hw%9!T9@X4LMnX8)D2R#L0mdw zJpX);#2hu8Hq>p`Q@{m14(c4EskF4F0XQ8&!GAtyC3@^W_G*7Rk1;b}h*-@M~j<;QV>Vl)O9D`pbJV=0(@G+mP?`8dqsabX&j>=VD> zv;tJ`_myldzfW_0E6ZJ~_&zQQ6)_DpkH+~5{!E>fPR0^KMHPjLse9f7$qHQ9&j+)v%1E8GsMjQmhK*y+RSjiK+&WYFMfg$HSDy5B%@%aBgk& zZ`Fp|imwq+BYLq~lf@GE1zu}hzB*Wlor~f&CO6=bhsOw%T~G1XR^pu7_cB?N^dpt| zpm7D;1V%Cg4-O1$=yY}9mTd*Zcok|4<;^`dll6<4L)k7}D%p=Ilk2~+_%_M!l_yrk z@)%CnAyvulJ2oKOBkVR~%AjU@)dvUAW_?Nj>U~*Yi!?l%X;f@_T2^REE8bP?Rz=b0 z9#yaQ7(t;zF<+Gt6q!tmW$dJ}iai%@jqqSIs2Z}mYFXj-O7>P{Z`{>u|1J;MKLR6L0`T47^(%3w(_rSG|@4k@b&-Oq*}(*N)VO z>C*^pmH2Govz6yLZi`WO4jpA=k#N^J{w zxG$*WSYxXd4&!qJg}o1qtz1a5T1*;Bo=UhgRx{61)s9bM*|jh-ZxdqD<3e@?r>z%} zwQB(pdEhzBCql6vpk#)p`WDErO$|R*FELXXkIoVe5%uZoM)A416a=yHgEj%ViCP&+ z+lG!wP7XX+LIpyUR>{suQA{Qk^pU%Cm3|ETY(A%S?$b4}ZO>oK=q+OOs)04eu;Tcc zv-M$!EWXu8;8AoICJcWNd%Cqg>?`IHR{{JG5ph=BYL7RDbHjy8w)(yS#!v7;yoPfV zhM#Tt25h?v3KIV!lTc9Qu2iikpF#@YFRz1X$@j@ zJbu9spB;bVEY76!{j;xVWwMZ66daSZ3*5oMi;6{)8aO+?nmgOCw*6+}ZYp*`#i+7r zv37_#MCrl=)sQ+LkOHCp?C+nOdHMYVOt>Mj*3V6#FA^I=-nt5L?bX%V)(uG#&Fz{$ zWfwwWa)me37T`3>qf&tkKmj#h9tA*A#-SMMv}M-1kubuH;0988^^v8RsQe@?j*q zj-vq6My`IJ&NObDt?$q_zuH|lv&@v<3Ao}-EHeWQ9SP&abbLpIOnsZIQ0O#} z2k(uq9?2r2)?B{w$*8UkA>i>wO{74HMqC1Apo+X0Cg(-8U?)3Bfri(iF5wjXcJM-d zFi%@Hm?wP82L}jC)wgUemuOKDl~EugP!5ntJKfn-jG1!9xd(W?J+m3~m5!Yuk}k!# zA_GEdZa_Mbf-pN@4}~tP5EV8l)U`2`%~jeA!MbdW1(6^(z8S<|pp=4MZ~s3ZbT!mU zsAXewX}qGb8G;h$ft=bnl*Q&G`t-C20N#MB)NbSMYDe=gb4S0fYRXM&K*@Tl6l#PN z8>uPVi}K}8<1&orPQDy$^Myj&s+{Kz*cY_$L29UbqitRb#m?LM$W4lM=|<^ZB-RRT z3bd`Wj2`6gV6h&+Ulw>$b1bgo+{I96U4^KyNue$VlzVDK9F)bAz8P^DZS3W3O%(ajWP#)034e!L{p;&){g1$K7%5dj{_ zhPjZ8agjd<<2s#PE6zw?opJd}=+sqpiXs7Nab!)_pQ0hfcuU#dPF+!_Y}PaqdIMPi z9Y%}Gy^UPt0g&qgNs(?Nmz6@%o<=U{$&avku9aQ6WT6xr#Lov97gYmzT^JXXVzo3r z5jbYhYUhV(-}#~;YMqwz>Ql(_ELLxn=wtRKvy;A@E5P|uTLCU`7do@XorDy9%1G_D zeO@Wl=)wOgc#a79&-5UFB_O}1;sEOcxGLnS(gT39;VB@0MaZL?#_qD2a6iZbA#!7a zV&UZq+lXzB&CbmTL6KcY&@>2zxIbdBAcH}IO|UB1yowNA#h<@DnCg&Ai8AeFYk%wa zzLLO++j}|D*mkl`D4h{yu#VBRd0!PY04IyI({Gz6em}$%L-RR=_87xOpT-YX1BOm9 zu8jJj2TwH^WEf%)em|lCM`m9g@;eF9ZkwC+yXI(u`J>Gvhh+65Zy?JO`ZD|zUW`H` z%%)HuqF@Yc+wvLTNOrd(q+H~|ZUeLI^6&mW`_!@5EJv`)G9=u6IcyEChdmR6%$UKE z2p+H+(f-(t-|>R)!%ZrVKPO_akw(?T4Aw2ijX-OSQn)$yEKV51Y}mN+QXl|cvD_BA z$XE54z7_ITGY_%bnqG&gxvyGB<7|@MG@j*DC+}9l4a2A!wu}xEX#6Ag@lo4Q3`D2* z+OA^-%Sxxa+B7H`7lGQ;@qHwMJCIFo7+L@Xdzka)^^J&l8ySukwI4t<`T|LX4TKSS zaKtoc%xNyJN7=7KT@54QPW!#(s#$TYRVdE-lHo@2FMaku|Aq}a`E95W-@L-w*M9lG zxX^&zh6;={LTIxVu3h_se_}0|3`8JiDXLe-#lmAH!$MJzTQo0AdkocF@_paw`cAfi z^tIS8UQb{=ovKUL2kD$7WXKPx_d}HQ2)K6I8kd|4M2jvegFepF)z-3^q!4zPfp2WSap2t(y>|LD*D zkG2Mem9y5`0vQ|%3=XZF?+^95MsQ6jRLG5>X7yJq8!)?C$uY|@>0p-2z$Tbop|wph zyG({;K8T}XwjMCMqNx}4x|m&73Ke=VqilEz=6StR-vZMrS&m5uGsa3u=k=K3mf8qL zi(7ElZ=lJz<+nIHr$PodRwu!<8Hj{Nc^~*;>q~<({ML%p9viCKk zP%nnmE9h zMazN4p}h`nAxif$GG-B(dsj$VAC!d2?lHTgeBrDamS{_J&Jxl@@ZQ`*YX*n zu)2zkw!oM`UWpab%>WV78nvMMFD6oXn3UAF3zN&VF~Xn>iqKq9GCCog)6Bo~6i(Sp zFw_(q4*GmqZBZ@2wfV`#kUOp|WR+1aTew)x1#-4y321R$_0~J}%np!C&PC+_ql^6o zrBF9-LuJ=O+4IUd-_8;28eX*ZC~inZD3|v*?@VWVs=9lXAb^6;#g_u*za0h2UuA}` z4FbimL7=4dYH;e;D)r|Jv9{ru-C?s`A-tNAzj7d%&{n!wODI}oU?1X9HXEpP_=niA zT%5F*i<5l0xC<`Pj=UUPl7mYGPMgII908`7rDYqy2-dLv>LbZ64a&eFlZk>hkpUWr z$MXezV#eA~r1*~fC_0#ElX}E`f#eeX%8P45=Xp_!M> zYY16t>19UT{&N-~1NS%HRB8ON5LUCM40~j3{2zVt<6ruvfBt*_bM$`HD({s`jNZ2Z zTrl%kS3v5)nYSG9yIdq2^1yMS4oWh*J7LMTs_ZMbEOs z;R4&aV)lQ=T1W{SRCn>fp41P)8T`k$g{K|iX{!}PIgjw1*NJNH_wjuRukPrAywHu# zzGxt0gd-p8)iJ(y%)ZQo>Lu*(WBCSDpVZz2;;~}xAksxo8!-OZ=tbEdAH6#^p!Z`D zJGpHuoRCXEIq9XHZ6W^<^H?SXZzBsz?g&p?t)S6PI}o6EY{(^|9h;MEeHz-a*aOvk z#6QxY9UIE2o71iY?MgZAI);J`MeI)x6imMm72>dsMGU^OIfLOWEVcK}AbnuBwjk4d zV1;c#7V~+jYwV_5kWtpolVJIVUS3$w$mudjY4QkK`8Lz5DL90 zbH#+BZ{zOdpIe9MOk(~F>`v}7L*o3W!i*jj_>P%z#|CEnMwA2z%e%CR5}14?LB855 z5#&pmlE z;ucBhP$FZJz%%H%~C+ z+O_0V_H}MI7NkKu4ORcTo0}DkKT^RCAo7XaVr2wpj-v*;fEc`cTBIOe9!BKmfXXxJ z|CNNUN7*NTO(#h}g(&;2&Yg{|^d@u=kkO1$1<~5rRIK!X3mu|c!Ar>gU7tv#ZX1aV zg7t0jdI(8?d_yOU8n4lbLTFi3&3O}LuRj0wc3*z%hf5nUfJyRNU=pJoX!Z63=2`!* zz5H)=xP|5Su&Jj9dKqYQ{$xq+hCz+>=iffRp#i#2chs4U-Ouo$By6!HL@axK-o8Fr z(ytfu7jk(6Qi9C}S%ABjB-!+n#A_e>qyOcrUmMOd#wFovjhF%i*TBXS&OQ4Q-lGX^ zp(|_bh3FLkf_%8r>}lPcDA2-$7T4K?wg=Eg4p30jEhEvU($6cPK~CKG+S>DmJs5ZH ziN-q9&-f)g!?qmfKB89M+KQ@Ou*d*Sg!ziZAuA7I;gYW5HHqjm+NOYCtg&1-+A4#JjfT zjj~QR@#R@oCm5?{u<7M`zho;un%KA3B&m_Axiq_n5Oc;Fi!uSm?!4#%ZZQVM@j$(2e46Umf-FH(_8K+9YhP-=B%8M3_~QS~W%x;on)G#S#l zA~p*G0WFK9EFyyQj#jo~#a8&)l&=F#5w2 zc!>rzD$^6*h9p?7QT)!8Mmpou<+|v~`VG|9d=b1^Lo)Sa7dANACNr9oP8s?GfLtrv zo-|d7fVvU=_8KUSbkQ%rHu}I0O@v0@Y@w?hW#ACJWJOhmmwtEAMfZ&z?Uu4nv6gb@ zha1i)8TbrA++cqcyF#Im@ZW|9k&;s1v`v*FC}8PTEUqxc9S9APQc{W@R7m|AjDqw>u%e4nAJYMeU{+ zj_?aWZO6D0rkzW*(&6SnGdgR)Far;Ks zr2P=l$!fN6ik&C@VUMnt>?PTk9fu!iUuu&5J%g+_(fuG23Zb|Fyc9?L#il;2e^s+( z!038TD~}SnQz0-GKqT7o=!_~Qr&M4N@7@gsk=F?`k!f1J4OVCGL{UvtaZe?~^waAH9Zyw;g) z46KF~&suAY_?iaTiWo@PlE+FBsvoR9iOieKBy4;Ror~>ok#^^#AX`T3Ta9&FtUqi6Z1-o2-y$t1x|gsB zMtVpNmyS$Ecz0v6UH_iYPRYh|?+kb#pC949WC_x8_NN6kHZO>rdJ`?})z zUswJzx0&p{8p6k+<|C++qL3>NDqzUZ}l4ir3xrSk2(X5~=XnIF4 z*o^Xjo;YM^0h!Vf;U#KJmIDN{B|t+T%KZ?Y-9 zFafH_a2w%9n~t5c_SP;|P#nNH=dA0^fq(!vl{&PGf=;ttxpopUOjRONii>lr|`wmNc#D`nzEwxOr0K9$?2F~53B%SX2@ z&DOz`H%v}hB*Ln|B+A|vguq}v0RFImR9-nU8M5TJgk%!SBu1fqPh#%J8f0D^E%<&p zy!Kubcb;?blp=IM9CX}ld*si&*I3Pn8vD{P!6*4P8sRP6r3`Ivas&NUoN=~t8A>tR zZjg$Yv7gy0o#kSzrwH4oV5dut=8%z?9C88|fUKReDNVxoeq`@{=M00qP`@B(k;#T% zHrO(p3+p0SSD;-Em$w@Ox?uwrknzDBbB2<<5E;Wk(`#eB8pTqYEc!vt^u!tA^YAF& z=7lx3W&NjvE5_WyIv3-386PpA>XS6kFyJleTkRyU25G;2uAE*5k)oU9mAl9{kO0b4 zrH=etNge5-`B1ka?|w&g+Z^moYMWU@l&=RO@)&JF3LctRdoRhPUMPD7p4>_9FpU1u%HaPZM#HoNtXpZ!dTduSz|vRe&V3_YR;A~y1#cfEvR z4f*s@#lR@DR`(aqi*L47!oq{WEhfW@B4FmZ@IGf z-CW?w>??`{pJev)JQlOf+)JOilTz3ro-C5HClojPeXxJ0oI z?_|EAg;%XWr!U9!HzT+gAGOdY7nWfZ021!N%k13h_^6T9)RqddYTot z;K>F+?Q9Nowy7cj*4Vz9AB5Kn0D@6vNfGRcVK z%UG^9oqj#CW5mf}Q93Y@%?!x3n5d+{MAm%7#^S6UDULRQkdnllA{E8rc6}oSePhm}UED~Do|DTZoh8H{l03vpEFims&Li$_grqWj$=nQf}mg7Jx6HWq93 z^Z+F!+xq8O8x2uK-66nqAs1kdCx$AXOD&ienOU_BMo5V-)SaT75nAyyw`P5mUV^o9OV?ooR)(#STa)l1o^@Kkq{FryO5sJQfPVMGsIM@Tu;Q<{!FqB+&dR;A0a zv@Y1C?AogdoAv;k?lg-8CQ)slHgC=9gljr6y3wXGSg<5s6$Z#&({@$mC-fEon|=h- zgEU2Aht0L3?BXxkA*-IiGF$3}h?b|c(_YR$I!*IR1ljYSM-XLObQ2{I0D-I1mMSFm zNLt}_)yrWzMVq>by0^7%8=jkD8#*BiwQ6i^k1d~CIEDt*V({ngl*&ee(EaPVv{?Bm03J8Fut-V9?tTdN> zWTvGSL~TXqi&?_&7-WkxQl)Iy%FI%xWoW8#4h|y%E8{I(=3b1>gdBH@5+ht3BqX6%p%>uq#hGunQMm&v!e;Wm>2Y3X%XY~$tB#EOvGU44srQI3Te zEr2N<(?W&p9%*H2>oCkUnCcGrREJIDyWzMuD2}hBLZ&M~U`paak|miHA}_X!m+5e0 z@R0$mJAiFCEYjX+Dh&dhSSqNgIPVPdbXegou#3(gIq5?Bu!~yRV2a8;|5&Q<; z*eWz?EQ8Uey$-xuABIg7Lzmc654PFe#~P_#8ydg`;$}G0REit+2WRrv(os8jsD`!c zI{$%;zNQt(G`lmrY0O^!`T(DBXhVDyNNJ1gzxL||W(Uet+o1zIS@E13P-Y4886R?> zUUQGUa1WsTM;NRbsg#|QkKjSRTBhYWy9JOPT%1~fP;lv}CI<9Jn|88(NnivSc%Rn_ z7g|I~vbB#`XK``s{X({X0HqU6^*RPdjAr|Vy59mrIK@KFl14bKVl5Ji&~?^}gmq&Jd_|Pn;O*UKaKf4mh@E=FvlbRoDH-&awhMHoLrI>?5}=%t zJXSZM!sk6mBvTuHg0rheCMlOdFJ;spt3+oz+IP;~pxXaf#e!j<`^9CQx z3I^RwHf%D3{@_3*?7mP94K4#cwaj2|tcB6Ww%agJ3`k6Mka(hl#1%uL%3GvG`?;PP zr>z};ZW0`YfdB57G3_AB)soSynp6N2zJWvy&aJ`K>SlvnG5Z8^i1n*2H?}GpTa8&l zf3`q2+`q8pa!WY!ha-gEIZ#o%Ii1r7oEv~8#QFGF+d$dIR!ER{>v<^!1t9KP)alq7 z4#=YYN{Eq*tt=gkAZu-zAq8H4f(jZstA0T5GxrRgTNRnkTWEC};Abjw?r{tb@PqE< z2#*Xv#*(i|bNlF_om}@x4%j5XV%jY41IshO&t1umu~X%D$qkAQyDa73S<2gJaUdIL z7yE5XJYy#%5|$bW+DIA=|g{qXX~{nQ8I# z;zuuO3?}zmTHcmN433-udch8XE#fcf=_kUIodrOXyd15#g*AidETT_}fGOLD?7O+- zQ10!PB&uqKasA|hAPYgrCED1qBJ3wng&0+c^vhZ(*k7>K2pJVDo;+Fjg>eg+<$S3k z&-5EItCq<@c%kOlhRm8}a>!lCTxTVt2kutOG3hb-1$u8{^#$-lE9l{?ITKJ@ow z?TdAM=B8P^HGi|LH(o1WlO{&jM#e_l&0E|P-L%EMH_sY#H?6gMW7aJpb6xjex9;l} zfU6yjye1tpp+ZADAVrE^j(1AsSW@~2@YLWesE?LM;!!S=a z@r>&Q^seM_;27;J?x5{S%GNl{SqMz8tuFM{Df9weglL-5F@VWM?z6$V6kei7ClnJV zh@_Y>7sZ4VC0b0_$F|~tP6(g8iY}Da; z5;0Bccy)l$G8WbP8vG7Hp#j4*ex(-x~U2n}T!O#8jy6VRn&mRqp3 z8)f@)Xeqx9r%r)OwxjlD*4>ar{3~PzU*K!-bQu@;|FYiR>EC|f5=ZICD5HUnzXi3sa_5BhPC<(*$ zvT-ur_hN%E69-2HH^7iUN~AtTSSh-IUZKMY{WL_`-sJHm4Sgk$==sSAd!mEj9!h~+ z&b&UWC2rz_#40GmqhCqn(VP)Q)9oASv&TSPirc4i*sR7Tkppngq@OWYK=i71(VD#mXn z1F`jt;T{twm1G?$VO@YVF{2MHg||-DI1eTlJ^Pzcth1H5l@;QSJS^*TUCm(*Qjv;;tkE%>>?VX#6V zCAAS9lc+HK=t=%}yHr=$i)NhIGB|X>sB^)o3jTw& zwgb74bU;9HjI>en~(9{}hvGvOHz%4RcF!!NIE<4+UWh?O3fGL7Vp@=ul`B;)m=cE}YsTrgLKs zj@}_Jho4B5WE*Nh`zcK^5^Uf%ahC&R%;`8l{LG1#&?_>{#+I_hH#%1K3y~eI0dH}L z^fn;;jbU1Rv{uoN8ixfoI44giKY|CenL63AX9Z5RwKF#mKaE3+r%XaL?iOnOyYkwq zNj||>troP!lVL9~z+V67r?1p;-uCy3tJpizH4W|I{eFq9c&0D@*YuMZwVB#EnlO>p zD(x!w@>$X^{@L!-M{(3P>D8~qf2LXWu}C7+%PeUsES7=goCq;Jl1q#F*{%+4JJD2? zAElK(BS9N|Ah~QN4z}lmTKY95Ri>c5@e`7yEX*loIt#LIxs3ddCS>-rh+{p1FQKP` zTO1R(FKG52{v%J_AR#c0IA@dnKaGqT*G6S)w zBg`2m?neEXSqXFWf^KLR1e3EcG)Zq%a(}$TPEggMljN4=Q8JtA%v+2c>?!R|S>lG- zCaJmkXTsMrrNNM)W8t@8tqb-9xru<{-ILVy;+S4r5lpf+Hd2kVD}2eOEPXQh^;pj(`Z;KXKuhheP$%IgMhl{ta@`Z+L)&l0xQ5R6?@RpE(EJIx1h;(rHt4V zv0M2OT*88GXpUmEWYh?tMQgw^pdv`>ma)z;54KheE*vl-Qc$7wiJ|mkEh9L3veJt3oKB^^A^N-O-O*0Xx z&;ayv$J7+FQN?W7`=TUCEq~W;-${&c-KB?4Oia%;r+jO{Pt46NOir|#esSWd=CX4P zF?~s?=$t#sM}J4{+qrw*Om=pa>d(cRzuwXxaFaq&0qVS3h*Q^wN49L;w*9spJ9mxV ze#e{c^!3JFp}_0q-%{&;@`YW-FN3B(=%G!8Pd}dlJ}I9npO5o#9%}M|{*EYs!+&(R zjE>ENjv2PA@agBn+Ks!9&v8DED7O9Spk!iB+)wfOM&by?I(OlP$Q>OYa%+F*Q_g+u z|0#mSFSxY2gChI+oZ{p3YgK7=mM;I=A6&ckH(uqp5Ol^!`uS*VY;5<~p0T}S`^NT< z9T+<}c4+MI?y=pwckkJ~clW;C`*$DMeQ@`o-G}##?b*F&&z`+|_U+lf=fIwWdk*b6 zymxHx?!9~V?%lg@@BY09_8#1OXz$^DWBYdR+p}-)zJ2@l?>n&X;J!oq4(}h^zkC0l z{d@QC+rNMRf&B;fAKHKTz}SJ^2lgD;dtl#z{Ra*lIC$XDfx`#K4(>j<=iuIh`ws3u zc;Mi{gNF_tJ~Vb{_n|$9_8!`IX#b%DhYlV(bm;J5P&`cYhXH+*pkA5&+Y zyLu#C?>-u?9|+f@pA6R*FNN#Ox9z(2{97}}-{p}>yFUM=ziZd#ul_0y-1;xNbn90~ z3V&NgdS%9CUnJ}9qjx#Cm8^HX_4^+`_P`;Py75n&ozoqUQ*_fm?d(f`@bP+MC+0aB ze%I~QS|Pn-x1RaPUpWi+*#ww=V({U)x#m+7 zbNrsg>sZWYc}`!pVv0dEZ=GT3B-LYc8+Mwfy`-%P%z-mzvAX`PRgfb4~Xi zQYL(>^^AM0xpZ=Rd3kza-cL2>r<+smy^GDIiIx&nkB5`ftrqF;S@6p%lTZ5-m=(XU zWnv<=CrDr-hi%ScW&E;ixOeLCUrk7jG?gu93R+<>01wS{j^i;c$AIUc7gOrxX5U^Q7R@jfIsZN4)b}Ps~kIso9)$?^s$`UVdPzspkCL^!zOK zC#D`)n5UBUFo1U7+wP@N1IW}Ft&N3GtSm#x zcP=cq{PeP)TKMpM=V4`jsW~zEv|*Y}O#yh>!$>N)@uh{?=Dc5=hGGwY_j_~Zw)v6w z0lH3vYHl2JKMOmpZ%ar9edzKpq~{{gW#YrjNJRJX@#dLUAcPV9!86TC@$TeAD{sC{ zO}M&p61j3&egie>o^gC({weqVXHGu3FgHDEMD}wF3yUTn^Ajg?u~4DMk&lxT^UwH& zMLH__Tjz5g&{Jc{RC9T9X?hWv1J@2)$0w$)&z4+GK0Q4*<=Y~uLT&Pxcy=VR+qR@& zSye^8-WEFYH-(58Z9lxQ0`P^&$(1Gd{+5Q;pktE@r%@hh*kenOe_>_W!{eruWrJo- zPN1ud&hFUCd=MnnpFX)b*Ay)Sw@xAg4A@cxdM+LYrFj@dZB3t;2J8|Fa+=cL8+a`s z6MngcB%M@8OwqcO%1cQ$C7HRBh2qN+Wz(`{_b7z$PfslSC!26%xk;aB=hY<+lPX||*O?dM+g-^{+uhkCu{Wa7!tFBtvg%83(*L$IY?hUY!T)Jz7G)un>4 z>L$KpV%~b$oJUPgU|)TpuwQ<9rDbNSJ1R`fAzo9@h{=mf3s1>V^%%b1R5XKuz*P6? ziTNqC;(f>~%F;Dr-?y^BSkOE(*=$aM99_X~POqGV%PaF!?k^hK^Q88OVPhN32XllK zW)Rl)^7L1t*tzwDPk|5q?Rtq%AD=QGe7JR-t#$mjb^Nz=e7W^OK0|zJe1`dq@Y%u# z4`6*8pY42Zw10{A zFVX%b+P_5mmuUYI?SFyxFVX%bJj%~Su1ftX^{dpc{%cOD`I1ZD;-thuNBsl?F(Z%+ z!2)W7iY=1mTTe6GLM_{9mZta4uguNe=bl2{oVG5dNA9`jp5^wxH^24G_e|b1c~h_- zUYfRse9hp!{H9AUCp%rg+qn~(;xOwg_}J!L+tW@r+)sYe-{+#yN(uuiB`B9VNV@4C zq-_7O_lZBgVpmae#qr&f-!7M5<;=JLV7_zq#Xdp(@!gwmX`L^`rt=3IT|r;25|a7u z&F>Vn8w9QMhu}G9UzL!|H~*0}w`-TaH~-s^k8r*2pWNE^A42V%eN{p--@W^ YbpB8a=j^KzlKIw6=F-B`Z*}hf11*>a&Hw-a literal 0 HcmV?d00001 From 18884d6025b4c56b20149365dbbdfedd7f94d9ee Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Thu, 11 Mar 2021 08:43:10 -0600 Subject: [PATCH 032/157] PR Feedback changes --- plugins/net_plugin/net_plugin.cpp | 63 +++++++++++-------------------- tests/CMakeLists.txt | 2 +- 2 files changed, 23 insertions(+), 42 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 8c4d13babcd..c8c47ce18f0 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -256,9 +256,9 @@ namespace eosio { bool use_socket_read_watermark = false; /** @} */ - mutable std::shared_mutex connections_mtx; + mutable std::shared_mutex connections_mtx; // protects both connections and security_group std::set< connection_ptr > connections; // todo: switch to a thread safe container to avoid big mutex over complete collection - std::unordered_multimap particiant_connections; + security_group_manager security_group; std::mutex connector_check_timer_mtx; unique_ptr connector_check_timer; @@ -339,13 +339,12 @@ namespace eosio { constexpr static uint16_t to_protocol_version(uint16_t v); connection_ptr find_connection(const string& host)const; // must call with held mutex - /** private security group management */ - security_group_manager security_group; /** \brief Update security group information * * Retrieves the security group information from the indicated block and * updates the cached security group information. If a participant is - * removed from the cached security group, the TLS connection is closed. + * removed from the cached security group, it is marked as non-participating + * and some messages are not sent. * * \param bs The block_state containing the security group update */ @@ -589,10 +588,6 @@ namespace eosio { class connection : public std::enable_shared_from_this { public: - /** @brief For non TLS connection setup - * - * @param endpoint The connection endpoint - */ explicit connection( string endpoint ); connection(); @@ -812,24 +807,15 @@ namespace eosio { // private: std::optional participant_name_; - std::atomic participating_{false}; + std::atomic participating_{true}; public: - /** @brief Set/change the participant name for the connection - * - * @param name The name of the participant - */ - void participant_name(chain::account_name name) { participant_name_ = name; } /** @brief Returns the optional name of the participant */ auto participant_name() { return participant_name_; } + /** @brief Set the participating status */ + void set_participating(bool status) { participating_.store(status, std::memory_order_relaxed); } /** returns true if connection is in the security group */ bool is_participating() const { return participating_.load(std::memory_order_relaxed); } - /** returns false if the connection is not in the security group */ - bool is_syncing() const { return !participating_.load(std::memory_order_relaxed); } - /** @brief Flag the tls connection as participating in block production */ - void now_participating() { participating_.store(true, std::memory_order_relaxed); } - /** @brief Flag the connection as only publishing sync information */ - void now_syncing() { participating_.store(false, std::memory_order_relaxed); } }; const string connection::unknown = ""; @@ -1478,7 +1464,7 @@ namespace eosio { verify_strand_in_this_thread( strand, __func__, __LINE__ ); // for tls connections, when the connection is not in the security group // certain message types will not be transmitted - if(participant_name_ && !participating_) { + if(!is_participating()) { const bool ignore = std::holds_alternative(m) || std::holds_alternative(m) || std::holds_alternative(m) || @@ -1499,6 +1485,9 @@ namespace eosio { } void connection::enqueue_block( const signed_block_ptr& b, bool to_sync_queue) { + if(!is_participating()) { + return; + } fc_dlog( logger, "enqueue block ${num}", ("num", b->block_num()) ); verify_strand_in_this_thread( strand, __func__, __LINE__ ); @@ -3484,8 +3473,10 @@ namespace eosio { // called from any thread void net_plugin_impl::update_security_group(const block_state_ptr& bs) { // update cache + // - connection_mtx is needed // auto& update = bs->get_security_group_info(); + std::shared_lock connection_guard(connections_mtx); if(!security_group.update_cache(update.version, update.participants)) { return; } @@ -3493,29 +3484,29 @@ namespace eosio { std::vector added_connections; added_connections.reserve(connections.size()); - // update connection + // update connections + // - connection_mtx still needed // - auto do_update = [&](auto& connection) { + for(auto& connection : connections) { const auto& participant = connection->participant_name(); if(!participant) { - return true; + return; } if(security_group.is_in_security_group(participant.value())) { - if(connection->is_syncing()) { - connection->now_participating(); + if(!connection->is_participating()) { + connection->set_participating(true); added_connections.push_back(connection); } } else { - connection->now_syncing(); + connection->set_participating(false); } - return true; }; - for_each_connection(do_update); - // send handshake when added to group + // - connection mutex no longer needd // + connection_guard.unlock(); for(auto& connection : added_connections) { connection->send_handshake(); } @@ -4013,16 +4004,6 @@ namespace eosio { if( my->find_connection( host ) ) return "already connected"; - /** @todo when making a shared tls connection, need to use the ctor that takes - * requires the participant name. - * connection_ptr c = std::make_shared(host, nam); - * - * @todo Need to add the connection to the - * unordered_multimap. - * - * participant_connections.emplace(name, c); - * - */ connection_ptr c = std::make_shared( host ); fc_dlog( logger, "calling active connector: ${h}", ("h", host) ); if( c->resolve_and_connect() ) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 734a4d8528e..0258d4ee8ef 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,7 +5,7 @@ include_directories( "${CMAKE_SOURCE_DIR}/plugins/wallet_plugin/include" ) file(GLOB UNIT_TESTS "*.cpp") add_executable( plugin_test ${UNIT_TESTS} ) -target_link_libraries( plugin_test eosio_testing eosio_chain chainbase chain_plugin wallet_plugin fc net_plugin ${PLATFORM_SPECIFIC_LIBS} ) +target_link_libraries( plugin_test eosio_testing eosio_chain chainbase chain_plugin wallet_plugin fc ${PLATFORM_SPECIFIC_LIBS} ) target_include_directories( plugin_test PUBLIC ${CMAKE_SOURCE_DIR}/plugins/net_plugin/include From 66e43ad3e02c1630b348aa14f20657b4bd807ef9 Mon Sep 17 00:00:00 2001 From: Victor Camacho Date: Thu, 11 Mar 2021 12:26:55 -0500 Subject: [PATCH 033/157] delete extension_v1 struct from snapshot_global_property_object --- .../eosio/chain/global_property_object.hpp | 42 ++++++------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index 1a64b84d92f..1c58e588233 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -128,31 +128,23 @@ namespace eosio { namespace chain { struct extension_v0 { // libstdc++ requires the following two constructors to work. extension_v0(){}; - extension_v0(block_num_type num, const flat_set& participants) + extension_v0(block_num_type num, const flat_set& participants, const vector trx_hooks) : proposed_security_group_block_num(num) - , proposed_security_group_participants(participants) {} - - block_num_type proposed_security_group_block_num = 0; - flat_set proposed_security_group_participants; - }; - - struct extension_v1 : extension_v0 { - // libstdc++ requires the following two constructors to work. - extension_v1(){}; - extension_v1(block_num_type num, const flat_set& participants, const vector& trx_hooks) - : extension_v0(num, participants), - transaction_hooks(trx_hooks) {} + , proposed_security_group_participants(participants) + , transaction_hooks(trx_hooks) {} + block_num_type proposed_security_group_block_num = 0; + flat_set proposed_security_group_participants; vector transaction_hooks; }; // for future extensions, please use the following pattern: // - // struct extension_v2 : extension_v1 { new_field_t new_field; }; - // using extension_t = std::variant; + // struct extension_v1 : extension_v0 { new_field_t new_field; }; + // using extension_t = std::variant; // - using extension_t = std::variant; + using extension_t = std::variant; extension_t extension; }; @@ -163,8 +155,7 @@ namespace eosio { namespace chain { using snapshot_type = snapshot_global_property_object; static_assert(std::is_same_v>, + std::variant>, "Please update to_snapshot_row()/from_snapshot_row() accordingly when " "snapshot_global_property_object::extension_t is changed"); @@ -176,7 +167,7 @@ namespace eosio { namespace chain { value.chain_id, value.kv_configuration, value.wasm_configuration, - snapshot_global_property_object::extension_v1{value.proposed_security_group_block_num, + snapshot_global_property_object::extension_v0{value.proposed_security_group_block_num, value.proposed_security_group_participants, value.transaction_hooks }}; @@ -195,9 +186,7 @@ namespace eosio { namespace chain { [&value](auto& ext) { value.proposed_security_group_block_num = ext.proposed_security_group_block_num; value.proposed_security_group_participants = std::move(ext.proposed_security_group_participants); - if constexpr( std::is_base_of::type>::value ) { - value.transaction_hooks = std::move(ext.transaction_hooks); - } + value.transaction_hooks = std::move(ext.transaction_hooks); }, row.extension); } @@ -235,8 +224,7 @@ CHAINBASE_SET_INDEX_TYPE(eosio::chain::dynamic_global_property_object, FC_REFLECT(eosio::chain::global_property_object, (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id)(kv_configuration)(wasm_configuration) - (proposed_security_group_block_num)(proposed_security_group_participants) - (transaction_hooks) + (proposed_security_group_block_num)(proposed_security_group_participants)(transaction_hooks) ) FC_REFLECT(eosio::chain::legacy::snapshot_global_property_object_v2, @@ -252,11 +240,7 @@ FC_REFLECT(eosio::chain::legacy::snapshot_global_property_object_v4, ) FC_REFLECT(eosio::chain::snapshot_global_property_object::extension_v0, - (proposed_security_group_block_num)(proposed_security_group_participants) - ) - -FC_REFLECT_DERIVED(eosio::chain::snapshot_global_property_object::extension_v1, - (eosio::chain::snapshot_global_property_object::extension_v0), (transaction_hooks) + (proposed_security_group_block_num)(proposed_security_group_participants)(transaction_hooks) ) FC_REFLECT(eosio::chain::snapshot_global_property_object, From c9aca3428c8618309b16a089759a247475d73326 Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Thu, 11 Mar 2021 13:06:28 -0600 Subject: [PATCH 034/157] PR Feedback for net_plugin security group message handling 1) security_group_manager::is_participating now returns true on empty cache 2) is_participating test has been added to enqueue_buffer access control 3) using lock_guard instead of shared_guard --- .../net_plugin/security_group_manager.hpp | 4 ++- plugins/net_plugin/net_plugin.cpp | 25 ++++++------------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp b/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp index 0bae90df7af..b29885ad67c 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp @@ -18,7 +18,9 @@ namespace eosio { */ bool update_cache(const uint32_t version, const participant_list_t& participant_list); /** @brief Determine if a participant is in the security group */ - bool is_in_security_group(chain::account_name participant) const { return cache_.find(participant) != cache_.end(); } + bool is_in_security_group(chain::account_name participant) const { + return cache_.empty() || (cache_.find(participant) != cache_.end()); + } private: uint32_t version_ {0}; ///! The security group version participant_list_t cache_; ///! Cache of participants in the security group diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index c8c47ce18f0..0bc8d2a4b06 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2237,6 +2237,10 @@ namespace eosio { return true; } + if(!cp->is_participating()) { + return true; + } + cp->strand.post( [this, cp, id, bnum, sb{std::move(sb)}]() { std::unique_lock g_conn( cp->conn_mtx ); bool has_block = cp->last_handshake_recv.last_irreversible_block_num >= bnum; @@ -2282,7 +2286,7 @@ namespace eosio { trx_buffer_factory buff_factory; for_each_connection( [this, &trx, &nts, &buff_factory]( auto& cp ) { - if( cp->is_blocks_only_connection() || !cp->current() ) { + if( cp->is_blocks_only_connection() || !cp->current() || !cp->is_participating() ) { return true; } nts.connection_id = cp->connection_id; @@ -3473,43 +3477,30 @@ namespace eosio { // called from any thread void net_plugin_impl::update_security_group(const block_state_ptr& bs) { // update cache - // - connection_mtx is needed // auto& update = bs->get_security_group_info(); - std::shared_lock connection_guard(connections_mtx); + std::lock_guard connection_guard(connections_mtx); if(!security_group.update_cache(update.version, update.participants)) { return; } - std::vector added_connections; - added_connections.reserve(connections.size()); - // update connections - // - connection_mtx still needed // for(auto& connection : connections) { const auto& participant = connection->participant_name(); if(!participant) { - return; + continue; } if(security_group.is_in_security_group(participant.value())) { if(!connection->is_participating()) { connection->set_participating(true); - added_connections.push_back(connection); + connection->send_handshake(); } } else { connection->set_participating(false); } }; - - // send handshake when added to group - // - connection mutex no longer needd - // - connection_guard.unlock(); - for(auto& connection : added_connections) { - connection->send_handshake(); - } } // called from application thread From d1bc614a29049c0410a42d4aca566c0edad87018 Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Thu, 11 Mar 2021 14:41:08 -0600 Subject: [PATCH 035/157] Fix security_group_manager unit test security_group_manager was changed to return true for a participant when the cache is empty. --- plugins/net_plugin/test/security_group_manager_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/net_plugin/test/security_group_manager_tests.cpp b/plugins/net_plugin/test/security_group_manager_tests.cpp index 2a9d813d5e4..a95f84a312b 100644 --- a/plugins/net_plugin/test/security_group_manager_tests.cpp +++ b/plugins/net_plugin/test/security_group_manager_tests.cpp @@ -46,7 +46,7 @@ BOOST_AUTO_TEST_CASE(test_remove_all) { BOOST_REQUIRE(manager.update_cache(2, clear)); for(auto participant : populate) { - BOOST_REQUIRE(!manager.is_in_security_group(participant)); + BOOST_REQUIRE(manager.is_in_security_group(participant)); } } From 23da2a8efebe8f1e3cba28b74a35597f7ed82c12 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 11 Mar 2021 17:50:52 -0600 Subject: [PATCH 036/157] Changed --help to display and exit and changed range of files names to start at 0. --- tests/generate-certificates.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/generate-certificates.sh b/tests/generate-certificates.sh index 5762a30aa49..21dbdf2c18a 100755 --- a/tests/generate-certificates.sh +++ b/tests/generate-certificates.sh @@ -41,6 +41,7 @@ then echo "--org-mask: Paritipant certificates name mask in format of name{number}" echo "--cn-mask: Paritipant certificates common name mask in format of name{number}" echo "--group-size: Number of participants signed by generated CA" + exit 0 fi #default arguments: @@ -68,7 +69,7 @@ echo " generating nodes certificates " echo "*************************************************" #client certificate requests + private keys -for n in $(seq 1 $GROUP_SIZE) +for n in $(seq 0 $(($GROUP_SIZE-1))) do ORG_NAME=$(sed "s/{NUMBER}/$n/" <<< "$ORG_MASK") CN_NAME=$(sed "s/{NUMBER}/$n/" <<< "$CN_MASK") From 32648b0519b0292a1b1f38e4ec28525a44ab55b2 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Thu, 11 Mar 2021 21:57:49 -0500 Subject: [PATCH 037/157] changing certificate generation script naming --- tests/generate-certificates.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generate-certificates.sh b/tests/generate-certificates.sh index 5762a30aa49..d2e31257e32 100755 --- a/tests/generate-certificates.sh +++ b/tests/generate-certificates.sh @@ -68,7 +68,7 @@ echo " generating nodes certificates " echo "*************************************************" #client certificate requests + private keys -for n in $(seq 1 $GROUP_SIZE) +for n in $(seq 0 $(($GROUP_SIZE-1)) ) do ORG_NAME=$(sed "s/{NUMBER}/$n/" <<< "$ORG_MASK") CN_NAME=$(sed "s/{NUMBER}/$n/" <<< "$CN_MASK") From 430bc65411e9ed94f8e1e8c190f43243ad1ec6e4 Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Fri, 12 Mar 2021 07:05:20 -0600 Subject: [PATCH 038/157] PR Feedback for net_plugin security group message handling 1) Provided security_group_manager::current_version() 2) Added unit test to verify new method 3) net_plugin_impl::update_security_group() now checks version prior to taking lock. 4) dispatch_manager::bcast_block() now checks connection participating flag before creating send_buffer --- .../eosio/net_plugin/security_group_manager.hpp | 2 ++ plugins/net_plugin/net_plugin.cpp | 13 ++++++++----- .../test/security_group_manager_tests.cpp | 10 ++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp b/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp index b29885ad67c..79e89f1ab3a 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp @@ -10,6 +10,8 @@ namespace eosio { class security_group_manager { public: using participant_list_t = boost::container::flat_set; + /** @brief Provides the current security group version */ + auto current_version() const { return version_; } /** @brief Update the security group participants * * @param version The version number for this update diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 0bc8d2a4b06..5d942289370 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2227,7 +2227,7 @@ namespace eosio { for_each_block_connection( [this, &id, &bnum, &b, &buff_factory]( auto& cp ) { peer_dlog( cp, "socket_is_open ${s}, connecting ${c}, syncing ${ss}", ("s", cp->socket_is_open())("c", cp->connecting.load())("ss", cp->syncing.load()) ); - if( !cp->current() ) return true; + if( !cp->current() || !cp->is_participating() ) return true; send_buffer_type sb = buff_factory.get_send_buffer( b, cp->protocol_version.load() ); if( !sb ) { peer_wlog( cp, "Sending go away for incomplete block #${n} ${id}...", @@ -2237,10 +2237,6 @@ namespace eosio { return true; } - if(!cp->is_participating()) { - return true; - } - cp->strand.post( [this, cp, id, bnum, sb{std::move(sb)}]() { std::unique_lock g_conn( cp->conn_mtx ); bool has_block = cp->last_handshake_recv.last_irreversible_block_num >= bnum; @@ -3478,7 +3474,14 @@ namespace eosio { void net_plugin_impl::update_security_group(const block_state_ptr& bs) { // update cache // + // Check the version before taking the lock. Since this is the only thread that + // touches the version information, no need to take the lock if the version is + // unchanged. + // auto& update = bs->get_security_group_info(); + if(security_group.current_version() == update.version) { + return; + } std::lock_guard connection_guard(connections_mtx); if(!security_group.update_cache(update.version, update.participants)) { return; diff --git a/plugins/net_plugin/test/security_group_manager_tests.cpp b/plugins/net_plugin/test/security_group_manager_tests.cpp index a95f84a312b..db2386532e8 100644 --- a/plugins/net_plugin/test/security_group_manager_tests.cpp +++ b/plugins/net_plugin/test/security_group_manager_tests.cpp @@ -37,6 +37,16 @@ BOOST_AUTO_TEST_CASE(test_initial_population) { BOOST_REQUIRE(manager.is_in_security_group(participant)); } } + +BOOST_AUTO_TEST_CASE(test_version) { + eosio::security_group_manager manager; + BOOST_REQUIRE(manager.current_version() == 0); + + auto populate = create_list({ 1, 2, 3, 4, 5, 6}); + BOOST_REQUIRE(manager.update_cache(1, populate)); + BOOST_REQUIRE(manager.current_version() == 1); + } + BOOST_AUTO_TEST_CASE(test_remove_all) { auto populate = create_list({1, 2, 3, 4, 5, 6}); eosio::security_group_manager manager; From ae8921a2b448f7829bf97ba70e8e5d507d4cf293 Mon Sep 17 00:00:00 2001 From: Rusty Fleming Date: Fri, 12 Mar 2021 10:51:30 -0600 Subject: [PATCH 039/157] PR Feedback for net_plugin security group message handling 1) Fixed indentation issue --- plugins/net_plugin/test/security_group_manager_tests.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/net_plugin/test/security_group_manager_tests.cpp b/plugins/net_plugin/test/security_group_manager_tests.cpp index db2386532e8..7c60fb3a625 100644 --- a/plugins/net_plugin/test/security_group_manager_tests.cpp +++ b/plugins/net_plugin/test/security_group_manager_tests.cpp @@ -42,10 +42,10 @@ BOOST_AUTO_TEST_CASE(test_version) { eosio::security_group_manager manager; BOOST_REQUIRE(manager.current_version() == 0); - auto populate = create_list({ 1, 2, 3, 4, 5, 6}); - BOOST_REQUIRE(manager.update_cache(1, populate)); - BOOST_REQUIRE(manager.current_version() == 1); - } + auto populate = create_list({ 1, 2, 3, 4, 5, 6}); + BOOST_REQUIRE(manager.update_cache(1, populate)); + BOOST_REQUIRE(manager.current_version() == 1); +} BOOST_AUTO_TEST_CASE(test_remove_all) { auto populate = create_list({1, 2, 3, 4, 5, 6}); From bfdde961fdec1b85a86047e89693636e140f3972 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 12 Mar 2021 14:19:51 -0600 Subject: [PATCH 040/157] Added a flag to configure the cluster for security groups. --- tests/Cluster.py | 53 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index bfd7e0e0bb2..a3b235b0fb0 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -166,7 +166,8 @@ def setAlternateVersionLabels(self, file): # pylint: disable=too-many-statements def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="mesh", delay=1, onlyBios=False, dontBootstrap=False, totalProducers=None, sharedProducers=0, extraNodeosArgs="", useBiosBootFile=True, specificExtraNodeosArgs=None, onlySetProds=False, - pfSetupPolicy=PFSetupPolicy.FULL, alternateVersionLabelsFile=None, associatedNodeLabels=None, loadSystemContract=True, manualProducerNodeConf={}): + pfSetupPolicy=PFSetupPolicy.FULL, alternateVersionLabelsFile=None, associatedNodeLabels=None, loadSystemContract=True, manualProducerNodeConf={}, + configSecurityGroup=False): """Launch cluster. pnodes: producer nodes count unstartedNodes: non-producer nodes that are configured into the launch, but not started. Should be included in totalNodes. @@ -189,6 +190,7 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me associatedNodeLabels: Supply a dictionary of node numbers to use an alternate label for a specific node. loadSystemContract: indicate whether the eosio.system contract should be loaded (setting this to False causes useBiosBootFile to be treated as False) manualProducerNodeConf: additional producer public keys which is not automatically generated by launcher + configSecurityGroup: configure the network for TLS and setup a certificate authority so the security group can be used """ assert(isinstance(topo, str)) assert PFSetupPolicy.isValid(pfSetupPolicy) @@ -214,6 +216,50 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me if pnodes + unstartedNodes > totalNodes: raise RuntimeError("totalNodes (%d) must be equal to or greater than pnodes(%d) + unstartedNodes(%d)." % (totalNodes, pnodes, unstartedNodes)) + def insertSpecificExtraNodeosArgs(node, insertStr): + arg = specificExtraNodeosArgs.get(node, "") + specificExtraNodeosArgs[node] = arg + " " + insertStr + + if configSecurityGroup: + privacyDir=os.path.join(Utils.ConfigDir, "privacy") + if not os.path.isdir(privacyDir): + if Utils.Debug: Utils.Print("creating dir %s in dir: %s" % (privacyDir, os.getcwd())) + os.mkdir(privacyDir) + original=os.getcwd() + os.chdir(privacyDir) + if Utils.Debug: Utils.Print("change to dir: %s" % (os.getcwd())) + genCertScript=os.path.join(original, "tests", "generate-certificates.sh") + totalNodesInNetwork = totalNodes + 1 # account for bios node + cmd = "{} --days 1 --CA-org Block.one --CA-CN test-domain --org-mask node{{NUMBER}} --cn-mask test-domain{{NUMBER}} --group-size {}".format(genCertScript, totalNodesInNetwork) + rtn=Utils.runCmdReturnStr(cmd, silentErrors=False) + with open("generate.log", 'w') as f: + f.write("executed cmd: {}".format(cmd)) + f.write("=========================") + f.write("OUTPUT") + f.write("=========================") + f.write("{}".format(rtn)) + os.chdir(original) + if Utils.Debug: Utils.Print("changed back to dir: %s" % (os.getcwd())) + if specificExtraNodeosArgs is None: + specificExtraNodeosArgs = {} + + certAuth = os.path.join(Utils.ConfigDir, "CA_cert.pem") + def getArguments(number): + nodeCert = os.path.join(Utils.ConfigDir, "node{}.crt".format(number)) + nodeKey = os.path.join(Utils.ConfigDir, "node{}_key.pem".format(number)) + return "--p2p-tls-own-certificate-file {} --p2p-tls-private-key-file {} --p2p-tls-ca-certificate-file {}".format(nodeCert, nodeKey, certAuth) + + for node in range(totalNodes): + arguments = getArguments(node) + if Utils.Debug: Utils.Print("adding arguments: {}".format(arguments)) + insertSpecificExtraNodeosArgs(node, arguments) + + arguments = getArguments(totalNodes + 1) + if Utils.Debug: Utils.Print("adding arguments: {}".format(arguments)) + biosNodeNum = -1 + insertSpecificExtraNodeosArgs(biosNodeNum, arguments) + + if self.walletMgr is None: self.walletMgr=WalletMgr(True) @@ -254,12 +300,11 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me specificExtraNodeosArgs = {} for node, conf in manualProducerNodeConf.items(): - arg = specificExtraNodeosArgs.get(node, "") account = conf['key'] - arg = arg + " --plugin eosio::producer_plugin --signature-provider {}=KEY:{} ".format(account.ownerPublicKey, account.ownerPrivateKey) + arg = "--plugin eosio::producer_plugin --signature-provider {}=KEY:{} ".format(account.ownerPublicKey, account.ownerPrivateKey) for name in conf['names']: arg = arg + "--producer-name {} ".format(name) - specificExtraNodeosArgs[node] = arg + insertSpecificExtraNodeosArgs(node, arg) Utils.Print("specificExtraNodeosArgs=", specificExtraNodeosArgs) From 0029b507aff6736bc8797d153fcb1f61521ef31f Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Sat, 13 Mar 2021 21:08:09 -0500 Subject: [PATCH 041/157] added ec option to test certificates generation --- tests/generate-certificates.sh | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/generate-certificates.sh b/tests/generate-certificates.sh index d2e31257e32..df135ca0842 100755 --- a/tests/generate-certificates.sh +++ b/tests/generate-certificates.sh @@ -27,11 +27,27 @@ do GROUP_SIZE=${2} shift ;; + --use-EC|-e) + USE_EC=1 + ;; + --use-RSA|-r) + USE_EC=0 + ;; esac shift done } +function get-algo-str { + #1 means elliptic curve. for elliptic curve we need parameter generated file + if [[ $1 == 1 ]] + then + echo "ec:ECPARAM.pem" + else + echo "rsa:2048" + fi +} + if [[ $1 == "--help" ]] then echo "Usage:" @@ -41,10 +57,13 @@ then echo "--org-mask: Paritipant certificates name mask in format of name{number}" echo "--cn-mask: Paritipant certificates common name mask in format of name{number}" echo "--group-size: Number of participants signed by generated CA" + echo "--use-EC: Use EC algorithm. Enabled by default." + echo "--use-RSA: Use RSA algorithm. Default is EC" fi #default arguments: DAYS=1 +USE_EC=1 CA_ORG="Block.one" CA_CN="test-domain" ORG_MASK="node{NUMBER}" @@ -54,12 +73,16 @@ GROUP_SIZE=4 #overrides default is set parse-args "${@}" +if [[ $USE_EC == 1 ]] +then + openssl genpkey -genparam -algorithm ec -pkeyopt ec_paramgen_curve:P-384 -out ECPARAM.pem +fi + echo "*************************************************" echo " generating CA_cert.pem " echo "*************************************************" -openssl req -newkey rsa:2048 -nodes -keyout CA_key.pem -x509 -days ${DAYS} -out CA_cert.pem -subj "/C=US/ST=VA/L=Blocksburg/O=${CA_ORG}/CN=${CA_CN}" - +openssl req -newkey $(get-algo-str $USE_EC) -nodes -keyout CA_key.pem -x509 -days ${DAYS} -out CA_cert.pem -subj "/C=US/ST=VA/L=Blocksburg/O=${CA_ORG}/CN=${CA_CN}" echo "*************************************************" openssl x509 -in CA_cert.pem -text -noout @@ -75,7 +98,7 @@ do echo "*************************************************" echo "generating certificate for $ORG_NAME / $CN_NAME " echo "*************************************************" - openssl req -newkey rsa:2048 -nodes -keyout "${ORG_NAME}_key.pem" -out "${ORG_NAME}.csr" -subj "/C=US/ST=VA/L=Blockburg/O=${ORG_NAME}/CN=${CN_NAME}" + openssl req -newkey $(get-algo-str $USE_EC) -nodes -keyout "${ORG_NAME}_key.pem" -out "${ORG_NAME}.csr" -subj "/C=US/ST=VA/L=Blockburg/O=${ORG_NAME}/CN=${CN_NAME}" openssl x509 -req -in "${ORG_NAME}.csr" -CA CA_cert.pem -CAkey CA_key.pem -CAcreateserial -out "${ORG_NAME}.crt" -days ${DAYS} -sha256 echo "*************************************************" openssl x509 -in "${ORG_NAME}.crt" -text -noout From b879e11e9d606c3da99f291610649e6a98fee136 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 15 Mar 2021 18:40:09 -0500 Subject: [PATCH 042/157] Changed eosio-launcher to treat bios as -1 for spec*-num parameters. --- programs/eosio-launcher/main.cpp | 47 +++++++++++--------------------- 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/programs/eosio-launcher/main.cpp b/programs/eosio-launcher/main.cpp index 111cb246176..877bafc01ea 100644 --- a/programs/eosio-launcher/main.cpp +++ b/programs/eosio-launcher/main.cpp @@ -46,6 +46,7 @@ using namespace eosio::launcher::config; const string block_dir = "blocks"; const string shared_mem_dir = "state"; +const int bios_node_num = -1; struct local_identity { vector addrs; @@ -403,8 +404,8 @@ struct launcher_def { bfs::path data_dir_base; bool skip_transaction_signatures = false; string eosd_extra_args; - std::map specific_nodeos_args; - std::map specific_nodeos_installation_paths; + std::map specific_nodeos_args; + std::map specific_nodeos_installation_paths; testnet_def network; string gelf_endpoint; vector aliases; @@ -486,9 +487,9 @@ launcher_def::set_options (bpo::options_description &cfg) { ("genesis,g",bpo::value()->default_value("./genesis.json"),"set the path to genesis.json") ("skip-signature", bpo::bool_switch(&skip_transaction_signatures)->default_value(false), (string(node_executable_name) + " does not require transaction signatures.").c_str()) (node_executable_name, bpo::value(&eosd_extra_args), ("forward " + string(node_executable_name) + " command line argument(s) to each instance of " + string(node_executable_name) + ", enclose arg(s) in quotes").c_str()) - ("specific-num", bpo::value>()->composing(), ("forward " + string(node_executable_name) + " command line argument(s) (using \"--specific-" + string(node_executable_name) + "\" flag) to this specific instance of " + string(node_executable_name) + ". This parameter can be entered multiple times and requires a paired \"--specific-" + string(node_executable_name) +"\" flag each time it is used").c_str()) + ("specific-num", bpo::value>()->composing(), ("forward " + string(node_executable_name) + " command line argument(s) (using \"--specific-" + string(node_executable_name) + "\" flag) to this specific instance of " + string(node_executable_name) + ". This parameter can be entered multiple times and requires a paired \"--specific-" + string(node_executable_name) +"\" flag each time it is used. Use -1 for bios node.").c_str()) (("specific-" + string(node_executable_name)).c_str(), bpo::value>()->composing(), ("forward " + string(node_executable_name) + " command line argument(s) to its paired specific instance of " + string(node_executable_name) + "(using \"--specific-num\"), enclose arg(s) in quotes").c_str()) - ("spcfc-inst-num", bpo::value>()->composing(), ("Specify a specific version installation path (using \"--spcfc-inst-"+ string(node_executable_name) + "\" flag) for launching this specific instance of " + string(node_executable_name) + ". This parameter can be entered multiple times and requires a paired \"--spcfc-inst-" + string(node_executable_name) + "\" flag each time it is used").c_str()) + ("spcfc-inst-num", bpo::value>()->composing(), ("Specify a specific version installation path (using \"--spcfc-inst-"+ string(node_executable_name) + "\" flag) for launching this specific instance of " + string(node_executable_name) + ". This parameter can be entered multiple times and requires a paired \"--spcfc-inst-" + string(node_executable_name) + "\" flag each time it is used. Use -1 for bios node.").c_str()) (("spcfc-inst-" + string(node_executable_name)).c_str(), bpo::value>()->composing(), ("Provide a specific version installation path to its paired specific instance of " + string(node_executable_name) + "(using \"--spcfc-inst-num\")").c_str()) ("delay,d",bpo::value(&start_delay)->default_value(0),"seconds delay before starting each node after the first") ("boot",bpo::bool_switch(&boot)->default_value(false),"After deploying the nodes and generating a boot script, invoke it.") @@ -514,9 +515,9 @@ inline enum_type& operator|=(enum_type&lhs, const enum_type& rhs) } template -void retrieve_paired_array_parameters (const variables_map &vmap, const std::string& num_selector, const std::string& paired_selector, std::map& selector_map) { +void retrieve_paired_array_parameters (const variables_map &vmap, const std::string& num_selector, const std::string& paired_selector, std::map& selector_map) { if (vmap.count(num_selector)) { - const auto specific_nums = vmap[num_selector].as>(); + const auto specific_nums = vmap[num_selector].as>(); const auto specific_args = vmap[paired_selector].as>(); if (specific_nums.size() != specific_args.size()) { cerr << "ERROR: every " << num_selector << " argument must be paired with a " << paired_selector << " argument" << endl; @@ -526,10 +527,15 @@ void retrieve_paired_array_parameters (const variables_map &vmap, const std::str for(uint i = 0; i < specific_nums.size(); ++i) { const auto& num = specific_nums[i]; - if (num >= total_nodes) { + if (num >= static_cast(total_nodes)) { cerr << "\"--" << num_selector << "\" provided value= " << num << " is higher than \"--nodes\" provided value=" << total_nodes << endl; exit (-1); } + else if (num < bios_node_num) { + cerr << "\"--" << num_selector << "\" provided value= " << num << " is negative, and the only negative value allowed is " + << bios_node_num << "(bios indicator)" << endl; + exit (-1); + } selector_map[num] = specific_args[i]; } } @@ -623,26 +629,6 @@ launcher_def::initialize (const variables_map &vmap) { exit (-1); } - if (vmap.count("specific-num")) { - const auto specific_nums = vmap["specific-num"].as>(); - const auto specific_args = vmap["specific-" + string(node_executable_name)].as>(); - if (specific_nums.size() != specific_args.size()) { - cerr << "ERROR: every specific-num argument must be paired with a specific-" << node_executable_name << " argument" << endl; - exit (-1); - } - // don't include bios - const auto allowed_nums = total_nodes - 1; - for(uint i = 0; i < specific_nums.size(); ++i) - { - const auto& num = specific_nums[i]; - if (num >= allowed_nums) { - cerr << "\"--specific-num\" provided value= " << num << " is higher than \"--nodes\" provided value=" << total_nodes << endl; - exit (-1); - } - specific_nodeos_args[num] = specific_args[i]; - } - } - char* erd_env_var = getenv ("EOSIO_HOME"); if (erd_env_var == nullptr || std::string(erd_env_var).empty()) { erd_env_var = getenv ("PWD"); @@ -1517,9 +1503,9 @@ launcher_def::launch (eosd_def &instance, string >s) { node_rt_info info; info.remote = !host->is_local(); + const auto node_num = (instance.name == "bios") ? -1 : boost::lexical_cast(instance.get_node_num()); string install_path; - if (instance.name != "bios" && !specific_nodeos_installation_paths.empty()) { - const auto node_num = boost::lexical_cast(instance.get_node_num()); + if (!specific_nodeos_installation_paths.empty()) { if (specific_nodeos_installation_paths.count(node_num)) { install_path = specific_nodeos_installation_paths[node_num] + "/"; } @@ -1531,8 +1517,7 @@ launcher_def::launch (eosd_def &instance, string >s) { if (!eosd_extra_args.empty()) { eosdcmd += eosd_extra_args + " "; } - if (instance.name != "bios" && !specific_nodeos_args.empty()) { - const auto node_num = boost::lexical_cast(instance.get_node_num()); + if (!specific_nodeos_args.empty()) { if (specific_nodeos_args.count(node_num)) { eosdcmd += specific_nodeos_args[node_num] + " "; } From 36d1f9c0b3503d366aaf78993fe42b07bf4fb252 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 16 Mar 2021 08:38:36 -0500 Subject: [PATCH 043/157] Added providing node mapping to topo parameter to select the exact connections between nodes. --- tests/Cluster.py | 69 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index a3b235b0fb0..50c5e919e2f 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -192,7 +192,7 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me manualProducerNodeConf: additional producer public keys which is not automatically generated by launcher configSecurityGroup: configure the network for TLS and setup a certificate authority so the security group can be used """ - assert(isinstance(topo, str)) + assert(isinstance(topo, (str,dict))) assert PFSetupPolicy.isValid(pfSetupPolicy) if alternateVersionLabelsFile is not None: assert(isinstance(alternateVersionLabelsFile, str)) @@ -349,25 +349,63 @@ def getArguments(number): cmdArr.append("--spcfc-inst-nodeos") cmdArr.append(path) - # must be last cmdArr.append before subprocess.call, so that everything is on the command line - # before constructing the shape.json file for "bridge" - if topo=="bridge": - shapeFilePrefix="shape_bridge" - shapeFile=shapeFilePrefix+".json" + def createDefaultShapeFile(newFile, cmdArr): cmdArrForOutput=copy.deepcopy(cmdArr) cmdArrForOutput.append("--output") - cmdArrForOutput.append(shapeFile) + cmdArrForOutput.append(newFile) s=" ".join(cmdArrForOutput) if Utils.Debug: Utils.Print("cmd: %s" % (s)) if 0 != subprocess.call(cmdArrForOutput): - Utils.Print("ERROR: Launcher failed to create shape file \"%s\"." % (shapeFile)) + Utils.Print("ERROR: Launcher failed to create shape file \"%s\"." % (newFile)) return False - Utils.Print("opening %s shape file: %s, current dir: %s" % (topo, shapeFile, os.getcwd())) - f = open(shapeFile, "r") - shapeFileJsonStr = f.read() + Utils.Print("opening shape file: %s, current dir: %s" % (newFile, os.getcwd())) + f = open(newFile, "r") + newFileJsonStr = f.read() f.close() - shapeFileObject = json.loads(shapeFileJsonStr) + return json.loads(newFileJsonStr) + + testnetPrefix = "testnet_" + def getTestnetNodeNum(nodeName): + p=re.compile(r'^{}(\d+)$'.format(testnetPrefix)) + m=p.match(nodeName) + return int(m.group(1)) + + if isinstance(topo, dict): + customShapeFile=os.path.join(Utils.ConfigDir, "customShape.json") + customShapeFileObject = createDefaultShapeFile(customShapeFile, cmdArr) + nodeArray = customShapeFileObject["nodes"] + for nodePair in nodeArray: + assert(len(nodePair)==2) + nodeName=nodePair[0] + nodeObject=nodePair[1] + if nodeName == "bios": + continue + nodeNum=getTestnetNodeNum(nodeName) + if nodeNum not in topo: + errorExit("node name: {} was not included in the topo structure: {}".format(nodeName, topo)) + peers = topo[nodeNum] + customShapePeers = [] + for peer in peers: + assert(isinstance(peer, int)) + assert(peer < totalNodes) + peerStr = str(peer) if peer > 9 else "0" + str(peer) + customShapePeers.append(testnetPrefix + peerStr) + nodeObject["peers"] = customShapePeers + + f=open(customShapeFile,"w") + f.write(json.dumps(customShapeFileObject, indent=4, sort_keys=True)) + f.close() + + cmdArr.append("--shape") + cmdArr.append(customShapeFile) + + # must be last cmdArr.append before subprocess.call, so that everything is on the command line + # before constructing the shape.json file for "bridge" + elif topo=="bridge": + shapeFilePrefix="shape_bridge" + shapeFile=shapeFilePrefix+".json" + shapeFileObject = createDefaultShapeFile(shapeFile, cmdArr) Utils.Print("shapeFileObject=%s" % (shapeFileObject)) # retrieve the nodes, which as a map of node name to node definition, which the fc library prints out as # an array of array, the first level of arrays is the pair entries of the map, the second is an array @@ -396,11 +434,6 @@ def getArguments(number): Utils.Print("producers=%s" % (producers)) shapeFileNodeMap = {} - def getNodeNum(nodeName): - p=re.compile(r'^testnet_(\d+)$') - m=p.match(nodeName) - return int(m.group(1)) - for shapeFileNodePair in shapeFileNodes: assert(len(shapeFileNodePair)==2) nodeName=shapeFileNodePair[0] @@ -410,7 +443,7 @@ def getNodeNum(nodeName): if nodeName=="bios": biosNodeObject=shapeFileNode continue - nodeNum=getNodeNum(nodeName) + nodeNum=getTestnetNodeNum(nodeName) Utils.Print("nodeNum=%d, shapeFileNode=%s" % (nodeNum, shapeFileNode)) assert("producers" in shapeFileNode) shapeFileNodeProds=shapeFileNode["producers"] From fa9581c9c50b8123d3f86055e8b9cafa3ef36f5b Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 16 Mar 2021 12:37:33 -0500 Subject: [PATCH 044/157] update ship protocol for global property changes --- libraries/abieos | 2 +- .../eosio/chain/global_property_object.hpp | 37 +++++++++++-------- .../eosio/state_history/serialization.hpp | 6 ++- unittests/state_history_tests.cpp | 2 +- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/libraries/abieos b/libraries/abieos index 210cb8f30af..5e47ec98c0e 160000 --- a/libraries/abieos +++ b/libraries/abieos @@ -1 +1 @@ -Subproject commit 210cb8f30af44bc5c8907f3b797d7b0ed9483055 +Subproject commit 5e47ec98c0e13eb1a35dac88899e876cce3af7d2 diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index 1c58e588233..eebf69f1044 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -143,22 +143,36 @@ namespace eosio { namespace chain { // struct extension_v1 : extension_v0 { new_field_t new_field; }; // using extension_t = std::variant; // + // In addition, get_gpo_extension(), set_gpo_extension() and + // eosio::ship_protocol::global_property from ship_protocol.hpp + // in abieos has to be changed accordingly. using extension_t = std::variant; extension_t extension; }; + inline snapshot_global_property_object::extension_t get_gpo_extension(const global_property_object& gpo) { + return snapshot_global_property_object::extension_v0{ + gpo.proposed_security_group_block_num, gpo.proposed_security_group_participants, gpo.transaction_hooks}; + } + + inline void set_gpo_extension(global_property_object& gpo, + const snapshot_global_property_object::extension_t& extension) { + std::visit( + [&gpo](auto& ext) { + gpo.proposed_security_group_block_num = ext.proposed_security_group_block_num; + gpo.proposed_security_group_participants = std::move(ext.proposed_security_group_participants); + gpo.transaction_hooks = std::move(ext.transaction_hooks); + }, + extension); + } + namespace detail { template<> struct snapshot_row_traits { using value_type = global_property_object; using snapshot_type = snapshot_global_property_object; - static_assert(std::is_same_v>, - "Please update to_snapshot_row()/from_snapshot_row() accordingly when " - "snapshot_global_property_object::extension_t is changed"); - static snapshot_global_property_object to_snapshot_row(const global_property_object& value, const chainbase::database&) { return {value.proposed_schedule_block_num, @@ -167,10 +181,7 @@ namespace eosio { namespace chain { value.chain_id, value.kv_configuration, value.wasm_configuration, - snapshot_global_property_object::extension_v0{value.proposed_security_group_block_num, - value.proposed_security_group_participants, - value.transaction_hooks - }}; + get_gpo_extension(value)}; } static void from_snapshot_row(snapshot_global_property_object&& row, global_property_object& value, @@ -182,13 +193,7 @@ namespace eosio { namespace chain { value.chain_id = row.chain_id; value.kv_configuration = row.kv_configuration; value.wasm_configuration = row.wasm_configuration; - std::visit( - [&value](auto& ext) { - value.proposed_security_group_block_num = ext.proposed_security_group_block_num; - value.proposed_security_group_participants = std::move(ext.proposed_security_group_participants); - value.transaction_hooks = std::move(ext.transaction_hooks); - }, - row.extension); + set_gpo_extension(value, row.extension); } }; } diff --git a/libraries/state_history/include/eosio/state_history/serialization.hpp b/libraries/state_history/include/eosio/state_history/serialization.hpp index 280d5c44543..1f47f2f0e25 100644 --- a/libraries/state_history/include/eosio/state_history/serialization.hpp +++ b/libraries/state_history/include/eosio/state_history/serialization.hpp @@ -359,13 +359,15 @@ ST& operator<<(ST& ds, const history_serial_wrapper& template ST& operator<<(ST& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(1)); + fc::raw::pack(ds, fc::unsigned_int(2)); // for global_property_v2 fc::raw::pack(ds, as_type>(obj.obj.proposed_schedule_block_num)); fc::raw::pack(ds, make_history_serial_wrapper( obj.db, as_type(obj.obj.proposed_schedule))); fc::raw::pack(ds, make_history_serial_wrapper(obj.db, as_type(obj.obj.configuration))); fc::raw::pack(ds, as_type(obj.obj.chain_id)); - + fc::raw::pack(ds, as_type(obj.obj.kv_configuration)); + fc::raw::pack(ds, as_type(obj.obj.wasm_configuration)); + fc::raw::pack(ds, eosio::chain::get_gpo_extension(obj.obj)); return ds; } diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp index 9ab4a9e2d7b..62ddcd6626b 100644 --- a/unittests/state_history_tests.cpp +++ b/unittests/state_history_tests.cpp @@ -854,7 +854,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_global_property_history) { BOOST_REQUIRE(result.first); auto &it_global_property = result.second; BOOST_REQUIRE_EQUAL(it_global_property->rows.obj.size(), 1); - auto global_properties = chain.deserialize_data(it_global_property); + auto global_properties = chain.deserialize_data(it_global_property); auto configuration = std::get(global_properties[0].configuration); BOOST_REQUIRE_EQUAL(configuration.max_transaction_delay, 60); } From d76726c40b93ba07533e53fc0684b3a1297d329a Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 16 Mar 2021 15:55:57 -0500 Subject: [PATCH 045/157] update resume from state commit --- pipeline.jsonc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline.jsonc b/pipeline.jsonc index 906131be910..76fdb578a7c 100644 --- a/pipeline.jsonc +++ b/pipeline.jsonc @@ -52,7 +52,7 @@ "test": [ { - "commit": "b3a96279395425890b3b7c9e8564383d7a364305" + "commit": "fa9581c9c50b8123d3f86055e8b9cafa3ef36f5b" } ] } From f08d96365f2dcc68319a9328ddf898e7a5150f97 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Tue, 16 Mar 2021 17:16:10 -0400 Subject: [PATCH 046/157] changes in the middle of implementation --- .../chain/include/eosio/chain/exceptions.hpp | 9 + plugins/net_plugin/net_plugin.cpp | 170 ++++++++++++++---- 2 files changed, 142 insertions(+), 37 deletions(-) diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index 21356b19dd5..202c1e3dda9 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -681,4 +681,13 @@ namespace eosio { namespace chain { FC_DECLARE_DERIVED_EXCEPTION( state_history_exception, chain_exception, 3280000, "State history exception" ) + FC_DECLARE_DERIVED_EXCEPTION( ssl_exception, chain_exception, + 3290000, "SSL exception") + + FC_DECLARE_DERIVED_EXCEPTION( ssl_incomplete_configuration, ssl_exception, + 3290001, "Incomplete SSL configuration") + FC_DECLARE_DERIVED_EXCEPTION( ssl_configuration_error, ssl_exception, + 3290002, "SSL configuration error") + FC_DECLARE_DERIVED_EXCEPTION( ssl_handshake_error, ssl_exception, + 3290003, "SSL handshake error") } } // eosio::chain diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 4202f111cd8..de4d5510460 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -25,12 +25,15 @@ #include #include #include +#include +#include #include #include #include using namespace eosio::chain::plugin_interface; +using namespace boost::asio; namespace eosio { static appbase::abstract_plugin& _net_plugin = app().register_plugin(); @@ -40,6 +43,7 @@ namespace eosio { using boost::asio::ip::tcp; using boost::asio::ip::address_v4; using boost::asio::ip::host_name; + using boost::system::error_code; using boost::multi_index_container; using fc::time_point; @@ -53,6 +57,8 @@ namespace eosio { using connection_wptr = std::weak_ptr; using io_work_t = boost::asio::executor_work_guard; + using ssl_context_ptr = std::unique_ptr; + using ssl_stream = ssl::stream; template void verify_strand_in_this_thread(const Strand& strand, const char* func, int line) { @@ -208,6 +214,8 @@ namespace eosio { class net_plugin_impl : public std::enable_shared_from_this { public: unique_ptr acceptor; + ssl_context_ptr ssl_context; + bool ssl_enabled; std::atomic current_connection_id{0}; unique_ptr< sync_manager > sync_master; @@ -349,6 +357,8 @@ namespace eosio { * \param bs The block_state containing the security group update */ void update_security_group(const block_state_ptr& bs); + + static ssl_context_ptr create_ssl_context(const std::string& cert, const std::string& pkey, const std::string& ca); }; const fc::string logger_name("net_plugin_impl"); @@ -588,8 +598,8 @@ namespace eosio { class connection : public std::enable_shared_from_this { public: - explicit connection( string endpoint ); - connection(); + explicit connection( string endpoint, bool ssl ); + explicit connection(bool ssl); ~connection() {} @@ -624,9 +634,11 @@ namespace eosio { std::atomic connection_type{both}; + std::shared_ptr socket; // only accessed through strand after construction + std::shared_ptr ssl_socket; public: + bool use_ssl; boost::asio::io_context::strand strand; - std::shared_ptr socket; // only accessed through strand after construction fc::message_buffer<1024*1024> pending_message_buffer; std::atomic outstanding_read_bytes{0}; // accessed only from strand threads @@ -816,6 +828,15 @@ namespace eosio { void set_participating(bool status) { participating_.store(status, std::memory_order_relaxed); } /** returns true if connection is in the security group */ bool is_participating() const { return participating_.load(std::memory_order_relaxed); } + + inline tcp::socket& get_socket() { + if (use_ssl){ + EOS_ASSERT(ssl_socket, ssl_configuration_error, "null ssl socket instance"); + return ssl_socket->next_layer(); + } + EOS_ASSERT(socket, ssl_configuration_error, "null socket"); + return *socket; + } }; const string connection::unknown = ""; @@ -892,28 +913,32 @@ namespace eosio { //--------------------------------------------------------------------------- - connection::connection( string endpoint ) + connection::connection( string endpoint, bool ssl ) : peer_addr( endpoint ), + use_ssl(ssl), strand( my_impl->thread_pool->get_executor() ), - socket( new tcp::socket( my_impl->thread_pool->get_executor() ) ), connection_id( ++my_impl->current_connection_id ), response_expected_timer( my_impl->thread_pool->get_executor() ), last_handshake_recv(), last_handshake_sent() { - fc_ilog( logger, "creating connection to ${n}", ("n", endpoint) ); + if (use_ssl) { + EOS_ASSERT(my_impl->ssl_context, ssl_configuration_error, "ssl context is empty"); + ssl_socket.reset( new ssl_stream{my_impl->thread_pool->get_executor(), *my_impl->ssl_context} ); + } + else { + socket.reset( new tcp::socket( my_impl->thread_pool->get_executor() ) ); + } + + if (endpoint.empty()) + fc_dlog( logger, "new connection object created" ); + else + fc_ilog( logger, "created connection to ${n}", ("n", endpoint) ); } - connection::connection() - : peer_addr(), - strand( my_impl->thread_pool->get_executor() ), - socket( new tcp::socket( my_impl->thread_pool->get_executor() ) ), - connection_id( ++my_impl->current_connection_id ), - response_expected_timer( my_impl->thread_pool->get_executor() ), - last_handshake_recv(), - last_handshake_sent() + connection::connection(bool ssl) + : connection({}, ssl) { - fc_dlog( logger, "new connection object created" ); } void connection::update_endpoints() { @@ -975,9 +1000,22 @@ namespace eosio { close(); return false; } else { - fc_dlog( logger, "connected to ${peer}", ("peer", peer_name()) ); - socket_open = true; - start_read_message(); + + auto start_read = [this](){ + fc_dlog( logger, "connected to ${peer}", ("peer", peer_name()) ); + socket_open = true; + start_read_message(); + }; + if (use_ssl) { + ssl_socket->async_handshake(ssl::stream_base::server, + [start_read](const auto& ec){ + EOS_ASSERT(!ec, ssl_handshake_error, "ssl handshake error: ${e}", ("e", ec.message())); + start_read(); + }); + } + else { + start_read(); + } return true; } } @@ -1221,8 +1259,7 @@ namespace eosio { buffer_queue.fill_out_buffer( bufs ); strand.post( [c{std::move(c)}, bufs{std::move(bufs)}]() { - boost::asio::async_write( *c->socket, bufs, - boost::asio::bind_executor( c->strand, [c, socket=c->socket]( boost::system::error_code ec, std::size_t w ) { + auto write_lambda = [c, socket=c->socket]( boost::system::error_code ec, std::size_t w ) { try { c->buffer_queue.clear_out_queue(); // May have closed connection and cleared buffer_queue @@ -1258,7 +1295,14 @@ namespace eosio { } catch( ... ) { fc_elog( logger, "Exception in do_queue_write to ${p}", ("p", c->peer_name()) ); } - })); + }; + + if (c->use_ssl){ + boost::asio::async_write( *c->ssl_socket, bufs, boost::asio::bind_executor( c->strand, write_lambda )); + } + else { + boost::asio::async_write( *c->socket, bufs, boost::asio::bind_executor( c->strand, write_lambda )); + } }); } @@ -2464,16 +2508,17 @@ namespace eosio { } void net_plugin_impl::start_listen_loop() { - connection_ptr new_connection = std::make_shared(); + connection_ptr new_connection = std::make_shared(ssl_enabled); new_connection->connecting = true; new_connection->strand.post( [this, new_connection = std::move( new_connection )](){ - acceptor->async_accept( *new_connection->socket, - boost::asio::bind_executor( new_connection->strand, [new_connection, socket=new_connection->socket, this]( boost::system::error_code ec ) { + acceptor->async_accept( new_connection->get_socket(), + boost::asio::bind_executor( new_connection->strand, [new_connection, this]( boost::system::error_code ec ) { if( !ec ) { uint32_t visitors = 0; uint32_t from_addr = 0; boost::system::error_code rec; - const auto& paddr_add = socket->remote_endpoint( rec ).address(); + tcp::socket& socket = new_connection->get_socket(); + const auto& paddr_add = socket.remote_endpoint( rec ).address(); string paddr_str; if( rec ) { fc_elog( logger, "Error getting remote endpoint: ${m}", ("m", rec.message())); @@ -2508,8 +2553,8 @@ namespace eosio { } // new_connection never added to connections and start_session not called, lifetime will end boost::system::error_code ec; - socket->shutdown( tcp::socket::shutdown_both, ec ); - socket->close( ec ); + socket.shutdown( tcp::socket::shutdown_both, ec ); + socket.close( ec ); } } } else { @@ -2538,13 +2583,12 @@ namespace eosio { std::size_t minimum_read = std::atomic_exchange( &outstanding_read_bytes, 0 ); minimum_read = minimum_read != 0 ? minimum_read : message_header_size; - if (my_impl->use_socket_read_watermark) { const size_t max_socket_read_watermark = 4096; std::size_t socket_read_watermark = std::min(minimum_read, max_socket_read_watermark); boost::asio::socket_base::receive_low_watermark read_watermark_opt(socket_read_watermark); boost::system::error_code ec; - socket->set_option( read_watermark_opt, ec ); + get_socket().set_option( read_watermark_opt, ec ); if( ec ) { fc_elog( logger, "unable to set read watermark ${peer}: ${e1}", ("peer", peer_name())( "e1", ec.message() ) ); } @@ -2565,13 +2609,9 @@ namespace eosio { close( false ); return; } - - boost::asio::async_read( *socket, - pending_message_buffer.get_buffer_sequence_for_boost_async_read(), completion_handler, - boost::asio::bind_executor( strand, - [conn = shared_from_this(), socket=socket]( boost::system::error_code ec, std::size_t bytes_transferred ) { + auto handle_read = [conn = shared_from_this(), psocket=&get_socket()]( boost::system::error_code ec, std::size_t bytes_transferred ) { // may have closed connection and cleared pending_message_buffer - if( !conn->socket_is_open() || socket != conn->socket ) return; + if( !conn->socket_is_open() || psocket != &conn->get_socket() ) return; bool close_connection = false; try { @@ -2654,7 +2694,19 @@ namespace eosio { fc_elog( logger, "Closing connection to: ${p}", ("p", conn->peer_name()) ); conn->close(); } - })); + }; + if (use_ssl){ + boost::asio::async_read( *ssl_socket, + pending_message_buffer.get_buffer_sequence_for_boost_async_read(), + completion_handler, + boost::asio::bind_executor( strand, handle_read)); + } + else { + boost::asio::async_read( *socket, + pending_message_buffer.get_buffer_sequence_for_boost_async_read(), + completion_handler, + boost::asio::bind_executor( strand, handle_read)); + } } catch (...) { fc_elog( logger, "Undefined exception in start_read_message, closing connection to: ${p}", ("p", peer_name()) ); close(); @@ -3642,6 +3694,34 @@ namespace eosio { return chain::signature_type(); } + ssl_context_ptr net_plugin_impl::create_ssl_context(const std::string& cert, const std::string& pkey, const std::string& ca){ + ssl_context_ptr context( new ssl::context(ssl::context::sslv23) ); + + //TLS-only connection + context->set_options(ssl::context::default_workarounds | + ssl::context::no_sslv2 | + ssl::context::no_sslv3 ); + //this ensures peer has valid certificate. no certificate-less connections + context->set_verify_mode(ssl::context::verify_peer | ssl::context::verify_fail_if_no_peer_cert); + context->set_password_callback([](auto,auto){ return "test"; }); + + error_code ec; + + if (!ca.empty()){ + context->load_verify_file(ca, ec); + EOS_ASSERT(!ec, ssl_configuration_error, "load_verify_file: ${e}", ("e", ec.message())); + } + + context->use_private_key_file(pkey, ssl::context::pem, ec); + EOS_ASSERT(!ec, ssl_configuration_error, "use_private_key_file: ${e}", ("e", ec.message())); + + context->use_certificate_chain_file(cert, ec);//cert_verify_chain.pem // //client1.crt + EOS_ASSERT(!ec, ssl_configuration_error, "use_certificate_chain_file: ${e}", ("e", ec.message())); + + + return context; + } + // call from connection strand bool connection::populate_handshake( handshake_message& hello, bool force ) { namespace sc = std::chrono; @@ -3843,6 +3923,22 @@ namespace eosio { my->chain_plug->enable_accept_transactions(); } + //if we have certificate option that TLS must be enabled + if ( options.count("p2p-tls-own-certificate-file") ) { + auto certificate = options["p2p-tls-own-certificate-file"].as(); + auto pkey = options["p2p-tls-private-key-file"].as(); + auto ca_cert = options["p2p-tls-ca-certificate-file"].as(); + + EOS_ASSERT(fc::is_regular_file({certificate}), ssl_incomplete_configuration, "p2p-tls-own-certificate-file doesn't contain regular file: ${p}", ("p", certificate)); + EOS_ASSERT(fc::is_regular_file({pkey}), ssl_incomplete_configuration, "p2p-tls-private-key-file doesn't contain regular file: ${p}", ("p", pkey)); + my->ssl_enabled = true; + if (!ca_cert.empty()){ + EOS_ASSERT(fc::is_regular_file({ca_cert}), ssl_incomplete_configuration, "2p-tls-ca-certificate-file doesn't contain regular file: ${p}", ("p", ca_cert)); + } + + my->ssl_context = net_plugin_impl::create_ssl_context(certificate, pkey, ca_cert); + } + } FC_LOG_AND_RETHROW() } @@ -4000,7 +4096,7 @@ namespace eosio { if( my->find_connection( host ) ) return "already connected"; - connection_ptr c = std::make_shared( host ); + connection_ptr c = std::make_shared( host, my->ssl_enabled ); fc_dlog( logger, "calling active connector: ${h}", ("h", host) ); if( c->resolve_and_connect() ) { fc_dlog( logger, "adding new connection to the list: ${c}", ("c", c->peer_name()) ); From 91d2cf5094f9d22857f1db0113fa95d8d952c7aa Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 16 Mar 2021 16:20:34 -0500 Subject: [PATCH 047/157] Fixed code for unreported nodes in custom shaped topology. --- tests/Cluster.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index 50c5e919e2f..153d7291682 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -382,12 +382,15 @@ def getTestnetNodeNum(nodeName): if nodeName == "bios": continue nodeNum=getTestnetNodeNum(nodeName) + customShapePeers = [] if nodeNum not in topo: - errorExit("node name: {} was not included in the topo structure: {}".format(nodeName, topo)) + if Utils.Debug: Utils.Print("node name: {} was not included in the topo structure: {}".format(nodeName, topo)) + nodeObject["peers"] = customShapePeers + continue peers = topo[nodeNum] - customShapePeers = [] + Utils.Print("nodeNum: {}, peers: {}".format(nodeNum, peers)) for peer in peers: - assert(isinstance(peer, int)) + Utils.Print("peer: {}".format(peer)) assert(peer < totalNodes) peerStr = str(peer) if peer > 9 else "0" + str(peer) customShapePeers.append(testnetPrefix + peerStr) From 13ee62a77d97c835da57dd7c56832bd458058b7d Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 16 Mar 2021 16:38:59 -0500 Subject: [PATCH 048/157] Added the privacy_startup_network test script. --- tests/CMakeLists.txt | 3 + tests/privacy_startup_network.py | 131 +++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100755 tests/privacy_startup_network.py diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0258d4ee8ef..1c47be50b08 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -61,6 +61,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/trace_plugin_test.py ${CMAKE_CURRENT_ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_contrl_c_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_contrl_c_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/blockvault_tests.py ${CMAKE_CURRENT_BINARY_DIR}/blockvault_tests.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/generate-certificates.sh ${CMAKE_CURRENT_BINARY_DIR}/generate-certificates.sh COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_startup_network.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_startup_network.py COPYONLY) #To run plugin_test with all log from blockchain displayed, put --verbose after --, i.e. plugin_test -- --verbose add_test(NAME plugin_test COMMAND plugin_test --report_level=detailed --color_output) @@ -115,6 +116,8 @@ add_test(NAME light_validation_sync_test COMMAND tests/light_validation_sync_tes set_property(TEST light_validation_sync_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME eosio_blocklog_prune_test COMMAND tests/eosio_blocklog_prune_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST eosio_blocklog_prune_test PROPERTY LABELS nonparallelizable_tests) +add_test(NAME privacy_startup_network COMMAND tests/privacy_startup_network.py -p 2 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST privacy_startup_network PROPERTY LABELS nonparallelizable_tests) # Long running tests add_test(NAME nodeos_sanity_lr_test COMMAND tests/nodeos_run_test.py -v --sanity-test --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py new file mode 100755 index 00000000000..494d43f7304 --- /dev/null +++ b/tests/privacy_startup_network.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 + +from testUtils import Account +from testUtils import Utils +from Cluster import Cluster +from WalletMgr import WalletMgr +from Node import BlockType +from Node import Node +from Node import ReturnType +from TestHelper import TestHelper + +import decimal +import re +import signal +import json +import os + +############################################################### +# privacy_startup_network +# +# General test for Privacy to verify TLS connections, and slowly adding participants to the security group and verifying +# how blocks and transactions are sent/not sent. +# +############################################################### + +Print=Utils.Print +errorExit=Utils.errorExit +cmdError=Utils.cmdError +from core_symbol import CORE_SYMBOL + +args = TestHelper.parse_args({"--port","-p","-n","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run" + ,"--sanity-test","--wallet-port"}) +port=args.port +pnodes=args.p +relayNodes=pnodes # every pnode paired with a relay node +apiNodes=2 # minimum number of apiNodes that +minTotalNodes=pnodes+relayNodes+apiNodes +totalNodes=args.n if args.n >= minTotalNodes else minTotalNodes +if totalNodes > minTotalNodes: + apiNodes += totalNodes - minTotalNodes + +debug=args.v +dumpErrorDetails=args.dump_error_details +keepLogs=args.keep_logs +dontKill=args.leave_running +onlyBios=False +killAll=args.clean_run +sanityTest=args.sanity_test +walletPort=args.wallet_port + +Utils.Debug=debug +cluster=Cluster(host=TestHelper.LOCAL_HOST, port=port, walletd=True) +walletMgr=WalletMgr(True, port=walletPort) +testSuccessful=False +killEosInstances=not dontKill +killWallet=not dontKill +dontBootstrap=sanityTest # intent is to limit the scope of the sanity test to just verifying that nodes can be started + +WalletdName=Utils.EosWalletName +ClientName="cleos" +timeout = .5 * 12 * pnodes + 60 # time for finalization with 1 producer + 60 seconds padding +Utils.setIrreversibleTimeout(timeout) + +try: + TestHelper.printSystemInfo("BEGIN") + cluster.setWalletMgr(walletMgr) + Print("SERVER: %s" % (TestHelper.LOCAL_HOST)) + Print("PORT: %d" % (port)) + + cluster.killall(allInstances=killAll) + cluster.cleanup() + Print("Stand up cluster") + topo = {} + firstApiNodeNum = pnodes + relayNodes + apiNodeNums = [x for x in range(firstApiNodeNum, totalNodes)] + for producerNum in range(pnodes): + topo[producerNum] = [] + pairedRelayNodeNum = producerNum + 1 + topo[producerNum].append(pairedRelayNodeNum) + topo[pairedRelayNodeNum] = [] + topo[pairedRelayNodeNum].extend(apiNodeNums) + Utils.Print("topo: {}".format(json.dumps(topo, indent=4, sort_keys=True))) + + if cluster.launch(totalNodes=totalNodes, prodCount=1, onlyBios=False, dontBootstrap=dontBootstrap, configSecurityGroup=True, topo=topo) is False: + cmdError("launcher") + errorExit("Failed to stand up eos cluster.") + + Print("Validating system accounts after bootstrap") + cluster.validateAccounts(None) + + cluster.biosNode.kill(signal.SIGTERM) + + producers = [cluster.getNode(2 * x) for x in range(pnodes) ] + relays = [cluster.getNode(2 * x + 1) for x in range(pnodes) ] + apiNodes = [cluster.getNode(x) for x in apiNodeNums] + + def verifyInSync(producerNum): + Utils.Print("Ensure all nodes are in-sync") + lib = producers[producerNum].getInfo()["last_irreversible_block_num"] + headBlockNum = producers[producerNum].getBlockNum() + headBlock = producers[producerNum].getBlock(headBlockNum) + Utils.Print("headBlock: {}".format(json.dumps(headBlock, indent=4, sort_keys=True))) + headBlockId = headBlock["id"] + for prod in producers: + if prod == producers[producerNum]: + continue + + assert prod.waitForBlock(headBlockNum, timeout = 10, reportInterval = 1) != None, "Producer node failed to get block number {}".format(headBlockNum) + prod.getBlock(headBlockId) # if it isn't there it will throw an exception + assert prod.waitForBlock(lib, blockType=BlockType.lib), \ + "Producer node is failing to advance its lib ({}) with producer {} ({})".format(node.getInfo()["last_irreversible_block_num"], producerNum, lib) + for node in apiNodes: + assert node.waitForBlock(headBlockNum, timeout = 10, reportInterval = 1) != None, "API node failed to get block number {}".format(headBlockNum) + node.getBlock(headBlockId) # if it isn't there it will throw an exception + assert node.waitForBlock(lib, blockType=BlockType.lib), \ + "API node is failing to advance its lib ({}) with producer {} ({})".format(node.getInfo()["last_irreversible_block_num"], producerNum, lib) + + Utils.Print("Ensure all nodes are in-sync") + assert node.waitForBlock(lib + 1, blockType=BlockType.lib) != None, "API node failed to get block number {}".format(headBlockNum) + + verifyInSync(producerNum=0) + + if sanityTest: + testSuccessful=True + exit(0) + + testSuccessful=True +finally: + TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails) + +exit(0) From ae13158ad96d16d946ef4438d9304633874afdec Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Tue, 16 Mar 2021 20:28:23 -0400 Subject: [PATCH 049/157] partial TLS implementation of net plugin --- plugins/net_plugin/net_plugin.cpp | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index de4d5510460..d92cd749f85 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -1041,11 +1041,15 @@ namespace eosio { void connection::_close( connection* self, bool reconnect, bool shutdown ) { self->socket_open = false; boost::system::error_code ec; - if( self->socket->is_open() ) { - self->socket->shutdown( tcp::socket::shutdown_both, ec ); - self->socket->close( ec ); + if( self->get_socket().is_open() ) { + self->get_socket().shutdown( tcp::socket::shutdown_both, ec ); + self->get_socket().close( ec ); } - self->socket.reset( new tcp::socket( my_impl->thread_pool->get_executor() ) ); + if (self->use_ssl) + self->ssl_socket.reset( new ssl_stream{my_impl->thread_pool->get_executor(), *my_impl->ssl_context} ); + else + self->socket.reset( new tcp::socket( my_impl->thread_pool->get_executor() ) ); + self->flush_queues(); self->connecting = false; self->syncing = false; @@ -1259,11 +1263,11 @@ namespace eosio { buffer_queue.fill_out_buffer( bufs ); strand.post( [c{std::move(c)}, bufs{std::move(bufs)}]() { - auto write_lambda = [c, socket=c->socket]( boost::system::error_code ec, std::size_t w ) { + auto write_lambda = [c, psocket=&c->get_socket()]( boost::system::error_code ec, std::size_t w ) { try { c->buffer_queue.clear_out_queue(); // May have closed connection and cleared buffer_queue - if( !c->socket_is_open() || socket != c->socket ) { + if( !c->socket_is_open() || psocket != &c->get_socket() ) { fc_ilog( logger, "async write socket ${r} before callback: ${p}", ("r", c->socket_is_open() ? "changed" : "closed")("p", c->peer_name()) ); c->close(); @@ -2493,10 +2497,10 @@ namespace eosio { connecting = true; pending_message_buffer.reset(); buffer_queue.clear_out_queue(); - boost::asio::async_connect( *socket, endpoints, + boost::asio::async_connect( get_socket(), endpoints, boost::asio::bind_executor( strand, - [resolver, c = shared_from_this(), socket=socket]( const boost::system::error_code& err, const tcp::endpoint& endpoint ) { - if( !err && socket->is_open() && socket == c->socket ) { + [resolver, c = shared_from_this(), psocket=&get_socket()]( const boost::system::error_code& err, const tcp::endpoint& endpoint ) { + if( !err && psocket->is_open() && psocket == &c->get_socket() ) { if( c->start_session() ) { c->send_handshake(); } From c80ef879d82e39137944084736ca7d941f929eed Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 17 Mar 2021 12:41:54 -0500 Subject: [PATCH 050/157] address PR comments --- .../chain/include/eosio/chain/global_property_object.hpp | 6 +++--- .../include/eosio/state_history/serialization.hpp | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index eebf69f1044..bfa7feb63b0 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -156,8 +156,8 @@ namespace eosio { namespace chain { gpo.proposed_security_group_block_num, gpo.proposed_security_group_participants, gpo.transaction_hooks}; } - inline void set_gpo_extension(global_property_object& gpo, - const snapshot_global_property_object::extension_t& extension) { + inline void set_gpo_extension(global_property_object& gpo, + snapshot_global_property_object::extension_t&& extension) { std::visit( [&gpo](auto& ext) { gpo.proposed_security_group_block_num = ext.proposed_security_group_block_num; @@ -193,7 +193,7 @@ namespace eosio { namespace chain { value.chain_id = row.chain_id; value.kv_configuration = row.kv_configuration; value.wasm_configuration = row.wasm_configuration; - set_gpo_extension(value, row.extension); + set_gpo_extension(value, std::move(row.extension)); } }; } diff --git a/libraries/state_history/include/eosio/state_history/serialization.hpp b/libraries/state_history/include/eosio/state_history/serialization.hpp index 1f47f2f0e25..fe11f67f253 100644 --- a/libraries/state_history/include/eosio/state_history/serialization.hpp +++ b/libraries/state_history/include/eosio/state_history/serialization.hpp @@ -359,7 +359,8 @@ ST& operator<<(ST& ds, const history_serial_wrapper& template ST& operator<<(ST& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(2)); // for global_property_v2 + const fc::unsigned_int global_property_version = 2; + fc::raw::pack(ds, global_property_version); fc::raw::pack(ds, as_type>(obj.obj.proposed_schedule_block_num)); fc::raw::pack(ds, make_history_serial_wrapper( obj.db, as_type(obj.obj.proposed_schedule))); From 4e648699a08f1115019d05503e95f21074767c84 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 17 Mar 2021 13:59:41 -0500 Subject: [PATCH 051/157] Fixed number ordering of producers and relay nodes and topology configuration. --- tests/Cluster.py | 1 + tests/Node.py | 4 ++-- tests/privacy_startup_network.py | 14 ++++++-------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index 153d7291682..d54ee38ac6b 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -372,6 +372,7 @@ def getTestnetNodeNum(nodeName): return int(m.group(1)) if isinstance(topo, dict): + if Utils.Debug: Utils.Print("Creating custom shape topology with the following node connections: {}".format(json.dumps(topo, indent=4, sort_keys=True))) customShapeFile=os.path.join(Utils.ConfigDir, "customShape.json") customShapeFileObject = createDefaultShapeFile(customShapeFile, cmdArr) nodeArray = customShapeFileObject["nodes"] diff --git a/tests/Node.py b/tests/Node.py index cba8668d55e..d2e22e2a61d 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -529,12 +529,12 @@ def __call__(self): currentBlockNum = self.node.getBlockNum(blockType=blockType) self.advanced = False if self.lastBlockNum is None or self.lastBlockNum < currentBlockNum: - self.advanced = True + self.advanced = True if self.lastBlockNum is not None else False elif self.lastBlockNum > currentBlockNum: Utils.Print("waitForBlock is waiting to reach block number: %d and the block number has rolled back from %d to %d." % (self.blockNum, self.lastBlockNum, currentBlockNum)) self.lastBlockNum = currentBlockNum - self.passed = self.lastBlockNum > self.blockNum + self.passed = self.lastBlockNum >= self.blockNum return self.passed def __enter__(self): diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index 494d43f7304..f13671ec1e7 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -74,11 +74,9 @@ firstApiNodeNum = pnodes + relayNodes apiNodeNums = [x for x in range(firstApiNodeNum, totalNodes)] for producerNum in range(pnodes): - topo[producerNum] = [] - pairedRelayNodeNum = producerNum + 1 - topo[producerNum].append(pairedRelayNodeNum) - topo[pairedRelayNodeNum] = [] - topo[pairedRelayNodeNum].extend(apiNodeNums) + pairedRelayNodeNum = pnodes + producerNum + topo[producerNum] = [pairedRelayNodeNum] + topo[pairedRelayNodeNum] = apiNodeNums Utils.Print("topo: {}".format(json.dumps(topo, indent=4, sort_keys=True))) if cluster.launch(totalNodes=totalNodes, prodCount=1, onlyBios=False, dontBootstrap=dontBootstrap, configSecurityGroup=True, topo=topo) is False: @@ -90,8 +88,8 @@ cluster.biosNode.kill(signal.SIGTERM) - producers = [cluster.getNode(2 * x) for x in range(pnodes) ] - relays = [cluster.getNode(2 * x + 1) for x in range(pnodes) ] + producers = [cluster.getNode(x) for x in range(pnodes) ] + relays = [cluster.getNode(pnodes + x) for x in range(pnodes) ] apiNodes = [cluster.getNode(x) for x in apiNodeNums] def verifyInSync(producerNum): @@ -116,7 +114,7 @@ def verifyInSync(producerNum): "API node is failing to advance its lib ({}) with producer {} ({})".format(node.getInfo()["last_irreversible_block_num"], producerNum, lib) Utils.Print("Ensure all nodes are in-sync") - assert node.waitForBlock(lib + 1, blockType=BlockType.lib) != None, "API node failed to get block number {}".format(headBlockNum) + assert node.waitForBlock(lib + 1, blockType=BlockType.lib, reportInterval = 1) != None, "Producer node failed to advance lib ahead one block to: {}".format(lib + 1) verifyInSync(producerNum=0) From feee04bb08b81c563e04d5ec66b1792b9cbaef2a Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 17 Mar 2021 14:49:06 -0500 Subject: [PATCH 052/157] Only currently working for 1 producer, will investigate and fix later. --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1c47be50b08..ea766291049 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -116,7 +116,7 @@ add_test(NAME light_validation_sync_test COMMAND tests/light_validation_sync_tes set_property(TEST light_validation_sync_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME eosio_blocklog_prune_test COMMAND tests/eosio_blocklog_prune_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST eosio_blocklog_prune_test PROPERTY LABELS nonparallelizable_tests) -add_test(NAME privacy_startup_network COMMAND tests/privacy_startup_network.py -p 2 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME privacy_startup_network COMMAND tests/privacy_startup_network.py -p 1 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST privacy_startup_network PROPERTY LABELS nonparallelizable_tests) # Long running tests From 9c0ae2589a96d02407e1f98cc4e1efe5444ab5aa Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 17 Mar 2021 14:50:40 -0500 Subject: [PATCH 053/157] Added passing pnodes to launch. --- tests/privacy_startup_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index f13671ec1e7..3974eb5e158 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -79,7 +79,7 @@ topo[pairedRelayNodeNum] = apiNodeNums Utils.Print("topo: {}".format(json.dumps(topo, indent=4, sort_keys=True))) - if cluster.launch(totalNodes=totalNodes, prodCount=1, onlyBios=False, dontBootstrap=dontBootstrap, configSecurityGroup=True, topo=topo) is False: + if cluster.launch(pnodes=pnodes, totalNodes=totalNodes, prodCount=1, onlyBios=False, dontBootstrap=dontBootstrap, configSecurityGroup=True, topo=topo) is False: cmdError("launcher") errorExit("Failed to stand up eos cluster.") From f2bdbf4b85e54679f522c82224def1fa1d0160d3 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 17 Mar 2021 17:38:49 -0500 Subject: [PATCH 054/157] Peer review comment fixes. --- programs/eosio-launcher/main.cpp | 2 +- tests/Cluster.py | 10 ++++------ tests/privacy_startup_network.py | 5 ++--- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/programs/eosio-launcher/main.cpp b/programs/eosio-launcher/main.cpp index 877bafc01ea..74a6e193920 100644 --- a/programs/eosio-launcher/main.cpp +++ b/programs/eosio-launcher/main.cpp @@ -1503,7 +1503,7 @@ launcher_def::launch (eosd_def &instance, string >s) { node_rt_info info; info.remote = !host->is_local(); - const auto node_num = (instance.name == "bios") ? -1 : boost::lexical_cast(instance.get_node_num()); + const auto node_num = (instance.name == "bios") ? bios_node_num : boost::lexical_cast(instance.get_node_num()); string install_path; if (!specific_nodeos_installation_paths.empty()) { if (specific_nodeos_installation_paths.count(node_num)) { diff --git a/tests/Cluster.py b/tests/Cluster.py index d54ee38ac6b..bfbf3bd60f1 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -239,7 +239,7 @@ def insertSpecificExtraNodeosArgs(node, insertStr): f.write("=========================") f.write("{}".format(rtn)) os.chdir(original) - if Utils.Debug: Utils.Print("changed back to dir: %s" % (os.getcwd())) + if Utils.Debug: Utils.Print("changed back to dir: {}".format(os.getcwd())) if specificExtraNodeosArgs is None: specificExtraNodeosArgs = {} @@ -360,9 +360,8 @@ def createDefaultShapeFile(newFile, cmdArr): return False Utils.Print("opening shape file: %s, current dir: %s" % (newFile, os.getcwd())) - f = open(newFile, "r") - newFileJsonStr = f.read() - f.close() + with open(newFile, 'r') as f: + newFileJsonStr = f.read() return json.loads(newFileJsonStr) testnetPrefix = "testnet_" @@ -393,8 +392,7 @@ def getTestnetNodeNum(nodeName): for peer in peers: Utils.Print("peer: {}".format(peer)) assert(peer < totalNodes) - peerStr = str(peer) if peer > 9 else "0" + str(peer) - customShapePeers.append(testnetPrefix + peerStr) + customShapePeers.append("{}{:02}".format(testnetPrefix, peerStr)) nodeObject["peers"] = customShapePeers f=open(customShapeFile,"w") diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index 3974eb5e158..51ba6a715ac 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -33,13 +33,13 @@ port=args.port pnodes=args.p relayNodes=pnodes # every pnode paired with a relay node -apiNodes=2 # minimum number of apiNodes that +apiNodes=2 # minimum number of apiNodes that will be used in this test minTotalNodes=pnodes+relayNodes+apiNodes totalNodes=args.n if args.n >= minTotalNodes else minTotalNodes if totalNodes > minTotalNodes: apiNodes += totalNodes - minTotalNodes -debug=args.v +Utils.Debug=args.v dumpErrorDetails=args.dump_error_details keepLogs=args.keep_logs dontKill=args.leave_running @@ -48,7 +48,6 @@ sanityTest=args.sanity_test walletPort=args.wallet_port -Utils.Debug=debug cluster=Cluster(host=TestHelper.LOCAL_HOST, port=port, walletd=True) walletMgr=WalletMgr(True, port=walletPort) testSuccessful=False From c571480c66f6b88fefe48e9bec438f70b277b498 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 18 Mar 2021 08:26:26 -0500 Subject: [PATCH 055/157] Missed changing parameter name in last fix. --- tests/Cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index bfbf3bd60f1..42e9735ac8d 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -392,7 +392,7 @@ def getTestnetNodeNum(nodeName): for peer in peers: Utils.Print("peer: {}".format(peer)) assert(peer < totalNodes) - customShapePeers.append("{}{:02}".format(testnetPrefix, peerStr)) + customShapePeers.append("{}{:02}".format(testnetPrefix, peer)) nodeObject["peers"] = customShapePeers f=open(customShapeFile,"w") From 7bd4fc3fbecfd7ad21ce169ca5a478ce9408846a Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 18 Mar 2021 12:04:38 -0500 Subject: [PATCH 056/157] Fixed path to generated certificates. --- tests/Cluster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index 42e9735ac8d..70c7e0ce93c 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -243,10 +243,10 @@ def insertSpecificExtraNodeosArgs(node, insertStr): if specificExtraNodeosArgs is None: specificExtraNodeosArgs = {} - certAuth = os.path.join(Utils.ConfigDir, "CA_cert.pem") + certAuth = os.path.join(privacyDir, "CA_cert.pem") def getArguments(number): - nodeCert = os.path.join(Utils.ConfigDir, "node{}.crt".format(number)) - nodeKey = os.path.join(Utils.ConfigDir, "node{}_key.pem".format(number)) + nodeCert = os.path.join(privacyDir, "node{}.crt".format(number)) + nodeKey = os.path.join(privacyDir, "node{}_key.pem".format(number)) return "--p2p-tls-own-certificate-file {} --p2p-tls-private-key-file {} --p2p-tls-ca-certificate-file {}".format(nodeCert, nodeKey, certAuth) for node in range(totalNodes): From 8e495b2081fe49ab3f0613bdc1788dc17a32b51c Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 18 Mar 2021 12:05:35 -0500 Subject: [PATCH 057/157] Changed to use with idiom for opening files and removed python 2 string formatting. --- tests/Cluster.py | 13 ++++++------- tests/privacy_startup_network.py | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index 70c7e0ce93c..96bd834daea 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -223,11 +223,11 @@ def insertSpecificExtraNodeosArgs(node, insertStr): if configSecurityGroup: privacyDir=os.path.join(Utils.ConfigDir, "privacy") if not os.path.isdir(privacyDir): - if Utils.Debug: Utils.Print("creating dir %s in dir: %s" % (privacyDir, os.getcwd())) + if Utils.Debug: Utils.Print("creating dir {} in dir: {}".format(privacyDir, os.getcwd())) os.mkdir(privacyDir) original=os.getcwd() os.chdir(privacyDir) - if Utils.Debug: Utils.Print("change to dir: %s" % (os.getcwd())) + if Utils.Debug: Utils.Print("change to dir: {}".format(os.getcwd())) genCertScript=os.path.join(original, "tests", "generate-certificates.sh") totalNodesInNetwork = totalNodes + 1 # account for bios node cmd = "{} --days 1 --CA-org Block.one --CA-CN test-domain --org-mask node{{NUMBER}} --cn-mask test-domain{{NUMBER}} --group-size {}".format(genCertScript, totalNodesInNetwork) @@ -356,10 +356,10 @@ def createDefaultShapeFile(newFile, cmdArr): s=" ".join(cmdArrForOutput) if Utils.Debug: Utils.Print("cmd: %s" % (s)) if 0 != subprocess.call(cmdArrForOutput): - Utils.Print("ERROR: Launcher failed to create shape file \"%s\"." % (newFile)) + Utils.Print("ERROR: Launcher failed to create shape file \"{}\".".format(newFile)) return False - Utils.Print("opening shape file: %s, current dir: %s" % (newFile, os.getcwd())) + Utils.Print("opening shape file: {}, current dir: {}".format(newFile, os.getcwd())) with open(newFile, 'r') as f: newFileJsonStr = f.read() return json.loads(newFileJsonStr) @@ -395,9 +395,8 @@ def getTestnetNodeNum(nodeName): customShapePeers.append("{}{:02}".format(testnetPrefix, peer)) nodeObject["peers"] = customShapePeers - f=open(customShapeFile,"w") - f.write(json.dumps(customShapeFileObject, indent=4, sort_keys=True)) - f.close() + with open(customShapeFile, 'w') as f: + f.write(json.dumps(customShapeFileObject, indent=4, sort_keys=True)) cmdArr.append("--shape") cmdArr.append(customShapeFile) diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index 51ba6a715ac..c7a11edaa93 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -63,8 +63,8 @@ try: TestHelper.printSystemInfo("BEGIN") cluster.setWalletMgr(walletMgr) - Print("SERVER: %s" % (TestHelper.LOCAL_HOST)) - Print("PORT: %d" % (port)) + Print("SERVER: {}".format(TestHelper.LOCAL_HOST)) + Print("PORT: {}".format(port)) cluster.killall(allInstances=killAll) cluster.cleanup() From 82a7256e7178c6c9247e502f897047b810b08ffd Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Thu, 18 Mar 2021 14:09:36 -0400 Subject: [PATCH 058/157] security groups implementation added to net plugin --- .../chain/include/eosio/chain/exceptions.hpp | 2 - plugins/net_plugin/net_plugin.cpp | 210 ++++++++++++------ 2 files changed, 143 insertions(+), 69 deletions(-) diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index 202c1e3dda9..34159e2861e 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -688,6 +688,4 @@ namespace eosio { namespace chain { 3290001, "Incomplete SSL configuration") FC_DECLARE_DERIVED_EXCEPTION( ssl_configuration_error, ssl_exception, 3290002, "SSL configuration error") - FC_DECLARE_DERIVED_EXCEPTION( ssl_handshake_error, ssl_exception, - 3290003, "SSL handshake error") } } // eosio::chain diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index d92cd749f85..eb76b03ce81 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -358,7 +358,7 @@ namespace eosio { */ void update_security_group(const block_state_ptr& bs); - static ssl_context_ptr create_ssl_context(const std::string& cert, const std::string& pkey, const std::string& ca); + void init_ssl_context(const std::string& cert, const std::string& pkey, const std::string& ca); }; const fc::string logger_name("net_plugin_impl"); @@ -596,6 +596,36 @@ namespace eosio { block_status_monitor& operator=( block_status_monitor&& ) = delete; }; + struct flex_socket { + std::shared_ptr socket; + std::shared_ptr ssl_socket; + const bool use_ssl; + + flex_socket(bool ssl) : use_ssl(ssl) { + reset_socket(); + } + + inline tcp::socket& get_socket() { + if (use_ssl){ + EOS_ASSERT(ssl_socket, ssl_configuration_error, "null ssl socket instance"); + return ssl_socket->next_layer(); + } + EOS_ASSERT(socket, ssl_configuration_error, "null socket"); + return *socket; + } + + void reset_socket() { + + if (use_ssl) { + EOS_ASSERT(my_impl->ssl_context, ssl_configuration_error, "ssl context is empty"); + ssl_socket.reset( new ssl_stream{my_impl->thread_pool->get_executor(), *my_impl->ssl_context} ); + } + else { + socket.reset( new tcp::socket( my_impl->thread_pool->get_executor() ) ); + } + } + }; + class connection : public std::enable_shared_from_this { public: explicit connection( string endpoint, bool ssl ); @@ -633,11 +663,8 @@ namespace eosio { }; std::atomic connection_type{both}; - - std::shared_ptr socket; // only accessed through strand after construction - std::shared_ptr ssl_socket; public: - bool use_ssl; + flex_socket socket; // only accessed through strand after construction boost::asio::io_context::strand strand; fc::message_buffer<1024*1024> pending_message_buffer; @@ -829,13 +856,66 @@ namespace eosio { /** returns true if connection is in the security group */ bool is_participating() const { return participating_.load(std::memory_order_relaxed); } - inline tcp::socket& get_socket() { - if (use_ssl){ - EOS_ASSERT(ssl_socket, ssl_configuration_error, "null ssl socket instance"); - return ssl_socket->next_layer(); + bool verify_certificate(bool preverified, ssl::verify_context &ctx) { + + //certificate depth means number of certificate issuers verified current certificate + //openssl provides those one by one starting from root certificate + //we don't use CA certificate or intermidiate issuers, so skipping those + //we interested only in last certificate in the chain, i.e. the one that identifies client + auto depth = X509_STORE_CTX_get_error_depth(ctx.native_handle()); + if (depth > 0) + //preverified is true when certificate matches verification chain, provided via load_verify_file + return preverified; + + //return pointer is managed by openssl + X509 *cert = X509_STORE_CTX_get_current_cert(ctx.native_handle()); + if (!cert) { + fc_dlog(logger, "X509_STORE_CTX_get_current_cert returned null certificate"); + return false; + } + + //max subject size is 256 bytes + char buf[256]; + //return pointer is managed by openssl + X509_NAME* subj = X509_get_subject_name(cert); + int length = X509_NAME_get_text_by_NID(subj, NID_organizationName, buf, std::size(buf)); + + if (length > 0) { + std::string organization{buf, (size_t)length}; + if (is_string_valid_name(organization)){ + account_name participant{organization}; + + std::lock_guard connection_guard(my_impl->connections_mtx); + if(my_impl->security_group.is_in_security_group(participant)) { + if(!is_participating()) { + set_participating(true); + } + + return true; + } + } + + fc_dlog(logger, "received unauthorized participant: ${s}", ("s", organization)); + } + + //printing full subject string of rejected certificate + //openssl doesn't recommend this function for use due to lack of support of unicode and issues with '\' + //however there is no single function that can do the same. + //TODO: investigate usage of X509_NAME_print_ex , compare performance + //see http://openssl.6102.n7.nabble.com/openssl-org-1425-Request-make-X509-NAME-oneline-use-same-formatter-as-X509-NAME-print-ex-td33142.html + char* subject_str = X509_NAME_oneline(subj, buf, std::size(buf)); + fc_dlog(logger, "certificate subject: ${s}", ("s", subject_str ? subject_str : "")); + return false; + } + + void reset_socket() { + socket.reset_socket(); + if (socket.use_ssl) { + socket.ssl_socket->set_verify_callback( + [this](auto bverified, auto& ctx){ + return verify_certificate(bverified, ctx); + }); } - EOS_ASSERT(socket, ssl_configuration_error, "null socket"); - return *socket; } }; @@ -915,20 +995,14 @@ namespace eosio { connection::connection( string endpoint, bool ssl ) : peer_addr( endpoint ), - use_ssl(ssl), + socket(ssl), strand( my_impl->thread_pool->get_executor() ), connection_id( ++my_impl->current_connection_id ), response_expected_timer( my_impl->thread_pool->get_executor() ), last_handshake_recv(), last_handshake_sent() { - if (use_ssl) { - EOS_ASSERT(my_impl->ssl_context, ssl_configuration_error, "ssl context is empty"); - ssl_socket.reset( new ssl_stream{my_impl->thread_pool->get_executor(), *my_impl->ssl_context} ); - } - else { - socket.reset( new tcp::socket( my_impl->thread_pool->get_executor() ) ); - } + reset_socket(); if (endpoint.empty()) fc_dlog( logger, "new connection object created" ); @@ -944,8 +1018,8 @@ namespace eosio { void connection::update_endpoints() { boost::system::error_code ec; boost::system::error_code ec2; - auto rep = socket->remote_endpoint(ec); - auto lep = socket->local_endpoint(ec2); + auto rep = socket.get_socket().remote_endpoint(ec); + auto lep = socket.get_socket().local_endpoint(ec2); std::lock_guard g_conn( conn_mtx ); remote_endpoint_ip = ec ? unknown : rep.address().to_string(); remote_endpoint_port = ec ? unknown : std::to_string(rep.port()); @@ -994,7 +1068,7 @@ namespace eosio { update_endpoints(); boost::asio::ip::tcp::no_delay nodelay( true ); boost::system::error_code ec; - socket->set_option( nodelay, ec ); + socket.get_socket().set_option( nodelay, ec ); if( ec ) { fc_elog( logger, "connection failed (set_option) ${peer}: ${e1}", ("peer", peer_name())( "e1", ec.message() ) ); close(); @@ -1006,12 +1080,15 @@ namespace eosio { socket_open = true; start_read_message(); }; - if (use_ssl) { - ssl_socket->async_handshake(ssl::stream_base::server, - [start_read](const auto& ec){ - EOS_ASSERT(!ec, ssl_handshake_error, "ssl handshake error: ${e}", ("e", ec.message())); + if (socket.use_ssl) { + socket.ssl_socket->async_handshake(ssl::stream_base::server, + boost::asio::bind_executor(strand, [start_read](const auto& ec){ + if (ec) { + fc_elog( logger, "ssl handshake error: ${e}", ("e", ec.message()) ); + return; + } start_read(); - }); + })); } else { start_read(); @@ -1041,14 +1118,12 @@ namespace eosio { void connection::_close( connection* self, bool reconnect, bool shutdown ) { self->socket_open = false; boost::system::error_code ec; - if( self->get_socket().is_open() ) { - self->get_socket().shutdown( tcp::socket::shutdown_both, ec ); - self->get_socket().close( ec ); + tcp::socket& cur_sock = self->socket.get_socket(); + if( cur_sock.is_open() ) { + cur_sock.shutdown( tcp::socket::shutdown_both, ec ); + cur_sock.close( ec ); } - if (self->use_ssl) - self->ssl_socket.reset( new ssl_stream{my_impl->thread_pool->get_executor(), *my_impl->ssl_context} ); - else - self->socket.reset( new tcp::socket( my_impl->thread_pool->get_executor() ) ); + self->reset_socket(); self->flush_queues(); self->connecting = false; @@ -1263,11 +1338,12 @@ namespace eosio { buffer_queue.fill_out_buffer( bufs ); strand.post( [c{std::move(c)}, bufs{std::move(bufs)}]() { - auto write_lambda = [c, psocket=&c->get_socket()]( boost::system::error_code ec, std::size_t w ) { + //capture by value is const so we need mutable keyword + auto write_lambda = [c, socket=c->socket]( boost::system::error_code ec, std::size_t w ) mutable { try { c->buffer_queue.clear_out_queue(); // May have closed connection and cleared buffer_queue - if( !c->socket_is_open() || psocket != &c->get_socket() ) { + if( !c->socket_is_open() || &socket.get_socket() != &c->socket.get_socket() ) { fc_ilog( logger, "async write socket ${r} before callback: ${p}", ("r", c->socket_is_open() ? "changed" : "closed")("p", c->peer_name()) ); c->close(); @@ -1301,11 +1377,11 @@ namespace eosio { } }; - if (c->use_ssl){ - boost::asio::async_write( *c->ssl_socket, bufs, boost::asio::bind_executor( c->strand, write_lambda )); + if (c->socket.use_ssl){ + boost::asio::async_write( *c->socket.ssl_socket, bufs, boost::asio::bind_executor( c->strand, write_lambda )); } else { - boost::asio::async_write( *c->socket, bufs, boost::asio::bind_executor( c->strand, write_lambda )); + boost::asio::async_write( *c->socket.socket, bufs, boost::asio::bind_executor( c->strand, write_lambda )); } }); } @@ -2497,10 +2573,12 @@ namespace eosio { connecting = true; pending_message_buffer.reset(); buffer_queue.clear_out_queue(); - boost::asio::async_connect( get_socket(), endpoints, + boost::asio::async_connect( socket.get_socket(), endpoints, boost::asio::bind_executor( strand, - [resolver, c = shared_from_this(), psocket=&get_socket()]( const boost::system::error_code& err, const tcp::endpoint& endpoint ) { - if( !err && psocket->is_open() && psocket == &c->get_socket() ) { + [resolver, + c = shared_from_this(), + socket=socket]( const boost::system::error_code& err, const tcp::endpoint& endpoint ) mutable { + if( !err && socket.get_socket().is_open() && &socket.get_socket() == &c->socket.get_socket() ) { if( c->start_session() ) { c->send_handshake(); } @@ -2515,13 +2593,13 @@ namespace eosio { connection_ptr new_connection = std::make_shared(ssl_enabled); new_connection->connecting = true; new_connection->strand.post( [this, new_connection = std::move( new_connection )](){ - acceptor->async_accept( new_connection->get_socket(), + acceptor->async_accept( new_connection->socket.get_socket(), boost::asio::bind_executor( new_connection->strand, [new_connection, this]( boost::system::error_code ec ) { if( !ec ) { uint32_t visitors = 0; uint32_t from_addr = 0; boost::system::error_code rec; - tcp::socket& socket = new_connection->get_socket(); + tcp::socket& socket = new_connection->socket.get_socket(); const auto& paddr_add = socket.remote_endpoint( rec ).address(); string paddr_str; if( rec ) { @@ -2592,7 +2670,7 @@ namespace eosio { std::size_t socket_read_watermark = std::min(minimum_read, max_socket_read_watermark); boost::asio::socket_base::receive_low_watermark read_watermark_opt(socket_read_watermark); boost::system::error_code ec; - get_socket().set_option( read_watermark_opt, ec ); + socket.get_socket().set_option( read_watermark_opt, ec ); if( ec ) { fc_elog( logger, "unable to set read watermark ${peer}: ${e1}", ("peer", peer_name())( "e1", ec.message() ) ); } @@ -2613,9 +2691,9 @@ namespace eosio { close( false ); return; } - auto handle_read = [conn = shared_from_this(), psocket=&get_socket()]( boost::system::error_code ec, std::size_t bytes_transferred ) { + auto handle_read = [conn = shared_from_this(), socket=socket]( boost::system::error_code ec, std::size_t bytes_transferred ) mutable { // may have closed connection and cleared pending_message_buffer - if( !conn->socket_is_open() || psocket != &conn->get_socket() ) return; + if( !conn->socket_is_open() || &socket.get_socket() != &conn->socket.get_socket() ) return; bool close_connection = false; try { @@ -2699,14 +2777,14 @@ namespace eosio { conn->close(); } }; - if (use_ssl){ - boost::asio::async_read( *ssl_socket, + if (socket.use_ssl){ + boost::asio::async_read( *socket.ssl_socket, pending_message_buffer.get_buffer_sequence_for_boost_async_read(), completion_handler, boost::asio::bind_executor( strand, handle_read)); } else { - boost::asio::async_read( *socket, + boost::asio::async_read( *socket.socket, pending_message_buffer.get_buffer_sequence_for_boost_async_read(), completion_handler, boost::asio::bind_executor( strand, handle_read)); @@ -3698,32 +3776,30 @@ namespace eosio { return chain::signature_type(); } - ssl_context_ptr net_plugin_impl::create_ssl_context(const std::string& cert, const std::string& pkey, const std::string& ca){ - ssl_context_ptr context( new ssl::context(ssl::context::sslv23) ); + void net_plugin_impl::init_ssl_context(const std::string& cert, const std::string& pkey, const std::string& ca){ + ssl_context.reset( new ssl::context(ssl::context::sslv23) ); - //TLS-only connection - context->set_options(ssl::context::default_workarounds | + //TLS-only connection, no SSL + ssl_context->set_options(ssl::context::default_workarounds | ssl::context::no_sslv2 | ssl::context::no_sslv3 ); - //this ensures peer has valid certificate. no certificate-less connections - context->set_verify_mode(ssl::context::verify_peer | ssl::context::verify_fail_if_no_peer_cert); - context->set_password_callback([](auto,auto){ return "test"; }); + ssl_context->set_password_callback([](auto,auto){ return "test"; }); error_code ec; if (!ca.empty()){ - context->load_verify_file(ca, ec); + ssl_context->load_verify_file(ca, ec); EOS_ASSERT(!ec, ssl_configuration_error, "load_verify_file: ${e}", ("e", ec.message())); + + //this ensures peer has trusted certificate. no certificate-less connections + ssl_context->set_verify_mode(ssl::context::verify_peer | ssl::context::verify_fail_if_no_peer_cert); } - context->use_private_key_file(pkey, ssl::context::pem, ec); + ssl_context->use_private_key_file(pkey, ssl::context::pem, ec); EOS_ASSERT(!ec, ssl_configuration_error, "use_private_key_file: ${e}", ("e", ec.message())); - context->use_certificate_chain_file(cert, ec);//cert_verify_chain.pem // //client1.crt + ssl_context->use_certificate_chain_file(cert, ec);//cert_verify_chain.pem // //client1.crt EOS_ASSERT(!ec, ssl_configuration_error, "use_certificate_chain_file: ${e}", ("e", ec.message())); - - - return context; } // call from connection strand @@ -3816,7 +3892,7 @@ namespace eosio { " _lip \tlocal IP address connected to peer\n\n" " _lport \tlocal port number connected to peer\n\n") ( "p2p-keepalive-interval-ms", bpo::value()->default_value(def_keepalive_interval), "peer heartbeat keepalive message interval in milliseconds") - ( "p2p-tls-ca-certificate-file", bpo::value(), "Certificate Authority's certificate file used for verifying peers TLS connection when security groups feature enabled" ) + ( "p2p-tls-security-group-ca-file", bpo::value(), "Certificate Authority's certificate file used for verifying peers TLS connection when security groups feature enabled" ) ( "p2p-tls-own-certificate-file", bpo::value(), "Certificate file that will be used to authenticate running node if TLS is enabled") ( "p2p-tls-private-key-file", bpo::value(), "Private key file that is used in conjunction with p2p-tls-own-certificate-file for server authorization in TLS connection. Together p2p-tls-private-key-file + p2p-tsl-own-certificate-file automatically enables TLS-only connection for peers.") ; @@ -3931,16 +4007,16 @@ namespace eosio { if ( options.count("p2p-tls-own-certificate-file") ) { auto certificate = options["p2p-tls-own-certificate-file"].as(); auto pkey = options["p2p-tls-private-key-file"].as(); - auto ca_cert = options["p2p-tls-ca-certificate-file"].as(); + auto ca_cert = options["p2p-tls-security-group-ca-file"].as(); EOS_ASSERT(fc::is_regular_file({certificate}), ssl_incomplete_configuration, "p2p-tls-own-certificate-file doesn't contain regular file: ${p}", ("p", certificate)); EOS_ASSERT(fc::is_regular_file({pkey}), ssl_incomplete_configuration, "p2p-tls-private-key-file doesn't contain regular file: ${p}", ("p", pkey)); my->ssl_enabled = true; if (!ca_cert.empty()){ - EOS_ASSERT(fc::is_regular_file({ca_cert}), ssl_incomplete_configuration, "2p-tls-ca-certificate-file doesn't contain regular file: ${p}", ("p", ca_cert)); + EOS_ASSERT(fc::is_regular_file({ca_cert}), ssl_incomplete_configuration, "p2p-tls-security-group-ca-file doesn't contain regular file: ${p}", ("p", ca_cert)); } - my->ssl_context = net_plugin_impl::create_ssl_context(certificate, pkey, ca_cert); + my->init_ssl_context(certificate, pkey, ca_cert); } } FC_LOG_AND_RETHROW() From 1a1e6dd8d4967fb0382f005b25229411c0ddf327 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 18 Mar 2021 15:53:01 -0500 Subject: [PATCH 059/157] Fixed file number for bios. --- tests/Cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index 96bd834daea..117334db2bf 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -254,7 +254,7 @@ def getArguments(number): if Utils.Debug: Utils.Print("adding arguments: {}".format(arguments)) insertSpecificExtraNodeosArgs(node, arguments) - arguments = getArguments(totalNodes + 1) + arguments = getArguments(totalNodes) if Utils.Debug: Utils.Print("adding arguments: {}".format(arguments)) biosNodeNum = -1 insertSpecificExtraNodeosArgs(biosNodeNum, arguments) From e853a8637d52216fc28fd38501cebd82adbf7388 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 18 Mar 2021 17:20:27 -0500 Subject: [PATCH 060/157] Moved useful protocol feature methods into Node.py. --- tests/Node.py | 46 ++++++++++++++++--- ..._multiple_version_protocol_feature_test.py | 34 ++++---------- 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index 7fb6dd50fa3..6844403020b 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1424,16 +1424,50 @@ def isLibAdvancing(): return self.getIrreversibleBlockNum() > currentLib return Utils.waitForTruth(isLibAdvancing, timeout) + def waitUntilBeginningOfProdTurn(self, producerName, timeout=30, sleepTime=0.4): + beginningOfProdTurnHead = 0 + def isDesiredProdTurn(): + beginningOfProdTurnHead = self.getHeadBlockNum() + res = self.getBlock(beginningOfProdTurnHead)["producer"] == producerName and \ + self.getBlock(beginningOfProdTurnHead-1)["producer"] != producerName + return res + ret = Utils.waitForTruth(isDesiredProdTurn, timeout, sleepTime) + assert ret != None, "Expected producer to arrive within {} seconds".format(timeout) + return beginningOfProdTurnHead + # Require producer_api_plugin - def activatePreactivateFeature(self): - protocolFeatureDigestDict = self.getSupportedProtocolFeatureDict() - preactivateFeatureDigest = protocolFeatureDigestDict["PREACTIVATE_FEATURE"]["feature_digest"] - assert preactivateFeatureDigest + def activateFeatures(self, features): + featureDigests = [] + for feature in features: + protocolFeatureDigestDict = self.getSupportedProtocolFeatureDict() + assert feature in protocolFeatureDigestDict + featureDigest = protocolFeatureDigestDict[feature]["feature_digest"] + assert featureDigest + featureDigests.append(featureDigest) - self.scheduleProtocolFeatureActivations([preactivateFeatureDigest]) + self.scheduleProtocolFeatureActivations(featureDigests) # Wait for the next block to be produced so the scheduled protocol feature is activated - assert self.waitForHeadToAdvance(blocksToAdvance=2), print("ERROR: TIMEOUT WAITING FOR PREACTIVATE") + assert self.waitForHeadToAdvance(blocksToAdvance=2), print("ERROR: TIMEOUT WAITING FOR activating features: {}".format(",".join(features))) + + # Require producer_api_plugin + def activatePreactivateFeature(self): + return self.activateFeatures(["PREACTIVATE_FEATURE"]) + + def containsFeatures(self, features): + protocolFeatureDict = self.getSupportedProtocolFeatureDict() + blockHeaderState = self.getLatestBlockHeaderState() + assert blockHeaderState, "blockHeaderState should not be empty" + for feature in features: + featureDigest = protocolFeatureDict[feature]["feature_digest"] + assert featureDigest, "{}'s Digest should not be empty".format(feature) + activatedProtocolFeatures = blockHeaderState["activated_protocol_features"]["protocol_features"] + if featureDigest not in activatedProtocolFeatures: + return False + return True + + def containsPreactivateFeature(self): + return containsFeatures(["PREACTIVATE_FEATURE"]) # Return an array of feature digests to be preactivated in a correct order respecting dependencies # Require producer_api_plugin diff --git a/tests/nodeos_multiple_version_protocol_feature_test.py b/tests/nodeos_multiple_version_protocol_feature_test.py index 384be146fb9..555e9abe512 100755 --- a/tests/nodeos_multiple_version_protocol_feature_test.py +++ b/tests/nodeos_multiple_version_protocol_feature_test.py @@ -43,24 +43,6 @@ def restartNode(node: Node, chainArg=None, addSwapFlags=None, nodeosPath=None): timeout=5, cachePopen=True, nodeosPath=nodeosPath) assert isRelaunchSuccess, "Fail to relaunch" -def shouldNodeContainPreactivateFeature(node): - preactivateFeatureDigest = node.getSupportedProtocolFeatureDict()["PREACTIVATE_FEATURE"]["feature_digest"] - assert preactivateFeatureDigest, "preactivateFeatureDigest should not be empty" - blockHeaderState = node.getLatestBlockHeaderState() - assert blockHeaderState, "blockHeaderState should not be empty" - activatedProtocolFeatures = blockHeaderState["activated_protocol_features"]["protocol_features"] - return preactivateFeatureDigest in activatedProtocolFeatures - -beginningOfProdTurnHead = 0 -def waitUntilBeginningOfProdTurn(node, producerName, timeout=30, sleepTime=0.4): - def isDesiredProdTurn(): - beginningOfProdTurnHead = node.getHeadBlockNum() - res = node.getBlock(beginningOfProdTurnHead)["producer"] == producerName and \ - node.getBlock(beginningOfProdTurnHead-1)["producer"] != producerName - return res - ret = Utils.waitForTruth(isDesiredProdTurn, timeout, sleepTime) - assert ret != None, "Expected producer to arrive within 19 seconds (with 3 other producers)" - def waitForOneRound(): time.sleep(24) # We have 4 producers for this test @@ -176,15 +158,15 @@ def nodeHasBlocks(node, blockIds, blockNums): for i in range(3): Utils.Print("1st node tries activatePreactivateFeature time(s): {}".format(i+1)) # 1st node waits for the start of the production turn each time it tries activatePreactivateFeature() - waitUntilBeginningOfProdTurn(newNodes[0], "defproducera") + beginningOfProdTurnHead = newNodes[0].waitUntilBeginningOfProdTurn("defproducera") newNodes[0].activatePreactivateFeature() - if shouldNodeContainPreactivateFeature(newNodes[0]): + if newNodes[0].containsPreactivateFeature(): break diff = newNodes[0].getInfo()["head_block_num"] - beginningOfProdTurnHead assert diff >= 12, "1st node should contain PREACTIVATE FEATURE since we set it during its production window" - assert shouldNodeContainPreactivateFeature(newNodes[0]), "1st node should contain PREACTIVATE FEATURE" - assert not (shouldNodeContainPreactivateFeature(newNodes[1]) or shouldNodeContainPreactivateFeature(newNodes[2])), \ + assert newNodes[0].containsPreactivateFeature(), "1st node should contain PREACTIVATE FEATURE" + assert not (newNodes[1].containsPreactivateFeature() or newNodes[2].containsPreactivateFeature()), \ "2nd and 3rd node should not contain PREACTIVATE FEATURE" Utils.Print("+++ 2nd, 3rd and 4th node should be in sync, and 1st node should be out of sync +++") assert areNodesInSync([newNodes[1], newNodes[2], oldNode], pauseAll=True, resumeAll=False), "2nd, 3rd and 4th node should be in sync" @@ -192,7 +174,7 @@ def nodeHasBlocks(node, blockIds, blockNums): waitForOneRound() - assert not shouldNodeContainPreactivateFeature(newNodes[0]), "PREACTIVATE_FEATURE should be dropped" + assert not newNodes[0].containsPreactivateFeature(), "PREACTIVATE_FEATURE should be dropped" assert areNodesInSync(allNodes), "All nodes should be in sync" # Then we set the earliest_allowed_activation_time of 2nd node and 3rd node with valid value @@ -203,13 +185,13 @@ def nodeHasBlocks(node, blockIds, blockNums): setValidityOfActTimeSubjRestriction(newNodes[1], "PREACTIVATE_FEATURE", True) setValidityOfActTimeSubjRestriction(newNodes[2], "PREACTIVATE_FEATURE", True) - waitUntilBeginningOfProdTurn(newNodes[0], "defproducera") + newNodes[0].waitUntilBeginningOfProdTurn("defproducera") libBeforePreactivation = newNodes[0].getIrreversibleBlockNum() newNodes[0].activatePreactivateFeature() assert areNodesInSync(newNodes, pauseAll=True, resumeAll=False), "New nodes should be in sync" assert not areNodesInSync(allNodes, pauseAll=False, resumeAll=True), "Nodes should not be in sync after preactivation" - for node in newNodes: assert shouldNodeContainPreactivateFeature(node), "New node should contain PREACTIVATE_FEATURE" + for node in newNodes: assert node.containsPreactivateFeature(), "New node should contain PREACTIVATE_FEATURE" activatedBlockNum = newNodes[0].getHeadBlockNum() # The PREACTIVATE_FEATURE should have been activated before or at this block num assert waitUntilBlockBecomeIrr(newNodes[0], activatedBlockNum), \ @@ -236,7 +218,7 @@ def nodeHasBlocks(node, blockIds, blockNums): time.sleep(2) # Give some time to replay assert areNodesInSync(allNodes), "All nodes should be in sync" - assert shouldNodeContainPreactivateFeature(oldNode), "4th node should contain PREACTIVATE_FEATURE" + assert oldNode.containsPreactivateFeature(), "4th node should contain PREACTIVATE_FEATURE" testSuccessful = True finally: From 26b4c863d84325cf8839b231baa5f662e9c45581 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 18 Mar 2021 17:22:50 -0500 Subject: [PATCH 061/157] Activating the SECURITY_GROUP protocol feature. --- tests/privacy_startup_network.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index c7a11edaa93..5d8340961d3 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -90,12 +90,16 @@ producers = [cluster.getNode(x) for x in range(pnodes) ] relays = [cluster.getNode(pnodes + x) for x in range(pnodes) ] apiNodes = [cluster.getNode(x) for x in apiNodeNums] + blockProducer = None def verifyInSync(producerNum): Utils.Print("Ensure all nodes are in-sync") lib = producers[producerNum].getInfo()["last_irreversible_block_num"] headBlockNum = producers[producerNum].getBlockNum() headBlock = producers[producerNum].getBlock(headBlockNum) + global blockProducer + if blockProducer is None: + blockProducer = headBlock["producer"] Utils.Print("headBlock: {}".format(json.dumps(headBlock, indent=4, sort_keys=True))) headBlockId = headBlock["id"] for prod in producers: @@ -117,6 +121,16 @@ def verifyInSync(producerNum): verifyInSync(producerNum=0) + featureDict = producers[0].getSupportedProtocolFeatureDict() + Utils.Print("feature dict: {}".format(json.dumps(featureDict, indent=4, sort_keys=True))) + + Utils.Print("act feature dict: {}".format(json.dumps(producers[0].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) + timeout = ( pnodes * 12 / 2 ) * 2 # (number of producers * blocks produced / 0.5 blocks per second) * 2 rounds + producers[0].waitUntilBeginningOfProdTurn(blockProducer, timeout=timeout) + feature = "SECURITY_GROUP" + producers[0].activateFeatures([feature]) + assert producers[0].containsFeatures([feature]), "{} feature was not activated".format(feature) + if sanityTest: testSuccessful=True exit(0) From a1a63b2540265f66881b6c8083a1cee71fc05397 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Thu, 18 Mar 2021 20:34:10 -0400 Subject: [PATCH 062/157] bfs added to command line parameters --- plugins/net_plugin/net_plugin.cpp | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index eb76b03ce81..1bcdbc17a20 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -34,6 +35,7 @@ using namespace eosio::chain::plugin_interface; using namespace boost::asio; +namespace bfs = boost::filesystem; namespace eosio { static appbase::abstract_plugin& _net_plugin = app().register_plugin(); @@ -3892,9 +3894,9 @@ namespace eosio { " _lip \tlocal IP address connected to peer\n\n" " _lport \tlocal port number connected to peer\n\n") ( "p2p-keepalive-interval-ms", bpo::value()->default_value(def_keepalive_interval), "peer heartbeat keepalive message interval in milliseconds") - ( "p2p-tls-security-group-ca-file", bpo::value(), "Certificate Authority's certificate file used for verifying peers TLS connection when security groups feature enabled" ) - ( "p2p-tls-own-certificate-file", bpo::value(), "Certificate file that will be used to authenticate running node if TLS is enabled") - ( "p2p-tls-private-key-file", bpo::value(), "Private key file that is used in conjunction with p2p-tls-own-certificate-file for server authorization in TLS connection. Together p2p-tls-private-key-file + p2p-tsl-own-certificate-file automatically enables TLS-only connection for peers.") + ( "p2p-tls-security-group-ca-file", bpo::value(), "Certificate Authority's certificate file used for verifying peers TLS connection when security groups feature enabled" ) + ( "p2p-tls-own-certificate-file", bpo::value(), "Certificate file that will be used to authenticate running node if TLS is enabled") + ( "p2p-tls-private-key-file", bpo::value(), "Private key file that is used in conjunction with p2p-tls-own-certificate-file for server authorization in TLS connection. Together p2p-tls-private-key-file + p2p-tsl-own-certificate-file automatically enables TLS-only connection for peers.") ; } @@ -4005,18 +4007,26 @@ namespace eosio { //if we have certificate option that TLS must be enabled if ( options.count("p2p-tls-own-certificate-file") ) { - auto certificate = options["p2p-tls-own-certificate-file"].as(); - auto pkey = options["p2p-tls-private-key-file"].as(); - auto ca_cert = options["p2p-tls-security-group-ca-file"].as(); + auto certificate = options["p2p-tls-own-certificate-file"].as(); + auto pkey = options["p2p-tls-private-key-file"].as(); + auto ca_cert = options["p2p-tls-security-group-ca-file"].as(); + auto relative_to_absolute = [](bfs::path& file) { + if( file.is_relative()) { + file = bfs::current_path() / file; + } + }; + relative_to_absolute(certificate); + relative_to_absolute(pkey); + relative_to_absolute(ca_cert); - EOS_ASSERT(fc::is_regular_file({certificate}), ssl_incomplete_configuration, "p2p-tls-own-certificate-file doesn't contain regular file: ${p}", ("p", certificate)); - EOS_ASSERT(fc::is_regular_file({pkey}), ssl_incomplete_configuration, "p2p-tls-private-key-file doesn't contain regular file: ${p}", ("p", pkey)); + EOS_ASSERT(fc::is_regular_file(certificate), ssl_incomplete_configuration, "p2p-tls-own-certificate-file doesn't contain regular file: ${p}", ("p", certificate.generic_string())); + EOS_ASSERT(fc::is_regular_file(pkey), ssl_incomplete_configuration, "p2p-tls-private-key-file doesn't contain regular file: ${p}", ("p", pkey.generic_string())); my->ssl_enabled = true; if (!ca_cert.empty()){ - EOS_ASSERT(fc::is_regular_file({ca_cert}), ssl_incomplete_configuration, "p2p-tls-security-group-ca-file doesn't contain regular file: ${p}", ("p", ca_cert)); + EOS_ASSERT(fc::is_regular_file(ca_cert), ssl_incomplete_configuration, "p2p-tls-security-group-ca-file doesn't contain regular file: ${p}", ("p", ca_cert.generic_string())); } - my->init_ssl_context(certificate, pkey, ca_cert); + my->init_ssl_context(certificate.generic_string(), pkey.generic_string(), ca_cert.generic_string()); } } FC_LOG_AND_RETHROW() From 01b56efb0b40d99bd31891025b4d4f6b6058c812 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Thu, 18 Mar 2021 22:09:57 -0400 Subject: [PATCH 063/157] change of commandline parameter in tests --- tests/Cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index 117334db2bf..aa8e653ca91 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -247,7 +247,7 @@ def insertSpecificExtraNodeosArgs(node, insertStr): def getArguments(number): nodeCert = os.path.join(privacyDir, "node{}.crt".format(number)) nodeKey = os.path.join(privacyDir, "node{}_key.pem".format(number)) - return "--p2p-tls-own-certificate-file {} --p2p-tls-private-key-file {} --p2p-tls-ca-certificate-file {}".format(nodeCert, nodeKey, certAuth) + return "--p2p-tls-own-certificate-file {} --p2p-tls-private-key-file {} --p2p-tls-security-group-ca-file {}".format(nodeCert, nodeKey, certAuth) for node in range(totalNodes): arguments = getArguments(node) From fccc758ded4947b56f6704c52682b0ee1a7bd6f7 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Fri, 19 Mar 2021 14:04:18 -0400 Subject: [PATCH 064/157] optimization & bugfixes in net_plugin --- plugins/net_plugin/net_plugin.cpp | 77 ++++++++++++++++++------------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 1bcdbc17a20..f1fb8b2d40b 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -599,31 +599,44 @@ namespace eosio { }; struct flex_socket { - std::shared_ptr socket; - std::shared_ptr ssl_socket; + using socket_ptr = std::shared_ptr; + using ssl_socket_ptr = std::shared_ptr; + using socket_variant = std::variant; + + private: + socket_variant socket; + public: const bool use_ssl; flex_socket(bool ssl) : use_ssl(ssl) { reset_socket(); } - inline tcp::socket& get_socket() { + inline tcp::socket& raw_socket() { if (use_ssl){ - EOS_ASSERT(ssl_socket, ssl_configuration_error, "null ssl socket instance"); - return ssl_socket->next_layer(); + return ssl_socket()->next_layer(); } - EOS_ASSERT(socket, ssl_configuration_error, "null socket"); - return *socket; + return *tcp_socket(); + } + + inline socket_ptr& tcp_socket() { + return std::get(socket); + } + + inline ssl_socket_ptr& ssl_socket() { + return std::get(socket); } void reset_socket() { if (use_ssl) { + socket = ssl_socket_ptr{}; EOS_ASSERT(my_impl->ssl_context, ssl_configuration_error, "ssl context is empty"); - ssl_socket.reset( new ssl_stream{my_impl->thread_pool->get_executor(), *my_impl->ssl_context} ); + ssl_socket().reset( new ssl_stream{my_impl->thread_pool->get_executor(), *my_impl->ssl_context} ); } else { - socket.reset( new tcp::socket( my_impl->thread_pool->get_executor() ) ); + socket = socket_ptr{}; + tcp_socket().reset( new tcp::socket( my_impl->thread_pool->get_executor() ) ); } } }; @@ -913,7 +926,7 @@ namespace eosio { void reset_socket() { socket.reset_socket(); if (socket.use_ssl) { - socket.ssl_socket->set_verify_callback( + socket.ssl_socket()->set_verify_callback( [this](auto bverified, auto& ctx){ return verify_certificate(bverified, ctx); }); @@ -1020,8 +1033,8 @@ namespace eosio { void connection::update_endpoints() { boost::system::error_code ec; boost::system::error_code ec2; - auto rep = socket.get_socket().remote_endpoint(ec); - auto lep = socket.get_socket().local_endpoint(ec2); + auto rep = socket.raw_socket().remote_endpoint(ec); + auto lep = socket.raw_socket().local_endpoint(ec2); std::lock_guard g_conn( conn_mtx ); remote_endpoint_ip = ec ? unknown : rep.address().to_string(); remote_endpoint_port = ec ? unknown : std::to_string(rep.port()); @@ -1070,7 +1083,7 @@ namespace eosio { update_endpoints(); boost::asio::ip::tcp::no_delay nodelay( true ); boost::system::error_code ec; - socket.get_socket().set_option( nodelay, ec ); + socket.raw_socket().set_option( nodelay, ec ); if( ec ) { fc_elog( logger, "connection failed (set_option) ${peer}: ${e1}", ("peer", peer_name())( "e1", ec.message() ) ); close(); @@ -1083,8 +1096,11 @@ namespace eosio { start_read_message(); }; if (socket.use_ssl) { - socket.ssl_socket->async_handshake(ssl::stream_base::server, - boost::asio::bind_executor(strand, [start_read](const auto& ec){ + socket.ssl_socket()->async_handshake(ssl::stream_base::server, + boost::asio::bind_executor(strand, [start_read, socket=socket](const auto& ec){ + //we use socket just to retain connection shared_ptr just in case it will be deleted + //when we will need it inside start_read_message + std::ignore = socket; if (ec) { fc_elog( logger, "ssl handshake error: ${e}", ("e", ec.message()) ); return; @@ -1120,7 +1136,7 @@ namespace eosio { void connection::_close( connection* self, bool reconnect, bool shutdown ) { self->socket_open = false; boost::system::error_code ec; - tcp::socket& cur_sock = self->socket.get_socket(); + tcp::socket& cur_sock = self->socket.raw_socket(); if( cur_sock.is_open() ) { cur_sock.shutdown( tcp::socket::shutdown_both, ec ); cur_sock.close( ec ); @@ -1345,7 +1361,7 @@ namespace eosio { try { c->buffer_queue.clear_out_queue(); // May have closed connection and cleared buffer_queue - if( !c->socket_is_open() || &socket.get_socket() != &c->socket.get_socket() ) { + if( !c->socket_is_open() || &socket.raw_socket() != &c->socket.raw_socket() ) { fc_ilog( logger, "async write socket ${r} before callback: ${p}", ("r", c->socket_is_open() ? "changed" : "closed")("p", c->peer_name()) ); c->close(); @@ -1380,10 +1396,10 @@ namespace eosio { }; if (c->socket.use_ssl){ - boost::asio::async_write( *c->socket.ssl_socket, bufs, boost::asio::bind_executor( c->strand, write_lambda )); + boost::asio::async_write( *c->socket.ssl_socket(), bufs, boost::asio::bind_executor( c->strand, write_lambda )); } else { - boost::asio::async_write( *c->socket.socket, bufs, boost::asio::bind_executor( c->strand, write_lambda )); + boost::asio::async_write( *c->socket.tcp_socket(), bufs, boost::asio::bind_executor( c->strand, write_lambda )); } }); } @@ -2575,12 +2591,12 @@ namespace eosio { connecting = true; pending_message_buffer.reset(); buffer_queue.clear_out_queue(); - boost::asio::async_connect( socket.get_socket(), endpoints, + boost::asio::async_connect( socket.raw_socket(), endpoints, boost::asio::bind_executor( strand, [resolver, c = shared_from_this(), socket=socket]( const boost::system::error_code& err, const tcp::endpoint& endpoint ) mutable { - if( !err && socket.get_socket().is_open() && &socket.get_socket() == &c->socket.get_socket() ) { + if( !err && socket.raw_socket().is_open() && &socket.raw_socket() == &c->socket.raw_socket() ) { if( c->start_session() ) { c->send_handshake(); } @@ -2595,14 +2611,13 @@ namespace eosio { connection_ptr new_connection = std::make_shared(ssl_enabled); new_connection->connecting = true; new_connection->strand.post( [this, new_connection = std::move( new_connection )](){ - acceptor->async_accept( new_connection->socket.get_socket(), - boost::asio::bind_executor( new_connection->strand, [new_connection, this]( boost::system::error_code ec ) { + acceptor->async_accept( new_connection->socket.raw_socket(), + boost::asio::bind_executor( new_connection->strand, [new_connection, socket=new_connection->socket, this]( boost::system::error_code ec ) mutable { if( !ec ) { uint32_t visitors = 0; uint32_t from_addr = 0; boost::system::error_code rec; - tcp::socket& socket = new_connection->socket.get_socket(); - const auto& paddr_add = socket.remote_endpoint( rec ).address(); + const auto& paddr_add = socket.raw_socket().remote_endpoint( rec ).address(); string paddr_str; if( rec ) { fc_elog( logger, "Error getting remote endpoint: ${m}", ("m", rec.message())); @@ -2637,8 +2652,8 @@ namespace eosio { } // new_connection never added to connections and start_session not called, lifetime will end boost::system::error_code ec; - socket.shutdown( tcp::socket::shutdown_both, ec ); - socket.close( ec ); + socket.raw_socket().shutdown( tcp::socket::shutdown_both, ec ); + socket.raw_socket().close( ec ); } } } else { @@ -2672,7 +2687,7 @@ namespace eosio { std::size_t socket_read_watermark = std::min(minimum_read, max_socket_read_watermark); boost::asio::socket_base::receive_low_watermark read_watermark_opt(socket_read_watermark); boost::system::error_code ec; - socket.get_socket().set_option( read_watermark_opt, ec ); + socket.raw_socket().set_option( read_watermark_opt, ec ); if( ec ) { fc_elog( logger, "unable to set read watermark ${peer}: ${e1}", ("peer", peer_name())( "e1", ec.message() ) ); } @@ -2695,7 +2710,7 @@ namespace eosio { } auto handle_read = [conn = shared_from_this(), socket=socket]( boost::system::error_code ec, std::size_t bytes_transferred ) mutable { // may have closed connection and cleared pending_message_buffer - if( !conn->socket_is_open() || &socket.get_socket() != &conn->socket.get_socket() ) return; + if( !conn->socket_is_open() || &socket.raw_socket() != &conn->socket.raw_socket() ) return; bool close_connection = false; try { @@ -2780,13 +2795,13 @@ namespace eosio { } }; if (socket.use_ssl){ - boost::asio::async_read( *socket.ssl_socket, + boost::asio::async_read( *socket.ssl_socket(), pending_message_buffer.get_buffer_sequence_for_boost_async_read(), completion_handler, boost::asio::bind_executor( strand, handle_read)); } else { - boost::asio::async_read( *socket.socket, + boost::asio::async_read( *socket.tcp_socket(), pending_message_buffer.get_buffer_sequence_for_boost_async_read(), completion_handler, boost::asio::bind_executor( strand, handle_read)); From d56ba12375701d1f1bfadff26e96c6d0d8d89932 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Mon, 22 Mar 2021 22:46:50 -0400 Subject: [PATCH 065/157] bugfixes + refactoring of net plugin --- plugins/net_plugin/net_plugin.cpp | 105 +++++++++++++++++------------- 1 file changed, 59 insertions(+), 46 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index f1fb8b2d40b..6f765e8b1fa 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -60,7 +60,7 @@ namespace eosio { using io_work_t = boost::asio::executor_work_guard; using ssl_context_ptr = std::unique_ptr; - using ssl_stream = ssl::stream; + template void verify_strand_in_this_thread(const Strand& strand, const char* func, int line) { @@ -600,23 +600,22 @@ namespace eosio { struct flex_socket { using socket_ptr = std::shared_ptr; + using ssl_stream = ssl::stream; using ssl_socket_ptr = std::shared_ptr; using socket_variant = std::variant; private: socket_variant socket; public: - const bool use_ssl; - - flex_socket(bool ssl) : use_ssl(ssl) { + flex_socket() { reset_socket(); } - inline tcp::socket& raw_socket() { - if (use_ssl){ - return ssl_socket()->next_layer(); + inline tcp::socket::lowest_layer_type& raw_socket() { + if (my_impl->ssl_enabled){ + return ssl_socket()->lowest_layer(); } - return *tcp_socket(); + return tcp_socket()->lowest_layer(); } inline socket_ptr& tcp_socket() { @@ -629,7 +628,7 @@ namespace eosio { void reset_socket() { - if (use_ssl) { + if (my_impl->ssl_enabled) { socket = ssl_socket_ptr{}; EOS_ASSERT(my_impl->ssl_context, ssl_configuration_error, "ssl context is empty"); ssl_socket().reset( new ssl_stream{my_impl->thread_pool->get_executor(), *my_impl->ssl_context} ); @@ -643,12 +642,12 @@ namespace eosio { class connection : public std::enable_shared_from_this { public: - explicit connection( string endpoint, bool ssl ); - explicit connection(bool ssl); + explicit connection( string endpoint ); + connection(); ~connection() {} - bool start_session(); + bool start_session(bool server); bool socket_is_open() const { return socket_open.load(); } // thread safe, atomic const string& peer_address() const { return peer_addr; } // thread safe, const @@ -871,6 +870,22 @@ namespace eosio { /** returns true if connection is in the security group */ bool is_participating() const { return participating_.load(std::memory_order_relaxed); } + std::string certificate_subject(ssl::verify_context &ctx) { + X509 *cert = X509_STORE_CTX_get_current_cert(ctx.native_handle()); + if (!cert) + return ""; + + char buf[256]; + //printing full subject string of rejected certificate + //openssl doesn't recommend this function for use due to lack of support of unicode and issues with '\' + //however there is no single function that can do the same. + //TODO: investigate usage of X509_NAME_print_ex , compare performance + //see http://openssl.6102.n7.nabble.com/openssl-org-1425-Request-make-X509-NAME-oneline-use-same-formatter-as-X509-NAME-print-ex-td33142.html + char* subject_str = X509_NAME_oneline(X509_get_subject_name(cert), buf, std::size(buf)); + + return {subject_str}; + } + bool verify_certificate(bool preverified, ssl::verify_context &ctx) { //certificate depth means number of certificate issuers verified current certificate @@ -878,9 +893,11 @@ namespace eosio { //we don't use CA certificate or intermidiate issuers, so skipping those //we interested only in last certificate in the chain, i.e. the one that identifies client auto depth = X509_STORE_CTX_get_error_depth(ctx.native_handle()); - if (depth > 0) + if (depth > 0) { + fc_dlog(logger, "preverified: ${p} certificate subject: ${s}", ("p", preverified)("s", certificate_subject(ctx))); //preverified is true when certificate matches verification chain, provided via load_verify_file return preverified; + } //return pointer is managed by openssl X509 *cert = X509_STORE_CTX_get_current_cert(ctx.native_handle()); @@ -899,33 +916,26 @@ namespace eosio { std::string organization{buf, (size_t)length}; if (is_string_valid_name(organization)){ account_name participant{organization}; - + std::lock_guard connection_guard(my_impl->connections_mtx); if(my_impl->security_group.is_in_security_group(participant)) { - if(!is_participating()) { - set_participating(true); - } - + set_participating(true); + fc_dlog(logger, "participant added: ${o}", ("o", organization)); return true; } } - + fc_dlog(logger, "received unauthorized participant: ${s}", ("s", organization)); } - //printing full subject string of rejected certificate - //openssl doesn't recommend this function for use due to lack of support of unicode and issues with '\' - //however there is no single function that can do the same. - //TODO: investigate usage of X509_NAME_print_ex , compare performance - //see http://openssl.6102.n7.nabble.com/openssl-org-1425-Request-make-X509-NAME-oneline-use-same-formatter-as-X509-NAME-print-ex-td33142.html - char* subject_str = X509_NAME_oneline(subj, buf, std::size(buf)); - fc_dlog(logger, "certificate subject: ${s}", ("s", subject_str ? subject_str : "")); + fc_dlog(logger, "certificate subject: ${s}", ("s", certificate_subject(ctx))); + set_participating(false); return false; } void reset_socket() { socket.reset_socket(); - if (socket.use_ssl) { + if (my_impl->ssl_enabled) { socket.ssl_socket()->set_verify_callback( [this](auto bverified, auto& ctx){ return verify_certificate(bverified, ctx); @@ -1008,9 +1018,9 @@ namespace eosio { //--------------------------------------------------------------------------- - connection::connection( string endpoint, bool ssl ) + connection::connection( string endpoint ) : peer_addr( endpoint ), - socket(ssl), + socket(), strand( my_impl->thread_pool->get_executor() ), connection_id( ++my_impl->current_connection_id ), response_expected_timer( my_impl->thread_pool->get_executor() ), @@ -1025,8 +1035,8 @@ namespace eosio { fc_ilog( logger, "created connection to ${n}", ("n", endpoint) ); } - connection::connection(bool ssl) - : connection({}, ssl) + connection::connection() + : connection(string{}) { } @@ -1077,7 +1087,7 @@ namespace eosio { return stat; } - bool connection::start_session() { + bool connection::start_session(bool server) { verify_strand_in_this_thread( strand, __func__, __LINE__ ); update_endpoints(); @@ -1095,14 +1105,15 @@ namespace eosio { socket_open = true; start_read_message(); }; - if (socket.use_ssl) { - socket.ssl_socket()->async_handshake(ssl::stream_base::server, - boost::asio::bind_executor(strand, [start_read, socket=socket](const auto& ec){ + if (my_impl->ssl_enabled) { + socket.ssl_socket()->async_handshake(server ? ssl::stream_base::server : ssl::stream_base::client, + boost::asio::bind_executor(strand, [start_read, c = shared_from_this(), socket=socket](const auto& ec){ //we use socket just to retain connection shared_ptr just in case it will be deleted //when we will need it inside start_read_message std::ignore = socket; if (ec) { fc_elog( logger, "ssl handshake error: ${e}", ("e", ec.message()) ); + c->close(); return; } start_read(); @@ -1136,7 +1147,7 @@ namespace eosio { void connection::_close( connection* self, bool reconnect, bool shutdown ) { self->socket_open = false; boost::system::error_code ec; - tcp::socket& cur_sock = self->socket.raw_socket(); + auto& cur_sock = self->socket.raw_socket(); if( cur_sock.is_open() ) { cur_sock.shutdown( tcp::socket::shutdown_both, ec ); cur_sock.close( ec ); @@ -1395,7 +1406,7 @@ namespace eosio { } }; - if (c->socket.use_ssl){ + if (my_impl->ssl_enabled){ boost::asio::async_write( *c->socket.ssl_socket(), bufs, boost::asio::bind_executor( c->strand, write_lambda )); } else { @@ -2597,7 +2608,7 @@ namespace eosio { c = shared_from_this(), socket=socket]( const boost::system::error_code& err, const tcp::endpoint& endpoint ) mutable { if( !err && socket.raw_socket().is_open() && &socket.raw_socket() == &c->socket.raw_socket() ) { - if( c->start_session() ) { + if( c->start_session(false) ) { c->send_handshake(); } } else { @@ -2608,7 +2619,7 @@ namespace eosio { } void net_plugin_impl::start_listen_loop() { - connection_ptr new_connection = std::make_shared(ssl_enabled); + connection_ptr new_connection = std::make_shared(); new_connection->connecting = true; new_connection->strand.post( [this, new_connection = std::move( new_connection )](){ acceptor->async_accept( new_connection->socket.raw_socket(), @@ -2638,7 +2649,7 @@ namespace eosio { if( from_addr < max_nodes_per_host && (max_client_count == 0 || visitors < max_client_count)) { fc_ilog( logger, "Accepted new connection: " + paddr_str ); new_connection->set_heartbeat_timeout( heartbeat_timeout ); - if( new_connection->start_session()) { + if( new_connection->start_session(true)) { std::lock_guard g_unique( connections_mtx ); connections.insert( new_connection ); } @@ -2794,7 +2805,7 @@ namespace eosio { conn->close(); } }; - if (socket.use_ssl){ + if (my_impl->ssl_enabled){ boost::asio::async_read( *socket.ssl_socket(), pending_message_buffer.get_buffer_sequence_for_boost_async_read(), completion_handler, @@ -3805,17 +3816,19 @@ namespace eosio { error_code ec; if (!ca.empty()){ + fc_dlog(logger, "using verify file: ${p}", ("p", ca)); ssl_context->load_verify_file(ca, ec); EOS_ASSERT(!ec, ssl_configuration_error, "load_verify_file: ${e}", ("e", ec.message())); //this ensures peer has trusted certificate. no certificate-less connections ssl_context->set_verify_mode(ssl::context::verify_peer | ssl::context::verify_fail_if_no_peer_cert); } - + fc_dlog(logger, "using private key file: ${p}", ("p", pkey)); ssl_context->use_private_key_file(pkey, ssl::context::pem, ec); EOS_ASSERT(!ec, ssl_configuration_error, "use_private_key_file: ${e}", ("e", ec.message())); - ssl_context->use_certificate_chain_file(cert, ec);//cert_verify_chain.pem // //client1.crt + fc_dlog(logger, "using chain file: ${p}", ("p", cert)); + ssl_context->use_certificate_chain_file(cert, ec); EOS_ASSERT(!ec, ssl_configuration_error, "use_certificate_chain_file: ${e}", ("e", ec.message())); } @@ -4027,7 +4040,7 @@ namespace eosio { auto ca_cert = options["p2p-tls-security-group-ca-file"].as(); auto relative_to_absolute = [](bfs::path& file) { if( file.is_relative()) { - file = bfs::current_path() / file; + file = bfs::current_path() / file; } }; relative_to_absolute(certificate); @@ -4040,7 +4053,7 @@ namespace eosio { if (!ca_cert.empty()){ EOS_ASSERT(fc::is_regular_file(ca_cert), ssl_incomplete_configuration, "p2p-tls-security-group-ca-file doesn't contain regular file: ${p}", ("p", ca_cert.generic_string())); } - + my->init_ssl_context(certificate.generic_string(), pkey.generic_string(), ca_cert.generic_string()); } @@ -4201,7 +4214,7 @@ namespace eosio { if( my->find_connection( host ) ) return "already connected"; - connection_ptr c = std::make_shared( host, my->ssl_enabled ); + connection_ptr c = std::make_shared( host ); fc_dlog( logger, "calling active connector: ${h}", ("h", host) ); if( c->resolve_and_connect() ) { fc_dlog( logger, "adding new connection to the list: ${c}", ("c", c->peer_name()) ); From b4cd62ec04883b60000a93536769484f97794261 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Wed, 24 Mar 2021 23:39:45 -0400 Subject: [PATCH 066/157] bugfixes to net_plugin tls and tests --- plugins/net_plugin/net_plugin.cpp | 16 ++++++++-------- tests/Cluster.py | 24 ++++++++++++++++++++++-- tests/generate-certificates.sh | 29 ++++++++++++++++++++++++++--- 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 6f765e8b1fa..78e6e4b4a13 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -887,14 +887,13 @@ namespace eosio { } bool verify_certificate(bool preverified, ssl::verify_context &ctx) { - + peer_dlog(this, "preverified: ${p} certificate subject: ${s}", ("p", preverified)("s", certificate_subject(ctx))); //certificate depth means number of certificate issuers verified current certificate //openssl provides those one by one starting from root certificate //we don't use CA certificate or intermidiate issuers, so skipping those //we interested only in last certificate in the chain, i.e. the one that identifies client auto depth = X509_STORE_CTX_get_error_depth(ctx.native_handle()); if (depth > 0) { - fc_dlog(logger, "preverified: ${p} certificate subject: ${s}", ("p", preverified)("s", certificate_subject(ctx))); //preverified is true when certificate matches verification chain, provided via load_verify_file return preverified; } @@ -902,7 +901,7 @@ namespace eosio { //return pointer is managed by openssl X509 *cert = X509_STORE_CTX_get_current_cert(ctx.native_handle()); if (!cert) { - fc_dlog(logger, "X509_STORE_CTX_get_current_cert returned null certificate"); + peer_dlog(this, "X509_STORE_CTX_get_current_cert returned null certificate"); return false; } @@ -920,17 +919,18 @@ namespace eosio { std::lock_guard connection_guard(my_impl->connections_mtx); if(my_impl->security_group.is_in_security_group(participant)) { set_participating(true); - fc_dlog(logger, "participant added: ${o}", ("o", organization)); + peer_dlog(this, "participant added: ${o}", ("o", organization)); return true; } } - fc_dlog(logger, "received unauthorized participant: ${s}", ("s", organization)); + peer_dlog(this, "received unauthorized participant: ${s}", ("s", organization)); } - - fc_dlog(logger, "certificate subject: ${s}", ("s", certificate_subject(ctx))); + set_participating(false); - return false; + //we keep connection if peer has valid certificate but participant name is not authorized + //however that connection doesn't receive updates + return preverified; } void reset_socket() { diff --git a/tests/Cluster.py b/tests/Cluster.py index aa8e653ca91..694dee85e9a 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -245,8 +245,28 @@ def insertSpecificExtraNodeosArgs(node, insertStr): certAuth = os.path.join(privacyDir, "CA_cert.pem") def getArguments(number): - nodeCert = os.path.join(privacyDir, "node{}.crt".format(number)) - nodeKey = os.path.join(privacyDir, "node{}_key.pem".format(number)) + # this function converts number to eos name string + # eos name can have only numbers 1-5 + # e.g. 0 -> 1, 6 -> 5.1, 12 -> 5.5.2 + def normalizeNumber(number): + if number <= 0: + number = 1 + if number <= 5: + return str(number) + cnt = number + ret = "5" + while cnt > 5: + cnt = cnt - 5 + if cnt > 5: + ret = "{}.5".format(ret) + else: + ret = "{}.{}".format(ret, cnt) + assert(len(ret) <= 13) + + return ret + + nodeCert = os.path.join(privacyDir, "node{}.crt".format(normalizeNumber(number+1))) + nodeKey = os.path.join(privacyDir, "node{}_key.pem".format(normalizeNumber(number+1))) return "--p2p-tls-own-certificate-file {} --p2p-tls-private-key-file {} --p2p-tls-security-group-ca-file {}".format(nodeCert, nodeKey, certAuth) for node in range(totalNodes): diff --git a/tests/generate-certificates.sh b/tests/generate-certificates.sh index 99ef8124d64..01d4a6dd565 100755 --- a/tests/generate-certificates.sh +++ b/tests/generate-certificates.sh @@ -48,6 +48,29 @@ function get-algo-str { fi } +function normalized-name { + if [ $2 -le 5 ] + then + NAME=$(sed "s/{NUMBER}/$2/" <<< "$1") + else + CNT=$2 + NUM="5" + while [ $CNT -gt 5 ] + do + CNT=$(( $CNT - 5 )) + if [ $CNT -gt 5 ] + then + NUM="${NUM}.5" + else + NUM="${NUM}.${CNT}" + fi + done + NAME=$(sed "s/{NUMBER}/$NUM/" <<< "$1") + fi + + echo $NAME +} + if [[ $1 == "--help" ]] then echo "Usage:" @@ -92,10 +115,10 @@ echo " generating nodes certificates " echo "*************************************************" #client certificate requests + private keys -for n in $(seq 0 $(($GROUP_SIZE-1)) ) +for n in $(seq 1 $(($GROUP_SIZE)) ) do - ORG_NAME=$(sed "s/{NUMBER}/$n/" <<< "$ORG_MASK") - CN_NAME=$(sed "s/{NUMBER}/$n/" <<< "$CN_MASK") + ORG_NAME=$(normalized-name "$ORG_MASK" $n) + CN_NAME=$(normalized-name "$CN_MASK" $n) echo "*************************************************" echo "generating certificate for $ORG_NAME / $CN_NAME " echo "*************************************************" From e7f2276e4c7016cd78040e5898d19c08cd8d21f8 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 29 Mar 2021 14:13:00 -0500 Subject: [PATCH 067/157] Minor changes to add accounts and load contract to exercise security group. --- tests/privacy_startup_network.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index 5d8340961d3..ed9aa478548 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -90,6 +90,20 @@ producers = [cluster.getNode(x) for x in range(pnodes) ] relays = [cluster.getNode(pnodes + x) for x in range(pnodes) ] apiNodes = [cluster.getNode(x) for x in apiNodeNums] + + def createAccount(newAcc): + producers[0].createInitializeAccount(newAcc, cluster.eosioAccount) + ignWallet = cluster.walletMgr.create("ignition") # will actually just look up the wallet + cluster.walletMgr.importKey(newAcc, ignWallet) + + numAccounts = 4 + testAccounts = Cluster.createAccountKeys(numAccounts) + accountPrefix = "testaccount" + for i in range(numAccounts): + testAccount = testAccounts[i] + testAccount.name = accountPrefix + str(i + 1) + createAccount(testAccount) + blockProducer = None def verifyInSync(producerNum): @@ -135,6 +149,12 @@ def verifyInSync(producerNum): testSuccessful=True exit(0) + def publishContract(account, wasmFile, waitForTransBlock=False): + Print("Publish contract") + return producers[0].publishContract(account, "unittests/test-contracts/security_group_test/", wasmFile, abiFile=None, waitForTransBlock=waitForTransBlock) + + publishContract(testAccounts[0], 'security_group_test.wasm', waitForTransBlock=True) + testSuccessful=True finally: TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails) From 718712981afd4def978cdfdf9bac06ed6a1c3fab Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 29 Mar 2021 14:14:24 -0500 Subject: [PATCH 068/157] Change configSecurityGroup to automatically NOT use the bios bootfile and also to inject a delay to work around 30 second delay we are seeing. --- tests/Cluster.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index 117334db2bf..eb2c3cf7b79 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -553,8 +553,9 @@ def connectGroup(group, producerNodes, bridgeNodes) : Utils.Print("Bootstrap cluster.") if not loadSystemContract: useBiosBootFile=False #ensure we use Cluster.bootstrap - if onlyBios or not useBiosBootFile: - self.biosNode=self.bootstrap(biosNode, startedNodes, prodCount + sharedProducers, totalProducers, pfSetupPolicy, onlyBios, onlySetProds, loadSystemContract, manualProducerNodeConf) + if onlyBios or not useBiosBootFile or configSecurityGroup: + delayProductionTransfer = 35 if configSecurityGroup else None # when TLS delay is analyzed, then this delay and ignoring of useBiosBootFile can be removed + self.biosNode=self.bootstrap(biosNode, startedNodes, prodCount + sharedProducers, totalProducers, pfSetupPolicy, onlyBios, onlySetProds, loadSystemContract, manualProducerNodeConf, delayProductionTransfer=delayProductionTransfer) if self.biosNode is None: Utils.Print("ERROR: Bootstrap failed.") return False @@ -1160,7 +1161,7 @@ def bios_bootstrap(self, biosNode, totalNodes, pfSetupPolicy, silent=False): return biosNode - def bootstrap(self, biosNode, totalNodes, prodCount, totalProducers, pfSetupPolicy, onlyBios=False, onlySetProds=False, loadSystemContract=True, manualProducerNodeConf=[]): + def bootstrap(self, biosNode, totalNodes, prodCount, totalProducers, pfSetupPolicy, onlyBios=False, onlySetProds=False, loadSystemContract=True, manualProducerNodeConf=[], delayProductionTransfer=None): """Create 'prodCount' init accounts and deposits 10000000000 SYS in each. If prodCount is -1 will initialize all possible producers. Ensure nodes are inter-connected prior to this call. One way to validate this will be to check if every node has block 1.""" @@ -1290,6 +1291,10 @@ def bootstrap(self, biosNode, totalNodes, prodCount, totalProducers, pfSetupPoli if Utils.Debug: Utils.Print("setprods: %s" % (setProdsStr)) Utils.Print("Setting producers: %s." % (", ".join(prodNames))) opts="--permission eosio@active" + + if delayProductionTransfer: + time.sleep(delayProductionTransfer) + # pylint: disable=redefined-variable-type trans=biosNode.pushMessage("eosio", "setprods", setProdsStr, opts) if trans is None or not trans[0]: From 90348f412c0e00c9e2570b30392b85881d0349ad Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 29 Mar 2021 15:49:37 -0500 Subject: [PATCH 069/157] Added improved error handling. --- tests/Node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/Node.py b/tests/Node.py index 6844403020b..aed9123d479 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1509,6 +1509,8 @@ def getLatestBlockHeaderState(self): def getActivatedProtocolFeatures(self): latestBlockHeaderState = self.getLatestBlockHeaderState() + if "activated_protocol_features" not in latestBlockHeaderState or "protocol_features" not in latestBlockHeaderState["activated_protocol_features"]: + Utils.errorExit("getLatestBlockHeaderState did not return expected output, should contain [\"activated_protocol_features\"][\"protocol_features\"]: {}".format(latestBlockHeaderState)) return latestBlockHeaderState["activated_protocol_features"]["protocol_features"] def modifyBuiltinPFSubjRestrictions(self, featureCodename, subjectiveRestriction={}): From 86b65ed6f6a0d614d2af4b4cf12d2797b5705fc3 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 29 Mar 2021 16:12:52 -0500 Subject: [PATCH 070/157] Fixing PR comment, and cleaning up uses of global that should be nonlocal. --- tests/Node.py | 1 + tests/nodeos_high_transaction_test.py | 2 +- tests/privacy_startup_network.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index aed9123d479..54d4179b2eb 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1427,6 +1427,7 @@ def isLibAdvancing(): def waitUntilBeginningOfProdTurn(self, producerName, timeout=30, sleepTime=0.4): beginningOfProdTurnHead = 0 def isDesiredProdTurn(): + nonlocal beginningOfProdTurnHead beginningOfProdTurnHead = self.getHeadBlockNum() res = self.getBlock(beginningOfProdTurnHead)["producer"] == producerName and \ self.getBlock(beginningOfProdTurnHead-1)["producer"] != producerName diff --git a/tests/nodeos_high_transaction_test.py b/tests/nodeos_high_transaction_test.py index 6108af9ffa0..1afaff11675 100755 --- a/tests/nodeos_high_transaction_test.py +++ b/tests/nodeos_high_transaction_test.py @@ -162,7 +162,7 @@ lastIrreversibleBlockNum = None def cacheTransIdInBlock(transId, transToBlock, node): - global lastIrreversibleBlockNum + nonlocal lastIrreversibleBlockNum lastPassLIB = None blockWaitTimeout = 60 transTimeDelayed = False diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index ed9aa478548..382e7921d12 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -111,7 +111,7 @@ def verifyInSync(producerNum): lib = producers[producerNum].getInfo()["last_irreversible_block_num"] headBlockNum = producers[producerNum].getBlockNum() headBlock = producers[producerNum].getBlock(headBlockNum) - global blockProducer + nonlocal blockProducer if blockProducer is None: blockProducer = headBlock["producer"] Utils.Print("headBlock: {}".format(json.dumps(headBlock, indent=4, sort_keys=True))) From b5071ccd92402d86dfccd4c89b67c478da1a9e50 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 29 Mar 2021 17:32:18 -0500 Subject: [PATCH 071/157] Could not use nonlocal in this case. --- tests/privacy_startup_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index 382e7921d12..ed9aa478548 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -111,7 +111,7 @@ def verifyInSync(producerNum): lib = producers[producerNum].getInfo()["last_irreversible_block_num"] headBlockNum = producers[producerNum].getBlockNum() headBlock = producers[producerNum].getBlock(headBlockNum) - nonlocal blockProducer + global blockProducer if blockProducer is None: blockProducer = headBlock["producer"] Utils.Print("headBlock: {}".format(json.dumps(headBlock, indent=4, sort_keys=True))) From c74a14803295bad284c935545923d9455d32691b Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 29 Mar 2021 18:43:16 -0500 Subject: [PATCH 072/157] Trying to prevent error on CentOS. --- tests/privacy_startup_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index ed9aa478548..9d6a7c91b7e 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -138,7 +138,7 @@ def verifyInSync(producerNum): featureDict = producers[0].getSupportedProtocolFeatureDict() Utils.Print("feature dict: {}".format(json.dumps(featureDict, indent=4, sort_keys=True))) - Utils.Print("act feature dict: {}".format(json.dumps(producers[0].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) + #Utils.Print("act feature dict: {}".format(json.dumps(producers[0].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) timeout = ( pnodes * 12 / 2 ) * 2 # (number of producers * blocks produced / 0.5 blocks per second) * 2 rounds producers[0].waitUntilBeginningOfProdTurn(blockProducer, timeout=timeout) feature = "SECURITY_GROUP" From 75e4823667c8c7932c085a105e79adb30fa72bb3 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 18 Mar 2021 17:20:27 -0500 Subject: [PATCH 073/157] Moved useful protocol feature methods into Node.py. --- tests/Node.py | 46 ++++++++++++++++--- ..._multiple_version_protocol_feature_test.py | 34 ++++---------- 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index 7fb6dd50fa3..6844403020b 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1424,16 +1424,50 @@ def isLibAdvancing(): return self.getIrreversibleBlockNum() > currentLib return Utils.waitForTruth(isLibAdvancing, timeout) + def waitUntilBeginningOfProdTurn(self, producerName, timeout=30, sleepTime=0.4): + beginningOfProdTurnHead = 0 + def isDesiredProdTurn(): + beginningOfProdTurnHead = self.getHeadBlockNum() + res = self.getBlock(beginningOfProdTurnHead)["producer"] == producerName and \ + self.getBlock(beginningOfProdTurnHead-1)["producer"] != producerName + return res + ret = Utils.waitForTruth(isDesiredProdTurn, timeout, sleepTime) + assert ret != None, "Expected producer to arrive within {} seconds".format(timeout) + return beginningOfProdTurnHead + # Require producer_api_plugin - def activatePreactivateFeature(self): - protocolFeatureDigestDict = self.getSupportedProtocolFeatureDict() - preactivateFeatureDigest = protocolFeatureDigestDict["PREACTIVATE_FEATURE"]["feature_digest"] - assert preactivateFeatureDigest + def activateFeatures(self, features): + featureDigests = [] + for feature in features: + protocolFeatureDigestDict = self.getSupportedProtocolFeatureDict() + assert feature in protocolFeatureDigestDict + featureDigest = protocolFeatureDigestDict[feature]["feature_digest"] + assert featureDigest + featureDigests.append(featureDigest) - self.scheduleProtocolFeatureActivations([preactivateFeatureDigest]) + self.scheduleProtocolFeatureActivations(featureDigests) # Wait for the next block to be produced so the scheduled protocol feature is activated - assert self.waitForHeadToAdvance(blocksToAdvance=2), print("ERROR: TIMEOUT WAITING FOR PREACTIVATE") + assert self.waitForHeadToAdvance(blocksToAdvance=2), print("ERROR: TIMEOUT WAITING FOR activating features: {}".format(",".join(features))) + + # Require producer_api_plugin + def activatePreactivateFeature(self): + return self.activateFeatures(["PREACTIVATE_FEATURE"]) + + def containsFeatures(self, features): + protocolFeatureDict = self.getSupportedProtocolFeatureDict() + blockHeaderState = self.getLatestBlockHeaderState() + assert blockHeaderState, "blockHeaderState should not be empty" + for feature in features: + featureDigest = protocolFeatureDict[feature]["feature_digest"] + assert featureDigest, "{}'s Digest should not be empty".format(feature) + activatedProtocolFeatures = blockHeaderState["activated_protocol_features"]["protocol_features"] + if featureDigest not in activatedProtocolFeatures: + return False + return True + + def containsPreactivateFeature(self): + return containsFeatures(["PREACTIVATE_FEATURE"]) # Return an array of feature digests to be preactivated in a correct order respecting dependencies # Require producer_api_plugin diff --git a/tests/nodeos_multiple_version_protocol_feature_test.py b/tests/nodeos_multiple_version_protocol_feature_test.py index 384be146fb9..555e9abe512 100755 --- a/tests/nodeos_multiple_version_protocol_feature_test.py +++ b/tests/nodeos_multiple_version_protocol_feature_test.py @@ -43,24 +43,6 @@ def restartNode(node: Node, chainArg=None, addSwapFlags=None, nodeosPath=None): timeout=5, cachePopen=True, nodeosPath=nodeosPath) assert isRelaunchSuccess, "Fail to relaunch" -def shouldNodeContainPreactivateFeature(node): - preactivateFeatureDigest = node.getSupportedProtocolFeatureDict()["PREACTIVATE_FEATURE"]["feature_digest"] - assert preactivateFeatureDigest, "preactivateFeatureDigest should not be empty" - blockHeaderState = node.getLatestBlockHeaderState() - assert blockHeaderState, "blockHeaderState should not be empty" - activatedProtocolFeatures = blockHeaderState["activated_protocol_features"]["protocol_features"] - return preactivateFeatureDigest in activatedProtocolFeatures - -beginningOfProdTurnHead = 0 -def waitUntilBeginningOfProdTurn(node, producerName, timeout=30, sleepTime=0.4): - def isDesiredProdTurn(): - beginningOfProdTurnHead = node.getHeadBlockNum() - res = node.getBlock(beginningOfProdTurnHead)["producer"] == producerName and \ - node.getBlock(beginningOfProdTurnHead-1)["producer"] != producerName - return res - ret = Utils.waitForTruth(isDesiredProdTurn, timeout, sleepTime) - assert ret != None, "Expected producer to arrive within 19 seconds (with 3 other producers)" - def waitForOneRound(): time.sleep(24) # We have 4 producers for this test @@ -176,15 +158,15 @@ def nodeHasBlocks(node, blockIds, blockNums): for i in range(3): Utils.Print("1st node tries activatePreactivateFeature time(s): {}".format(i+1)) # 1st node waits for the start of the production turn each time it tries activatePreactivateFeature() - waitUntilBeginningOfProdTurn(newNodes[0], "defproducera") + beginningOfProdTurnHead = newNodes[0].waitUntilBeginningOfProdTurn("defproducera") newNodes[0].activatePreactivateFeature() - if shouldNodeContainPreactivateFeature(newNodes[0]): + if newNodes[0].containsPreactivateFeature(): break diff = newNodes[0].getInfo()["head_block_num"] - beginningOfProdTurnHead assert diff >= 12, "1st node should contain PREACTIVATE FEATURE since we set it during its production window" - assert shouldNodeContainPreactivateFeature(newNodes[0]), "1st node should contain PREACTIVATE FEATURE" - assert not (shouldNodeContainPreactivateFeature(newNodes[1]) or shouldNodeContainPreactivateFeature(newNodes[2])), \ + assert newNodes[0].containsPreactivateFeature(), "1st node should contain PREACTIVATE FEATURE" + assert not (newNodes[1].containsPreactivateFeature() or newNodes[2].containsPreactivateFeature()), \ "2nd and 3rd node should not contain PREACTIVATE FEATURE" Utils.Print("+++ 2nd, 3rd and 4th node should be in sync, and 1st node should be out of sync +++") assert areNodesInSync([newNodes[1], newNodes[2], oldNode], pauseAll=True, resumeAll=False), "2nd, 3rd and 4th node should be in sync" @@ -192,7 +174,7 @@ def nodeHasBlocks(node, blockIds, blockNums): waitForOneRound() - assert not shouldNodeContainPreactivateFeature(newNodes[0]), "PREACTIVATE_FEATURE should be dropped" + assert not newNodes[0].containsPreactivateFeature(), "PREACTIVATE_FEATURE should be dropped" assert areNodesInSync(allNodes), "All nodes should be in sync" # Then we set the earliest_allowed_activation_time of 2nd node and 3rd node with valid value @@ -203,13 +185,13 @@ def nodeHasBlocks(node, blockIds, blockNums): setValidityOfActTimeSubjRestriction(newNodes[1], "PREACTIVATE_FEATURE", True) setValidityOfActTimeSubjRestriction(newNodes[2], "PREACTIVATE_FEATURE", True) - waitUntilBeginningOfProdTurn(newNodes[0], "defproducera") + newNodes[0].waitUntilBeginningOfProdTurn("defproducera") libBeforePreactivation = newNodes[0].getIrreversibleBlockNum() newNodes[0].activatePreactivateFeature() assert areNodesInSync(newNodes, pauseAll=True, resumeAll=False), "New nodes should be in sync" assert not areNodesInSync(allNodes, pauseAll=False, resumeAll=True), "Nodes should not be in sync after preactivation" - for node in newNodes: assert shouldNodeContainPreactivateFeature(node), "New node should contain PREACTIVATE_FEATURE" + for node in newNodes: assert node.containsPreactivateFeature(), "New node should contain PREACTIVATE_FEATURE" activatedBlockNum = newNodes[0].getHeadBlockNum() # The PREACTIVATE_FEATURE should have been activated before or at this block num assert waitUntilBlockBecomeIrr(newNodes[0], activatedBlockNum), \ @@ -236,7 +218,7 @@ def nodeHasBlocks(node, blockIds, blockNums): time.sleep(2) # Give some time to replay assert areNodesInSync(allNodes), "All nodes should be in sync" - assert shouldNodeContainPreactivateFeature(oldNode), "4th node should contain PREACTIVATE_FEATURE" + assert oldNode.containsPreactivateFeature(), "4th node should contain PREACTIVATE_FEATURE" testSuccessful = True finally: From 8c6389e933f80717dbedb6a44d6367b33e1323c4 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 18 Mar 2021 17:22:50 -0500 Subject: [PATCH 074/157] Activating the SECURITY_GROUP protocol feature. --- tests/privacy_startup_network.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index c7a11edaa93..5d8340961d3 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -90,12 +90,16 @@ producers = [cluster.getNode(x) for x in range(pnodes) ] relays = [cluster.getNode(pnodes + x) for x in range(pnodes) ] apiNodes = [cluster.getNode(x) for x in apiNodeNums] + blockProducer = None def verifyInSync(producerNum): Utils.Print("Ensure all nodes are in-sync") lib = producers[producerNum].getInfo()["last_irreversible_block_num"] headBlockNum = producers[producerNum].getBlockNum() headBlock = producers[producerNum].getBlock(headBlockNum) + global blockProducer + if blockProducer is None: + blockProducer = headBlock["producer"] Utils.Print("headBlock: {}".format(json.dumps(headBlock, indent=4, sort_keys=True))) headBlockId = headBlock["id"] for prod in producers: @@ -117,6 +121,16 @@ def verifyInSync(producerNum): verifyInSync(producerNum=0) + featureDict = producers[0].getSupportedProtocolFeatureDict() + Utils.Print("feature dict: {}".format(json.dumps(featureDict, indent=4, sort_keys=True))) + + Utils.Print("act feature dict: {}".format(json.dumps(producers[0].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) + timeout = ( pnodes * 12 / 2 ) * 2 # (number of producers * blocks produced / 0.5 blocks per second) * 2 rounds + producers[0].waitUntilBeginningOfProdTurn(blockProducer, timeout=timeout) + feature = "SECURITY_GROUP" + producers[0].activateFeatures([feature]) + assert producers[0].containsFeatures([feature]), "{} feature was not activated".format(feature) + if sanityTest: testSuccessful=True exit(0) From 69d4232abb2c27d6895e1ed538eccd169ef0247f Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 29 Mar 2021 14:13:00 -0500 Subject: [PATCH 075/157] Minor changes to add accounts and load contract to exercise security group. --- tests/privacy_startup_network.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index 5d8340961d3..ed9aa478548 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -90,6 +90,20 @@ producers = [cluster.getNode(x) for x in range(pnodes) ] relays = [cluster.getNode(pnodes + x) for x in range(pnodes) ] apiNodes = [cluster.getNode(x) for x in apiNodeNums] + + def createAccount(newAcc): + producers[0].createInitializeAccount(newAcc, cluster.eosioAccount) + ignWallet = cluster.walletMgr.create("ignition") # will actually just look up the wallet + cluster.walletMgr.importKey(newAcc, ignWallet) + + numAccounts = 4 + testAccounts = Cluster.createAccountKeys(numAccounts) + accountPrefix = "testaccount" + for i in range(numAccounts): + testAccount = testAccounts[i] + testAccount.name = accountPrefix + str(i + 1) + createAccount(testAccount) + blockProducer = None def verifyInSync(producerNum): @@ -135,6 +149,12 @@ def verifyInSync(producerNum): testSuccessful=True exit(0) + def publishContract(account, wasmFile, waitForTransBlock=False): + Print("Publish contract") + return producers[0].publishContract(account, "unittests/test-contracts/security_group_test/", wasmFile, abiFile=None, waitForTransBlock=waitForTransBlock) + + publishContract(testAccounts[0], 'security_group_test.wasm', waitForTransBlock=True) + testSuccessful=True finally: TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails) From 17e2cbe74fe3eb3a1f15aad92d6bd3eff2b439d6 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 29 Mar 2021 14:14:24 -0500 Subject: [PATCH 076/157] Change configSecurityGroup to automatically NOT use the bios bootfile and also to inject a delay to work around 30 second delay we are seeing. --- tests/Cluster.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index 694dee85e9a..b498a8d0b0d 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -573,8 +573,9 @@ def connectGroup(group, producerNodes, bridgeNodes) : Utils.Print("Bootstrap cluster.") if not loadSystemContract: useBiosBootFile=False #ensure we use Cluster.bootstrap - if onlyBios or not useBiosBootFile: - self.biosNode=self.bootstrap(biosNode, startedNodes, prodCount + sharedProducers, totalProducers, pfSetupPolicy, onlyBios, onlySetProds, loadSystemContract, manualProducerNodeConf) + if onlyBios or not useBiosBootFile or configSecurityGroup: + delayProductionTransfer = 35 if configSecurityGroup else None # when TLS delay is analyzed, then this delay and ignoring of useBiosBootFile can be removed + self.biosNode=self.bootstrap(biosNode, startedNodes, prodCount + sharedProducers, totalProducers, pfSetupPolicy, onlyBios, onlySetProds, loadSystemContract, manualProducerNodeConf, delayProductionTransfer=delayProductionTransfer) if self.biosNode is None: Utils.Print("ERROR: Bootstrap failed.") return False @@ -1180,7 +1181,7 @@ def bios_bootstrap(self, biosNode, totalNodes, pfSetupPolicy, silent=False): return biosNode - def bootstrap(self, biosNode, totalNodes, prodCount, totalProducers, pfSetupPolicy, onlyBios=False, onlySetProds=False, loadSystemContract=True, manualProducerNodeConf=[]): + def bootstrap(self, biosNode, totalNodes, prodCount, totalProducers, pfSetupPolicy, onlyBios=False, onlySetProds=False, loadSystemContract=True, manualProducerNodeConf=[], delayProductionTransfer=None): """Create 'prodCount' init accounts and deposits 10000000000 SYS in each. If prodCount is -1 will initialize all possible producers. Ensure nodes are inter-connected prior to this call. One way to validate this will be to check if every node has block 1.""" @@ -1310,6 +1311,10 @@ def bootstrap(self, biosNode, totalNodes, prodCount, totalProducers, pfSetupPoli if Utils.Debug: Utils.Print("setprods: %s" % (setProdsStr)) Utils.Print("Setting producers: %s." % (", ".join(prodNames))) opts="--permission eosio@active" + + if delayProductionTransfer: + time.sleep(delayProductionTransfer) + # pylint: disable=redefined-variable-type trans=biosNode.pushMessage("eosio", "setprods", setProdsStr, opts) if trans is None or not trans[0]: From fa1e796a16169781f8d5533f91629f71bb65cab5 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 29 Mar 2021 15:49:37 -0500 Subject: [PATCH 077/157] Added improved error handling. --- tests/Node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/Node.py b/tests/Node.py index 6844403020b..aed9123d479 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1509,6 +1509,8 @@ def getLatestBlockHeaderState(self): def getActivatedProtocolFeatures(self): latestBlockHeaderState = self.getLatestBlockHeaderState() + if "activated_protocol_features" not in latestBlockHeaderState or "protocol_features" not in latestBlockHeaderState["activated_protocol_features"]: + Utils.errorExit("getLatestBlockHeaderState did not return expected output, should contain [\"activated_protocol_features\"][\"protocol_features\"]: {}".format(latestBlockHeaderState)) return latestBlockHeaderState["activated_protocol_features"]["protocol_features"] def modifyBuiltinPFSubjRestrictions(self, featureCodename, subjectiveRestriction={}): From 9c976cc0d5f5120f2f4876d05da132426a7b6607 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 29 Mar 2021 16:12:52 -0500 Subject: [PATCH 078/157] Fixing PR comment, and cleaning up uses of global that should be nonlocal. --- tests/Node.py | 1 + tests/nodeos_high_transaction_test.py | 2 +- tests/privacy_startup_network.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index aed9123d479..54d4179b2eb 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1427,6 +1427,7 @@ def isLibAdvancing(): def waitUntilBeginningOfProdTurn(self, producerName, timeout=30, sleepTime=0.4): beginningOfProdTurnHead = 0 def isDesiredProdTurn(): + nonlocal beginningOfProdTurnHead beginningOfProdTurnHead = self.getHeadBlockNum() res = self.getBlock(beginningOfProdTurnHead)["producer"] == producerName and \ self.getBlock(beginningOfProdTurnHead-1)["producer"] != producerName diff --git a/tests/nodeos_high_transaction_test.py b/tests/nodeos_high_transaction_test.py index 6108af9ffa0..1afaff11675 100755 --- a/tests/nodeos_high_transaction_test.py +++ b/tests/nodeos_high_transaction_test.py @@ -162,7 +162,7 @@ lastIrreversibleBlockNum = None def cacheTransIdInBlock(transId, transToBlock, node): - global lastIrreversibleBlockNum + nonlocal lastIrreversibleBlockNum lastPassLIB = None blockWaitTimeout = 60 transTimeDelayed = False diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index ed9aa478548..382e7921d12 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -111,7 +111,7 @@ def verifyInSync(producerNum): lib = producers[producerNum].getInfo()["last_irreversible_block_num"] headBlockNum = producers[producerNum].getBlockNum() headBlock = producers[producerNum].getBlock(headBlockNum) - global blockProducer + nonlocal blockProducer if blockProducer is None: blockProducer = headBlock["producer"] Utils.Print("headBlock: {}".format(json.dumps(headBlock, indent=4, sort_keys=True))) From 8c78f3172d948c683ffe6a6c55728bd1bf32a579 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 29 Mar 2021 17:32:18 -0500 Subject: [PATCH 079/157] Could not use nonlocal in this case. --- tests/privacy_startup_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index 382e7921d12..ed9aa478548 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -111,7 +111,7 @@ def verifyInSync(producerNum): lib = producers[producerNum].getInfo()["last_irreversible_block_num"] headBlockNum = producers[producerNum].getBlockNum() headBlock = producers[producerNum].getBlock(headBlockNum) - nonlocal blockProducer + global blockProducer if blockProducer is None: blockProducer = headBlock["producer"] Utils.Print("headBlock: {}".format(json.dumps(headBlock, indent=4, sort_keys=True))) From 05e7f7960bc4eaea7823566ea5c4b409c4d1f81a Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 29 Mar 2021 18:43:16 -0500 Subject: [PATCH 080/157] Trying to prevent error on CentOS. --- tests/privacy_startup_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index ed9aa478548..9d6a7c91b7e 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -138,7 +138,7 @@ def verifyInSync(producerNum): featureDict = producers[0].getSupportedProtocolFeatureDict() Utils.Print("feature dict: {}".format(json.dumps(featureDict, indent=4, sort_keys=True))) - Utils.Print("act feature dict: {}".format(json.dumps(producers[0].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) + #Utils.Print("act feature dict: {}".format(json.dumps(producers[0].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) timeout = ( pnodes * 12 / 2 ) * 2 # (number of producers * blocks produced / 0.5 blocks per second) * 2 rounds producers[0].waitUntilBeginningOfProdTurn(blockProducer, timeout=timeout) feature = "SECURITY_GROUP" From 31829e1c94fcefd4ef408d8bc79f63e09df35b80 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Tue, 30 Mar 2021 14:57:06 -0400 Subject: [PATCH 081/157] added openssl to EC2 docker images --- .cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile | 2 +- .cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile b/.cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile index d606a48cdbe..1a37dcb85c2 100644 --- a/.cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile +++ b/.cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile @@ -3,7 +3,7 @@ ENV VERSION 1 # install dependencies. RUN yum update -y && \ yum install -y which git sudo procps-ng util-linux autoconf automake \ - libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ + libtool make bzip2 bzip2-devel openssl openssl-devel gmp-devel libstdc++ libcurl-devel \ libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ graphviz patch gcc gcc-c++ vim-common jq && \ yum clean all && rm -rf /var/cache/yum diff --git a/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile b/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile index e7aac5f374c..055861b129a 100644 --- a/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile @@ -3,7 +3,7 @@ ENV VERSION 1 # install dependencies. RUN yum update -y && \ yum install -y which git sudo procps-ng util-linux autoconf automake \ - libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ + libtool make bzip2 bzip2-devel openssl openssl-devel gmp-devel libstdc++ libcurl-devel \ libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ graphviz clang patch llvm-devel llvm-static vim-common jq && \ yum clean all && rm -rf /var/cache/yum From 43926e4ea685815b58fed39e20d7460ea54179fa Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Wed, 31 Mar 2021 13:13:21 -0400 Subject: [PATCH 082/157] PR concerns addressed and centos fix --- .cicd/platforms/pinned/centos-7.7-pinned.dockerfile | 2 +- .cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile | 2 +- tests/Cluster.py | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile b/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile index fde27ef38f9..ee386e0b794 100644 --- a/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile +++ b/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile @@ -6,7 +6,7 @@ RUN yum update -y && \ yum --enablerepo=extras install -y centos-release-scl && \ yum --enablerepo=extras install -y devtoolset-8 && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ - graphviz bzip2-devel openssl-devel gmp-devel ocaml \ + graphviz bzip2-devel openssl openssl-devel gmp-devel ocaml \ python python-devel rh-python36 file libusbx-devel \ libcurl-devel patch vim-common jq glibc-locale-source glibc-langpack-en && \ yum clean all && rm -rf /var/cache/yum diff --git a/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile b/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile index 712bfb61327..291464b054d 100644 --- a/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile @@ -6,7 +6,7 @@ RUN yum update -y && \ yum --enablerepo=extras install -y centos-release-scl && \ yum --enablerepo=extras install -y devtoolset-8 && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ - graphviz bzip2-devel openssl-devel gmp-devel ocaml \ + graphviz bzip2-devel openssl openssl-devel gmp-devel ocaml \ python python-devel rh-python36 file libusbx-devel \ libcurl-devel patch vim-common jq llvm-toolset-7.0-llvm-devel llvm-toolset-7.0-llvm-static \ glibc-locale-source glibc-langpack-en && \ diff --git a/tests/Cluster.py b/tests/Cluster.py index b498a8d0b0d..30e79490071 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -249,8 +249,7 @@ def getArguments(number): # eos name can have only numbers 1-5 # e.g. 0 -> 1, 6 -> 5.1, 12 -> 5.5.2 def normalizeNumber(number): - if number <= 0: - number = 1 + assert(number > 0) if number <= 5: return str(number) cnt = number @@ -261,7 +260,7 @@ def normalizeNumber(number): ret = "{}.5".format(ret) else: ret = "{}.{}".format(ret, cnt) - assert(len(ret) <= 13) + assert(len(ret) <= 13) return ret From 90f730144a853336fde84065f60d207ef2e1a4d3 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Wed, 7 Apr 2021 19:05:48 -0400 Subject: [PATCH 083/157] logs fix + certain systems tests fix --- plugins/net_plugin/net_plugin.cpp | 6 +++--- tests/Cluster.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 78e6e4b4a13..a0c1cede3a3 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -3816,18 +3816,18 @@ namespace eosio { error_code ec; if (!ca.empty()){ - fc_dlog(logger, "using verify file: ${p}", ("p", ca)); + dlog("using verify file: ${p}", ("p", ca)); ssl_context->load_verify_file(ca, ec); EOS_ASSERT(!ec, ssl_configuration_error, "load_verify_file: ${e}", ("e", ec.message())); //this ensures peer has trusted certificate. no certificate-less connections ssl_context->set_verify_mode(ssl::context::verify_peer | ssl::context::verify_fail_if_no_peer_cert); } - fc_dlog(logger, "using private key file: ${p}", ("p", pkey)); + dlog("using private key file: ${p}", ("p", pkey)); ssl_context->use_private_key_file(pkey, ssl::context::pem, ec); EOS_ASSERT(!ec, ssl_configuration_error, "use_private_key_file: ${e}", ("e", ec.message())); - fc_dlog(logger, "using chain file: ${p}", ("p", cert)); + dlog("using chain file: ${p}", ("p", cert)); ssl_context->use_certificate_chain_file(cert, ec); EOS_ASSERT(!ec, ssl_configuration_error, "use_certificate_chain_file: ${e}", ("e", ec.message())); } diff --git a/tests/Cluster.py b/tests/Cluster.py index 30e79490071..ff064cc0364 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -230,7 +230,7 @@ def insertSpecificExtraNodeosArgs(node, insertStr): if Utils.Debug: Utils.Print("change to dir: {}".format(os.getcwd())) genCertScript=os.path.join(original, "tests", "generate-certificates.sh") totalNodesInNetwork = totalNodes + 1 # account for bios node - cmd = "{} --days 1 --CA-org Block.one --CA-CN test-domain --org-mask node{{NUMBER}} --cn-mask test-domain{{NUMBER}} --group-size {}".format(genCertScript, totalNodesInNetwork) + cmd = "{} --days 1 --CA-org Block.one --CA-CN test-domain --org-mask node{{NUMBER}} --cn-mask test-domain{{NUMBER}} --group-size {} --use-RSA".format(genCertScript, totalNodesInNetwork) rtn=Utils.runCmdReturnStr(cmd, silentErrors=False) with open("generate.log", 'w') as f: f.write("executed cmd: {}".format(cmd)) From 6138d87f93f8e59af4cf09d73b4584d8ec2e6ee5 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Wed, 7 Apr 2021 22:18:44 -0400 Subject: [PATCH 084/157] corrected behavior of tls for some corner cases --- .gitignore | 2 ++ plugins/net_plugin/net_plugin.cpp | 29 ++++++++++++++++------------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 8cd0213e1d7..08b6880c0e9 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,5 @@ var/lib/node_* .DS_Store !*.swagger.* + +ZScaler* \ No newline at end of file diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index a0c1cede3a3..b601bad6991 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -650,6 +650,7 @@ namespace eosio { bool start_session(bool server); bool socket_is_open() const { return socket_open.load(); } // thread safe, atomic + inline void set_socket_open(bool open = true) { socket_open = open; } const string& peer_address() const { return peer_addr; } // thread safe, const void set_connection_type( const string& peer_addr ); @@ -918,16 +919,14 @@ namespace eosio { std::lock_guard connection_guard(my_impl->connections_mtx); if(my_impl->security_group.is_in_security_group(participant)) { - set_participating(true); - peer_dlog(this, "participant added: ${o}", ("o", organization)); + participant_name_ = participant; return true; } } peer_dlog(this, "received unauthorized participant: ${s}", ("s", organization)); } - - set_participating(false); + //we keep connection if peer has valid certificate but participant name is not authorized //however that connection doesn't receive updates return preverified; @@ -1100,27 +1099,31 @@ namespace eosio { return false; } else { - auto start_read = [this](){ - fc_dlog( logger, "connected to ${peer}", ("peer", peer_name()) ); - socket_open = true; - start_read_message(); - }; if (my_impl->ssl_enabled) { socket.ssl_socket()->async_handshake(server ? ssl::stream_base::server : ssl::stream_base::client, - boost::asio::bind_executor(strand, [start_read, c = shared_from_this(), socket=socket](const auto& ec){ + boost::asio::bind_executor(strand, [c = shared_from_this(), socket=socket](const auto& ec){ //we use socket just to retain connection shared_ptr just in case it will be deleted //when we will need it inside start_read_message std::ignore = socket; if (ec) { + c->set_participating(false); fc_elog( logger, "ssl handshake error: ${e}", ("e", ec.message()) ); c->close(); return; } - start_read(); + + c->set_socket_open(); + c->set_participating(true); + fc_dlog( logger, "connected to ${peer}", ("peer", c->peer_name()) ); + fc_dlog( logger, "participant added: ${o}", ("o", c->participant_name())); + + c->start_read_message(); })); } else { - start_read(); + fc_dlog( logger, "connected to ${peer}", ("peer", peer_name()) ); + set_socket_open(); + start_read_message(); } return true; } @@ -1145,7 +1148,7 @@ namespace eosio { } void connection::_close( connection* self, bool reconnect, bool shutdown ) { - self->socket_open = false; + self->set_socket_open(false); boost::system::error_code ec; auto& cur_sock = self->socket.raw_socket(); if( cur_sock.is_open() ) { From e01029030eaa407cf93bbbfa08c2927371ecc128 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Thu, 8 Apr 2021 11:36:48 -0400 Subject: [PATCH 085/157] tls connection bugfix --- plugins/net_plugin/net_plugin.cpp | 33 ++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index b601bad6991..2f16f6511b1 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -647,7 +647,10 @@ namespace eosio { ~connection() {} - bool start_session(bool server); + //this function calls callback synchronously in case of non-ssl connection + //and asynchronously in case of ssl after asynchronous ssl handshake + template + void start_session(bool server, _Callback callback); bool socket_is_open() const { return socket_open.load(); } // thread safe, atomic inline void set_socket_open(bool open = true) { socket_open = open; } @@ -1086,7 +1089,8 @@ namespace eosio { return stat; } - bool connection::start_session(bool server) { + template + void connection::start_session(bool server, _Callback callback) { verify_strand_in_this_thread( strand, __func__, __LINE__ ); update_endpoints(); @@ -1096,12 +1100,13 @@ namespace eosio { if( ec ) { fc_elog( logger, "connection failed (set_option) ${peer}: ${e1}", ("peer", peer_name())( "e1", ec.message() ) ); close(); - return false; + callback(false); } else { if (my_impl->ssl_enabled) { socket.ssl_socket()->async_handshake(server ? ssl::stream_base::server : ssl::stream_base::client, - boost::asio::bind_executor(strand, [c = shared_from_this(), socket=socket](const auto& ec){ + boost::asio::bind_executor(strand, + [c = shared_from_this(), socket=socket, callback](const auto& ec){ //we use socket just to retain connection shared_ptr just in case it will be deleted //when we will need it inside start_read_message std::ignore = socket; @@ -1118,14 +1123,15 @@ namespace eosio { fc_dlog( logger, "participant added: ${o}", ("o", c->participant_name())); c->start_read_message(); + callback(true); })); } else { fc_dlog( logger, "connected to ${peer}", ("peer", peer_name()) ); set_socket_open(); start_read_message(); + callback(true); } - return true; } } @@ -2611,9 +2617,10 @@ namespace eosio { c = shared_from_this(), socket=socket]( const boost::system::error_code& err, const tcp::endpoint& endpoint ) mutable { if( !err && socket.raw_socket().is_open() && &socket.raw_socket() == &c->socket.raw_socket() ) { - if( c->start_session(false) ) { - c->send_handshake(); - } + c->start_session(false, [c](bool success){ + if (success) + c->send_handshake(); + }); } else { fc_elog( logger, "connection failed to ${peer}: ${error}", ("peer", c->peer_name())( "error", err.message())); c->close( false ); @@ -2652,10 +2659,12 @@ namespace eosio { if( from_addr < max_nodes_per_host && (max_client_count == 0 || visitors < max_client_count)) { fc_ilog( logger, "Accepted new connection: " + paddr_str ); new_connection->set_heartbeat_timeout( heartbeat_timeout ); - if( new_connection->start_session(true)) { - std::lock_guard g_unique( connections_mtx ); - connections.insert( new_connection ); - } + new_connection->start_session(true, [c = shared_from_this(), new_connection](bool success) { + if (success) { + std::lock_guard g_unique( c->connections_mtx ); + c->connections.insert( new_connection ); + } + }); } else { if( from_addr >= max_nodes_per_host ) { From 196b6f7e49aea4a599f7affdee22c14fc458f50f Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Thu, 8 Apr 2021 13:42:43 -0400 Subject: [PATCH 086/157] Node.py test bugfix --- tests/Node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Node.py b/tests/Node.py index 54d4179b2eb..5f22044dcb2 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1468,7 +1468,7 @@ def containsFeatures(self, features): return True def containsPreactivateFeature(self): - return containsFeatures(["PREACTIVATE_FEATURE"]) + return self.containsFeatures(["PREACTIVATE_FEATURE"]) # Return an array of feature digests to be preactivated in a correct order respecting dependencies # Require producer_api_plugin From e090180358e15e8166cf5c15bf710a228e848923 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Thu, 8 Apr 2021 15:49:14 -0400 Subject: [PATCH 087/157] bugfixes to privacy feature --- plugins/net_plugin/net_plugin.cpp | 57 ++++++++++++++++++------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 2f16f6511b1..693938b338c 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -653,7 +653,6 @@ namespace eosio { void start_session(bool server, _Callback callback); bool socket_is_open() const { return socket_open.load(); } // thread safe, atomic - inline void set_socket_open(bool open = true) { socket_open = open; } const string& peer_address() const { return peer_addr; } // thread safe, const void set_connection_type( const string& peer_addr ); @@ -865,14 +864,23 @@ namespace eosio { private: std::optional participant_name_; std::atomic participating_{true}; + volatile bool verified; public: /** @brief Returns the optional name of the participant */ - auto participant_name() { return participant_name_; } + inline auto participant_name() { return participant_name_; } /** @brief Set the participating status */ - void set_participating(bool status) { participating_.store(status, std::memory_order_relaxed); } + inline void set_participating(bool status) { + if (my_impl->ssl_enabled) + participating_.store(status, std::memory_order_relaxed); + } /** returns true if connection is in the security group */ - bool is_participating() const { return participating_.load(std::memory_order_relaxed); } + inline bool is_participating() const { + if (my_impl->ssl_enabled) + return participating_.load(std::memory_order_relaxed); + + return true; + } std::string certificate_subject(ssl::verify_context &ctx) { X509 *cert = X509_STORE_CTX_get_current_cert(ctx.native_handle()); @@ -918,12 +926,15 @@ namespace eosio { if (length > 0) { std::string organization{buf, (size_t)length}; if (is_string_valid_name(organization)){ - account_name participant{organization}; - + participant_name_ = account_name{organization}; + std::lock_guard connection_guard(my_impl->connections_mtx); - if(my_impl->security_group.is_in_security_group(participant)) { - participant_name_ = participant; - return true; + if(my_impl->security_group.is_in_security_group(*participant_name_)) { + fc_dlog( logger, "participant added: ${o}", ("o", participant_name())); + verified = true; + //must be true since we reach there if root certificate was preverified + //but just in case let's return preverified other that true + return preverified; } } @@ -1102,35 +1113,31 @@ namespace eosio { close(); callback(false); } else { - + auto start_read = [this, callback](){ + fc_dlog( logger, "connected to ${peer}", ("peer", peer_name()) ); + socket_open = true; + set_participating(verified); + start_read_message(); + callback(true); + }; if (my_impl->ssl_enabled) { socket.ssl_socket()->async_handshake(server ? ssl::stream_base::server : ssl::stream_base::client, boost::asio::bind_executor(strand, - [c = shared_from_this(), socket=socket, callback](const auto& ec){ + [start_read, c = shared_from_this(), socket=socket, callback](const auto& ec){ //we use socket just to retain connection shared_ptr just in case it will be deleted //when we will need it inside start_read_message std::ignore = socket; if (ec) { - c->set_participating(false); fc_elog( logger, "ssl handshake error: ${e}", ("e", ec.message()) ); c->close(); return; } - c->set_socket_open(); - c->set_participating(true); - fc_dlog( logger, "connected to ${peer}", ("peer", c->peer_name()) ); - fc_dlog( logger, "participant added: ${o}", ("o", c->participant_name())); - - c->start_read_message(); - callback(true); + start_read(); })); } else { - fc_dlog( logger, "connected to ${peer}", ("peer", peer_name()) ); - set_socket_open(); - start_read_message(); - callback(true); + start_read(); } } } @@ -1154,7 +1161,9 @@ namespace eosio { } void connection::_close( connection* self, bool reconnect, bool shutdown ) { - self->set_socket_open(false); + self->socket_open = false; + self->verified = false; + self->set_participating(false); boost::system::error_code ec; auto& cur_sock = self->socket.raw_socket(); if( cur_sock.is_open() ) { From 8977c5011e1f7d219556dec380fc5ea49622e7de Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Thu, 8 Apr 2021 20:05:42 -0400 Subject: [PATCH 088/157] PR review concerns addressed + bugfixes --- plugins/net_plugin/net_plugin.cpp | 36 ++++++++++++++++--------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 693938b338c..37bf7fbf700 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -864,7 +864,6 @@ namespace eosio { private: std::optional participant_name_; std::atomic participating_{true}; - volatile bool verified; public: /** @brief Returns the optional name of the participant */ @@ -881,6 +880,16 @@ namespace eosio { return true; } + inline void setup_participant() { + if(my_impl->ssl_enabled) { + account_name participant = participant_name_ ? *participant_name_ : account_name{}; + bool participating = my_impl->security_group.is_in_security_group(participant); + + fc_dlog( logger, "[${peer}] participant: [${name}] participating: [${enabled}]", + ("peer", peer_name())("name", participant_name())("enabled", participating)); + set_participating(participating); + } + } std::string certificate_subject(ssl::verify_context &ctx) { X509 *cert = X509_STORE_CTX_get_current_cert(ctx.native_handle()); @@ -899,7 +908,7 @@ namespace eosio { } bool verify_certificate(bool preverified, ssl::verify_context &ctx) { - peer_dlog(this, "preverified: ${p} certificate subject: ${s}", ("p", preverified)("s", certificate_subject(ctx))); + peer_dlog(this, "preverified: [${p}] certificate subject: ${s}", ("p", preverified)("s", certificate_subject(ctx))); //certificate depth means number of certificate issuers verified current certificate //openssl provides those one by one starting from root certificate //we don't use CA certificate or intermidiate issuers, so skipping those @@ -927,18 +936,12 @@ namespace eosio { std::string organization{buf, (size_t)length}; if (is_string_valid_name(organization)){ participant_name_ = account_name{organization}; - - std::lock_guard connection_guard(my_impl->connections_mtx); - if(my_impl->security_group.is_in_security_group(*participant_name_)) { - fc_dlog( logger, "participant added: ${o}", ("o", participant_name())); - verified = true; - //must be true since we reach there if root certificate was preverified - //but just in case let's return preverified other that true - return preverified; - } + //participant name will be later used in start_session to determine if + //participant is participating or not + } + else { + peer_dlog(this, "received unauthorized participant: ${s}", ("s", organization)); } - - peer_dlog(this, "received unauthorized participant: ${s}", ("s", organization)); } //we keep connection if peer has valid certificate but participant name is not authorized @@ -1114,9 +1117,9 @@ namespace eosio { callback(false); } else { auto start_read = [this, callback](){ - fc_dlog( logger, "connected to ${peer}", ("peer", peer_name()) ); socket_open = true; - set_participating(verified); + setup_participant(); + fc_dlog( logger, "connected to ${peer}", ("peer", peer_name()) ); start_read_message(); callback(true); }; @@ -1162,7 +1165,6 @@ namespace eosio { void connection::_close( connection* self, bool reconnect, bool shutdown ) { self->socket_open = false; - self->verified = false; self->set_participating(false); boost::system::error_code ec; auto& cur_sock = self->socket.raw_socket(); @@ -3932,7 +3934,7 @@ namespace eosio { "Number of worker threads in net_plugin thread pool" ) ( "sync-fetch-span", bpo::value()->default_value(def_sync_fetch_span), "number of blocks to retrieve in a chunk from any individual peer during synchronization") ( "use-socket-read-watermark", bpo::value()->default_value(false), "Enable experimental socket read watermark optimization") - ( "peer-log-format", bpo::value()->default_value( "[\"${_name}\" ${_ip}:${_port}]" ), + ( "peer-log-format", bpo::value()->default_value( "[\"${_name}\" ${_ip}:${_port}] " ), "The string used to format peers when logging messages about them. Variables are escaped with ${}.\n" "Available Variables:\n" " _name \tself-reported name\n\n" From 2bf22ef8622e4563d1f548f1e281f23426dea05d Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Fri, 9 Apr 2021 10:56:42 -0400 Subject: [PATCH 089/157] PR review concerns addressed --- plugins/net_plugin/net_plugin.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 37bf7fbf700..f6e8934c214 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -883,6 +883,8 @@ namespace eosio { inline void setup_participant() { if(my_impl->ssl_enabled) { account_name participant = participant_name_ ? *participant_name_ : account_name{}; + + std::lock_guard connection_guard(my_impl->connections_mtx); bool participating = my_impl->security_group.is_in_security_group(participant); fc_dlog( logger, "[${peer}] participant: [${name}] participating: [${enabled}]", From 40af6c5ff47787f77362b204a20640865faec0fe Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 30 Mar 2021 15:11:44 -0500 Subject: [PATCH 090/157] Extra diagnostics for CentOS failure. --- tests/Node.py | 11 ++++++++--- tests/privacy_startup_network.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index 5f22044dcb2..12b7f218241 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1504,9 +1504,14 @@ def preactivateAllBuiltinProtocolFeature(self): def getLatestBlockHeaderState(self): headBlockNum = self.getHeadBlockNum() - cmdDesc = "get block {} --header-state".format(headBlockNum) - latestBlockHeaderState = self.processCleosCmd(cmdDesc, cmdDesc) - return latestBlockHeaderState + for i in range(10): + cmdDesc = "get block {} --header-state".format(headBlockNum) + latestBlockHeaderState = self.processCleosCmd(cmdDesc, cmdDesc) + Utils.Print("block num: {}, block state: {}, head: {}".format(headBlockNum, latestBlockHeaderState, self.getHeadBlockNum())) + if latestBlockHeaderState: + return latestBlockHeaderState + time.sleep(1) + return None def getActivatedProtocolFeatures(self): latestBlockHeaderState = self.getLatestBlockHeaderState() diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index 9d6a7c91b7e..ed9aa478548 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -138,7 +138,7 @@ def verifyInSync(producerNum): featureDict = producers[0].getSupportedProtocolFeatureDict() Utils.Print("feature dict: {}".format(json.dumps(featureDict, indent=4, sort_keys=True))) - #Utils.Print("act feature dict: {}".format(json.dumps(producers[0].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) + Utils.Print("act feature dict: {}".format(json.dumps(producers[0].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) timeout = ( pnodes * 12 / 2 ) * 2 # (number of producers * blocks produced / 0.5 blocks per second) * 2 rounds producers[0].waitUntilBeginningOfProdTurn(blockProducer, timeout=timeout) feature = "SECURITY_GROUP" From 8c05ad5f87f381b2f5d5781a9da903a1c116157c Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 8 Apr 2021 15:56:56 -0500 Subject: [PATCH 091/157] Fixed use of non-shared containers in multi-index container. --- libraries/chain/controller.cpp | 21 +++++++++++----- .../chain/include/eosio/chain/controller.hpp | 2 +- .../eosio/chain/global_property_object.hpp | 25 ++++++++++++------- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 392c98dca1e..fffeb6e3c7d 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1442,7 +1442,11 @@ struct controller_impl { } ++bb._pending_block_header_state.security_group.version; - bb._pending_block_header_state.security_group.participants = std::move(gpo.proposed_security_group_participants); + bb._pending_block_header_state.security_group.participants = { + gpo.proposed_security_group_participants.begin(), + gpo.proposed_security_group_participants.end(), + bb._pending_block_header_state.security_group.participants.key_comp(), + bb._pending_block_header_state.security_group.participants.get_allocator()}; db.modify(gpo, [&](auto& gp) { gp.proposed_security_group_block_num = 0; @@ -2260,8 +2264,9 @@ struct controller_impl { } flat_set proposed_participants = gpo.proposed_security_group_block_num == 0 - ? self.active_security_group().participants - : gpo.proposed_security_group_participants; + ? self.active_security_group().participants + : flat_set{gpo.proposed_security_group_participants.begin(), + gpo.proposed_security_group_participants.end()}; auto orig_participants_size = proposed_participants.size(); @@ -2274,7 +2279,10 @@ struct controller_impl { db.modify(gpo, [&proposed_participants, cur_block_num](auto& gp) { gp.proposed_security_group_block_num = cur_block_num; - gp.proposed_security_group_participants = std::move(proposed_participants); + gp.proposed_security_group_participants = {proposed_participants.begin(), + proposed_participants.end(), + gp.proposed_security_group_participants.key_comp(), + gp.proposed_security_group_participants.get_allocator()}; }); return 0; @@ -2876,8 +2884,9 @@ const security_group_info_t& controller::active_security_group() const { my->pending->_block_stage); } -const flat_set& controller::proposed_security_group_participants() const { - return get_global_properties().proposed_security_group_participants; +flat_set controller::proposed_security_group_participants() const { + return {get_global_properties().proposed_security_group_participants.begin(), + get_global_properties().proposed_security_group_participants.end()}; } int64_t controller::add_security_group_participants(const flat_set& participants) { diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 4c61a7d92de..1c9e214afac 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -294,7 +294,7 @@ namespace eosio { namespace chain { int64_t set_proposed_producers( vector producers ); const security_group_info_t& active_security_group() const; - const flat_set& proposed_security_group_participants() const; + flat_set proposed_security_group_participants() const; int64_t add_security_group_participants(const flat_set& participants); int64_t remove_security_group_participants(const flat_set& participants); diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index bfa7feb63b0..062ddca05e9 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -62,7 +62,7 @@ namespace eosio { namespace chain { */ class global_property_object : public chainbase::object { - OBJECT_CTOR(global_property_object, (proposed_schedule)) + OBJECT_CTOR(global_property_object, (proposed_schedule)(proposed_security_group_participants)(transaction_hooks)) public: id_type id; @@ -73,8 +73,8 @@ namespace eosio { namespace chain { kv_database_config kv_configuration; wasm_config wasm_configuration; block_num_type proposed_security_group_block_num = 0; - flat_set proposed_security_group_participants; - vector transaction_hooks; + shared_set proposed_security_group_participants; + shared_vector transaction_hooks; void initalize_from(const legacy::snapshot_global_property_object_v2& legacy, const chain_id_type& chain_id_val, const kv_database_config& kv_config_val, const wasm_config& wasm_config_val) { @@ -128,10 +128,10 @@ namespace eosio { namespace chain { struct extension_v0 { // libstdc++ requires the following two constructors to work. extension_v0(){}; - extension_v0(block_num_type num, const flat_set& participants, const vector trx_hooks) + extension_v0(block_num_type num, flat_set participants, vector trx_hooks) : proposed_security_group_block_num(num) - , proposed_security_group_participants(participants) - , transaction_hooks(trx_hooks) {} + , proposed_security_group_participants(std::move(participants)) + , transaction_hooks(std::move(trx_hooks)) {} block_num_type proposed_security_group_block_num = 0; flat_set proposed_security_group_participants; @@ -153,7 +153,9 @@ namespace eosio { namespace chain { inline snapshot_global_property_object::extension_t get_gpo_extension(const global_property_object& gpo) { return snapshot_global_property_object::extension_v0{ - gpo.proposed_security_group_block_num, gpo.proposed_security_group_participants, gpo.transaction_hooks}; + gpo.proposed_security_group_block_num, + {gpo.proposed_security_group_participants.begin(), gpo.proposed_security_group_participants.end()}, + {gpo.transaction_hooks.begin(), gpo.transaction_hooks.end()}}; } inline void set_gpo_extension(global_property_object& gpo, @@ -161,8 +163,13 @@ namespace eosio { namespace chain { std::visit( [&gpo](auto& ext) { gpo.proposed_security_group_block_num = ext.proposed_security_group_block_num; - gpo.proposed_security_group_participants = std::move(ext.proposed_security_group_participants); - gpo.transaction_hooks = std::move(ext.transaction_hooks); + gpo.proposed_security_group_participants = {ext.proposed_security_group_participants.begin(), + ext.proposed_security_group_participants.end(), + gpo.proposed_security_group_participants.key_comp(), + gpo.proposed_security_group_participants.get_allocator()}; + gpo.transaction_hooks = {ext.transaction_hooks.begin(), + ext.transaction_hooks.end(), + gpo.transaction_hooks.get_allocator()}; }, extension); } From 0181fa39628a43ac29f9b59aa3639f92e92bff16 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 8 Apr 2021 16:03:42 -0500 Subject: [PATCH 092/157] Fixed setting participating_ correctly for invalid participant names. --- plugins/net_plugin/net_plugin.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index f6e8934c214..610ee7380ac 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -3678,10 +3678,8 @@ namespace eosio { // for(auto& connection : connections) { const auto& participant = connection->participant_name(); - if(!participant) { - continue; - } - if(security_group.is_in_security_group(participant.value())) { + if(participant && security_group.is_in_security_group(participant.value())) { + ilog("REM update_security_group participant in: ${part}, participating: ${p}",("part",participant->to_string())("p",connection->is_participating())); if(!connection->is_participating()) { connection->set_participating(true); connection->send_handshake(); From 5f7cd700988a335b5b7f41de894f3c06d5eeaf92 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 9 Apr 2021 10:09:48 -0500 Subject: [PATCH 093/157] More global_property_object changes. --- unittests/security_group_tests.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/unittests/security_group_tests.cpp b/unittests/security_group_tests.cpp index 5aa935478fd..85afdc4e1a4 100644 --- a/unittests/security_group_tests.cpp +++ b/unittests/security_group_tests.cpp @@ -5,6 +5,9 @@ #include +using boost::container::flat_set; +using eosio::chain::name; + namespace eosio { namespace chain { // block_header_state_v0 should has the same layout with block_header_state except @@ -190,7 +193,9 @@ BOOST_AUTO_TEST_CASE(test_snapshot_global_property_object) { std::visit( [&gpo](const auto& ext) { BOOST_CHECK_EQUAL(ext.proposed_security_group_block_num, gpo.proposed_security_group_block_num); - BOOST_TEST(ext.proposed_security_group_participants == gpo.proposed_security_group_participants ); + flat_set gpo_sg_participants{gpo.proposed_security_group_participants.begin(), + gpo.proposed_security_group_participants.end()}; + BOOST_TEST(ext.proposed_security_group_participants == gpo_sg_participants); }, row.extension); } From c879f9091a9a7c26f9853ec02d75b2c77b60e6b0 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 9 Apr 2021 12:36:49 -0500 Subject: [PATCH 094/157] Integration test script fixes. --- tests/Cluster.py | 28 ++++++---------------------- tests/Node.py | 40 +++++++++++++++++++++++++++++++++++++--- tests/WalletMgr.py | 3 ++- 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index ff064cc0364..b3a81794639 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -245,27 +245,9 @@ def insertSpecificExtraNodeosArgs(node, insertStr): certAuth = os.path.join(privacyDir, "CA_cert.pem") def getArguments(number): - # this function converts number to eos name string - # eos name can have only numbers 1-5 - # e.g. 0 -> 1, 6 -> 5.1, 12 -> 5.5.2 - def normalizeNumber(number): - assert(number > 0) - if number <= 5: - return str(number) - cnt = number - ret = "5" - while cnt > 5: - cnt = cnt - 5 - if cnt > 5: - ret = "{}.5".format(ret) - else: - ret = "{}.{}".format(ret, cnt) - assert(len(ret) <= 13) - - return ret - - nodeCert = os.path.join(privacyDir, "node{}.crt".format(normalizeNumber(number+1))) - nodeKey = os.path.join(privacyDir, "node{}_key.pem".format(normalizeNumber(number+1))) + participantName = Node.participantName(number) + nodeCert = os.path.join(privacyDir, "{}.crt".format(participantName)) + nodeKey = os.path.join(privacyDir, "{}_key.pem".format(participantName)) return "--p2p-tls-own-certificate-file {} --p2p-tls-private-key-file {} --p2p-tls-security-group-ca-file {}".format(nodeCert, nodeKey, certAuth) for node in range(totalNodes): @@ -370,9 +352,11 @@ def normalizeNumber(number): def createDefaultShapeFile(newFile, cmdArr): cmdArrForOutput=copy.deepcopy(cmdArr) + cmdArrForOutput.append("--shape") + cmdArrForOutput.append("ring") cmdArrForOutput.append("--output") cmdArrForOutput.append(newFile) - s=" ".join(cmdArrForOutput) + s=" ".join([("'{0}'".format(element) if (' ' in element) else element) for element in cmdArrForOutput.copy()]) if Utils.Debug: Utils.Print("cmd: %s" % (s)) if 0 != subprocess.call(cmdArrForOutput): Utils.Print("ERROR: Launcher failed to create shape file \"{}\".".format(newFile)) diff --git a/tests/Node.py b/tests/Node.py index 12b7f218241..17a53ee2ac2 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -821,6 +821,7 @@ def getTableColumns(self, contract, scope, table): # returns tuple with indication if transaction was successfully sent and either the transaction or else the exception output def pushMessage(self, account, action, data, opts, silentErrors=False, signatures=None): cmd="%s %s push action -j %s %s" % (Utils.EosClientPath, self.eosClientArgs(), account, action) + Utils.Print("cmd: {}".format(cmd)) cmdArr=cmd.split() # not using __sign_str, since cmdArr messes up the string if signatures is not None: @@ -1410,12 +1411,23 @@ def getSupportedProtocolFeatureDict(self, excludeDisabled=False, excludeUnactiva break return protocolFeatureDigestDict - def waitForHeadToAdvance(self, blocksToAdvance=1, timeout=None): - currentHead = self.getHeadBlockNum() + def waitForHeadToAdvance(self, blocksToAdvance=1, timeout=None, reportInterval=None): + originalHead = self.getHeadBlockNum() + targetHead = originalHead + blocksToAdvance if timeout is None: timeout = 6 + blocksToAdvance / 2 + count = 0 def isHeadAdvancing(): - return self.getHeadBlockNum() >= currentHead + blocksToAdvance + nonlocal count + nonlocal reportInterval + nonlocal targetHead + head = self.getHeadBlockNum() + count += 1 + done = head >= targetHead + if not done and reportInterval and count % reportInterval == 0: + Utils.Print("waitForHeadToAdvance to {}, currently at {}".format(targetHead, head)) + + return done return Utils.waitForTruth(isHeadAdvancing, timeout) def waitForLibToAdvance(self, timeout=30): @@ -1571,6 +1583,28 @@ def findStderrFiles(path): files.sort() return files + @staticmethod + def participantName(nodeNumber): + # this function converts number to eos name string + # eos name can have only numbers 1-5 + # e.g. 0 -> 1, 6 -> 5.1, 12 -> 5.5.2 + def normalizeNumber(number): + assert(number > 0) + if number <= 5: + return str(number) + cnt = number + ret = "5" + while cnt > 5: + cnt = cnt - 5 + if cnt > 5: + ret = "{}.5".format(ret) + else: + ret = "{}.{}".format(ret, cnt) + assert(len(ret) <= 13) + + return ret + return "node{}".format(normalizeNumber(nodeNumber + 1)) + def analyzeProduction(self, specificBlockNum=None, thresholdMs=500): dataDir=Utils.getNodeDataDir(self.nodeId) files=Node.findStderrFiles(dataDir) diff --git a/tests/WalletMgr.py b/tests/WalletMgr.py index 8b7e4957277..dbc891f9259 100644 --- a/tests/WalletMgr.py +++ b/tests/WalletMgr.py @@ -127,7 +127,7 @@ def create(self, name, accounts=None, exitOnError=True): portStatus="AVAILABLE" else: portStatus="NOT AVAILABLE" - if Utils.Debug: Utils.Print("%s was not accepted, delaying for %d seconds and trying again. port %d is %s. %s - {%s}" % (cmdDesc, delay, self.port, pgrepCmd, psOut)) + if Utils.Debug: Utils.Print("%s was not accepted, delaying for %d seconds and trying again. port %d is %s. %s - {%s}" % (cmdDesc, delay, self.port, portStatus, pgrepCmd, psOut)) time.sleep(delay) continue @@ -153,6 +153,7 @@ def create(self, name, accounts=None, exitOnError=True): if accounts: self.importKeys(accounts,wallet) + Utils.Print("Wallet \"%s\" password=%s." % (name, p.encode("utf-8"))) return wallet def importKeys(self, accounts, wallet, ignoreDupKeyWarning=False): From 6ea14112d065ddae7e623c99530a056655f1cf4f Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 9 Apr 2021 12:41:06 -0500 Subject: [PATCH 095/157] Added simpler scenario for security group testing. --- tests/CMakeLists.txt | 3 + tests/privacy_simple_network.py | 230 ++++++++++++++++++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100755 tests/privacy_simple_network.py diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ea766291049..7f13b5304d7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -62,6 +62,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_contrl_c_test.py ${CMAKE_CURRE configure_file(${CMAKE_CURRENT_SOURCE_DIR}/blockvault_tests.py ${CMAKE_CURRENT_BINARY_DIR}/blockvault_tests.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/generate-certificates.sh ${CMAKE_CURRENT_BINARY_DIR}/generate-certificates.sh COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_startup_network.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_startup_network.py COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_simple_network.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_simple_network.py COPYONLY) #To run plugin_test with all log from blockchain displayed, put --verbose after --, i.e. plugin_test -- --verbose add_test(NAME plugin_test COMMAND plugin_test --report_level=detailed --color_output) @@ -118,6 +119,8 @@ add_test(NAME eosio_blocklog_prune_test COMMAND tests/eosio_blocklog_prune_test. set_property(TEST eosio_blocklog_prune_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME privacy_startup_network COMMAND tests/privacy_startup_network.py -p 1 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST privacy_startup_network PROPERTY LABELS nonparallelizable_tests) +add_test(NAME privacy_simple_network COMMAND tests/privacy_simple_network.py -p 1 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST privacy_simple_network PROPERTY LABELS nonparallelizable_tests) # Long running tests add_test(NAME nodeos_sanity_lr_test COMMAND tests/nodeos_run_test.py -v --sanity-test --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py new file mode 100755 index 00000000000..5f2b511d1ea --- /dev/null +++ b/tests/privacy_simple_network.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 + +from testUtils import Account +from testUtils import Utils +from Cluster import Cluster +from WalletMgr import WalletMgr +from Node import BlockType +from Node import Node +from Node import ReturnType +from TestHelper import TestHelper + +import copy +import decimal +import re +import signal +import json +import os +import time + +############################################################### +# privacy_startup_network +# +# General test for Privacy to verify TLS connections, and slowly adding participants to the security group and verifying +# how blocks and transactions are sent/not sent. +# +############################################################### + +Print=Utils.Print +errorExit=Utils.errorExit +cmdError=Utils.cmdError +from core_symbol import CORE_SYMBOL + +args = TestHelper.parse_args({"--port","-p","-n","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run" + ,"--sanity-test","--wallet-port"}) +port=args.port +pnodes=args.p +apiNodes=0 # minimum number of apiNodes that will be used in this test +# bios is also treated as an API node +minTotalNodes=pnodes+apiNodes +totalNodes=args.n if args.n >= minTotalNodes else minTotalNodes +if totalNodes > minTotalNodes: + apiNodes += totalNodes - minTotalNodes + +Utils.Debug=args.v +dumpErrorDetails=args.dump_error_details +keepLogs=args.keep_logs +dontKill=args.leave_running +onlyBios=False +killAll=args.clean_run +sanityTest=args.sanity_test +walletPort=args.wallet_port + +cluster=Cluster(host=TestHelper.LOCAL_HOST, port=port, walletd=True) +walletMgr=WalletMgr(True, port=walletPort) +testSuccessful=False +killEosInstances=not dontKill +killWallet=not dontKill +dontBootstrap=sanityTest # intent is to limit the scope of the sanity test to just verifying that nodes can be started + +WalletdName=Utils.EosWalletName +ClientName="cleos" +timeout = .5 * 12 * pnodes + 60 # time for finalization with 1 producer + 60 seconds padding +Utils.setIrreversibleTimeout(timeout) + +try: + TestHelper.printSystemInfo("BEGIN") + cluster.setWalletMgr(walletMgr) + Print("SERVER: {}".format(TestHelper.LOCAL_HOST)) + Print("PORT: {}".format(port)) + + cluster.killall(allInstances=killAll) + cluster.cleanup() + Print("Stand up cluster") + + if cluster.launch(pnodes=pnodes, totalNodes=totalNodes, prodCount=1, onlyBios=False, dontBootstrap=dontBootstrap, configSecurityGroup=True) is False: + cmdError("launcher") + errorExit("Failed to stand up eos cluster.") + + Print("Validating system accounts after bootstrap") + cluster.validateAccounts(None) + + def getNodes(): + global cluster + nodes = [cluster.biosNode] + nodes.extend(cluster.getNodes()) + return nodes + +# cluster.biosNode.kill(signal.SIGTERM) + + def reportInfo(): + Utils.Print("\n\n\n\n\nInfo:") + for node in getNodes(): + Utils.Print("Info: {}".format(json.dumps(node.getInfo(), indent=4, sort_keys=True))) + + Utils.Print("\n\n\n\n\nCheck after KILL:") + Utils.Print("\n\n\n\n\nNext Round of Info:") + reportInfo() + + producers = [cluster.getNode(x) for x in range(pnodes) ] + apiNodes = [cluster.getNode(x) for x in range(pnodes, totalNodes)] + apiNodes.append(cluster.biosNode) + + def createAccount(newAcc): + producers[0].createInitializeAccount(newAcc, cluster.eosioAccount) + ignWallet = cluster.walletMgr.create("ignition") # will actually just look up the wallet + cluster.walletMgr.importKey(newAcc, ignWallet) + + # numAccounts = 4 + # testAccounts = Cluster.createAccountKeys(numAccounts) + # accountPrefix = "testaccount" + # for i in range(numAccounts): + # testAccount = testAccounts[i] + # testAccount.name = accountPrefix + str(i + 1) + # createAccount(testAccount) + + blockProducer = None + + def verifyInSync(producerNum): + Utils.Print("Ensure all nodes are in-sync") + lib = producers[producerNum].getInfo()["last_irreversible_block_num"] + headBlockNum = producers[producerNum].getBlockNum() + headBlock = producers[producerNum].getBlock(headBlockNum) + global blockProducer + if blockProducer is None: + blockProducer = headBlock["producer"] + Utils.Print("headBlock: {}".format(json.dumps(headBlock, indent=4, sort_keys=True))) + headBlockId = headBlock["id"] + error = None + for prod in producers: + if prod == producers[producerNum]: + continue + + if prod.waitForBlock(headBlockNum, reportInterval = 1) is None: + error = "Producer node failed to get block number {}. Current node info: {}".format(headBlockNum, json.dumps(prod.getInfo(), indent=4, sort_keys=True)) + break + + if prod.getBlock(headBlockId) is None: + error = "Producer node has block number: {}, but it is not id: {}. Block: {}".format(headBlockNum, headBlockId, json.dumps(prod.getBlock(headBlockNum), indent=4, sort_keys=True)) + break + + if prod.waitForBlock(lib, blockType=BlockType.lib) is None: + error = "Producer node is failing to advance its lib ({}) with producer {} ({})".format(prod.getInfo()["last_irreversible_block_num"], producerNum, lib) + break + + if error: + reportInfo() + Utils.exitError(error) + + for node in apiNodes: + if node.waitForBlock(headBlockNum, reportInterval = 1) == None: + error = "API node failed to get block number {}".format(headBlockNum) + break + + if node.getBlock(headBlockId) is None: + error = "API node has block number: {}, but it is not id: {}. Block: {}".format(headBlockNum, headBlockId, json.dumps(node.getBlock(headBlockNum), indent=4, sort_keys=True)) + break + + if node.waitForBlock(lib, blockType=BlockType.lib) == None: + error = "API node is failing to advance its lib ({}) with producer {} ({})".format(node.getInfo()["last_irreversible_block_num"], producerNum, lib) + break + + Utils.Print("Ensure all nodes are in-sync") + if node.waitForBlock(lib + 1, blockType=BlockType.lib, reportInterval = 1) == None: + error = "Producer node failed to advance lib ahead one block to: {}".format(lib + 1) + break + + if error: + reportInfo() + Utils.exitError(error) + + verifyInSync(producerNum=0) + + featureDict = producers[0].getSupportedProtocolFeatureDict() + Utils.Print("feature dict: {}".format(json.dumps(featureDict, indent=4, sort_keys=True))) + + reportInfo() + Utils.Print("Activating SECURITY_GROUP Feature") + + #Utils.Print("act feature dict: {}".format(json.dumps(producers[0].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) + timeout = ( pnodes * 12 / 2 ) * 2 # (number of producers * blocks produced / 0.5 blocks per second) * 2 rounds + for producer in producers: + producers[0].waitUntilBeginningOfProdTurn(blockProducer, timeout=timeout) + feature = "SECURITY_GROUP" + producers[0].activateFeatures([feature]) + if producers[0].containsFeatures([feature]): + break + + Utils.Print("SECURITY_GROUP Feature activated") + reportInfo() + + assert producers[0].containsFeatures([feature]), "{} feature was not activated".format(feature) + + if sanityTest: + testSuccessful=True + exit(0) + + def publishContract(account, file, waitForTransBlock=False): + Print("Publish contract") + reportInfo() + return producers[0].publishContract(account, "unittests/test-contracts/security_group_test/", "{}.wasm".format(file), "{}.abi".format(file), waitForTransBlock=waitForTransBlock) + + publishContract(cluster.eosioAccount, 'eosio.secgrp', waitForTransBlock=True) + + addTrans = [] + action = None + for producerNum in range(pnodes): + if action is None: + action = '[[' + else: + action += ',' + action += '"{}"'.format(Node.participantName(producerNum)) + action += ']]' + + addTrans.append(producers[0].pushMessage(cluster.eosioAccount.name, "add", action, "--permission eosio@active")) + addTrans.append(producers[0].pushMessage(cluster.eosioAccount.name, "publish", "[0]", "--permission eosio@active")) + + highestBlock = None + for trans in addTrans: + Utils.Print("trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) + + for i in range(10): + Utils.Print("\n\n\n\n\nNext Round of Info:") + reportInfo() + time.sleep(5) + + testSuccessful=True +finally: + TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails) + +exit(0) From caf3eb4d403e59a4f3fd14b43f32104089d47bb0 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 9 Apr 2021 12:46:54 -0500 Subject: [PATCH 096/157] Added wasm and abi for simple test usage. --- .../security_group_test/CMakeLists.txt | 3 + .../security_group_test/eosio.secgrp.abi | 88 ++++++++++++++++++ .../security_group_test/eosio.secgrp.wasm | Bin 0 -> 43959 bytes 3 files changed, 91 insertions(+) create mode 100644 unittests/test-contracts/security_group_test/eosio.secgrp.abi create mode 100644 unittests/test-contracts/security_group_test/eosio.secgrp.wasm diff --git a/unittests/test-contracts/security_group_test/CMakeLists.txt b/unittests/test-contracts/security_group_test/CMakeLists.txt index b5ca6ad849d..b69f12723ef 100644 --- a/unittests/test-contracts/security_group_test/CMakeLists.txt +++ b/unittests/test-contracts/security_group_test/CMakeLists.txt @@ -4,3 +4,6 @@ else() configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/security_group_test.wasm ${CMAKE_CURRENT_BINARY_DIR}/security_group_test.wasm COPYONLY ) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/security_group_test.abi ${CMAKE_CURRENT_BINARY_DIR}/security_group_test.abi COPYONLY ) endif() + +configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/eosio.secgrp.wasm ${CMAKE_CURRENT_BINARY_DIR}/eosio.secgrp.wasm COPYONLY ) +configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/eosio.secgrp.abi ${CMAKE_CURRENT_BINARY_DIR}/eosio.secgrp.abi COPYONLY ) diff --git a/unittests/test-contracts/security_group_test/eosio.secgrp.abi b/unittests/test-contracts/security_group_test/eosio.secgrp.abi new file mode 100644 index 00000000000..445cdd5b6fb --- /dev/null +++ b/unittests/test-contracts/security_group_test/eosio.secgrp.abi @@ -0,0 +1,88 @@ +{ + "____comment": "This file was generated with eosio-abigen. DO NOT EDIT ", + "version": "eosio::abi/1.2", + "types": [ + { + "new_type_name": "participants_t", + "type": "name[]" + } + ], + "structs": [ + { + "name": "add", + "base": "", + "fields": [ + { + "name": "participants", + "type": "participants_t" + } + ] + }, + { + "name": "clear", + "base": "", + "fields": [ + { + "name": "max", + "type": "uint32" + } + ] + }, + { + "name": "publish", + "base": "", + "fields": [ + { + "name": "max", + "type": "uint32" + } + ] + }, + { + "name": "remove", + "base": "", + "fields": [ + { + "name": "participants", + "type": "participants_t" + } + ] + } + ], + "actions": [ + { + "name": "add", + "type": "add", + "ricardian_contract": "" + }, + { + "name": "clear", + "type": "clear", + "ricardian_contract": "" + }, + { + "name": "publish", + "type": "publish", + "ricardian_contract": "" + }, + { + "name": "remove", + "type": "remove", + "ricardian_contract": "" + } + ], + "tables": [], + "kv_tables": {}, + "ricardian_clauses": [], + "variants": [], + "action_results": [ + { + "name": "clear", + "result_type": "bool" + }, + { + "name": "publish", + "result_type": "bool" + } + ] +} \ No newline at end of file diff --git a/unittests/test-contracts/security_group_test/eosio.secgrp.wasm b/unittests/test-contracts/security_group_test/eosio.secgrp.wasm new file mode 100644 index 0000000000000000000000000000000000000000..387eca0b1a6ef44f11471101a6d8d5c73fafc199 GIT binary patch literal 43959 zcmeIb3zS_~dFQ#$IrrYWx6ZAqBdH`yDuK_v*oh1ll-@EiW1Rxa4-6PE1V}P+x$4TQ zhw6DtmT6g42IF>mASiUYLzwA~lF1qeVrOY4Nrxusp%O8HOgd?0Xa=W4646c4^d!>} zYjwIw(uwBx|Mt0$l1jD(nWnQ=mvkRzpS{2R?eD$6z0baGbn!&soD2Re_^yYYdpKHN zUJjP|II|p_dAPvUvU@l<;~p-Y;g36^3LN)JDzM7o@)Pdi;c@e1`jm%9)45)5|%m<%fgQd6rI9bH^(EHqt7MakIG!a`%Fd32#M)|fciSa3ZmE{;AjOBJW@r4x-4i;boW zbkW1biP@74f2CVvbC3IrD*Y`sXTd=omQ}m&=+VZhW@Bc2@mTZN9#>IW`RLITOViC` zdtI+CJW5m;9UpI3Xu}vc7_nh-^ys4vL9}5}!$2&}tN}7Uak34h z2Sn!E4Y-=_H0Nrjb4BlqOOLc0R*xQSE-cNAHII$cT~(cTiWVvAvoXyrOw2SFk52m$ z40hA`=!wSS;^?Cd*RPrzAky^gV~vGJW|z8IBWBG&E2G_jYW7jGG&erl^z=+rR$pj{ z_(vBTux+CG_|ZofW|!uU&W$cKC&nh`M!~^tP{o^};7iv^M#q{HvolBOWc29b#QPg= zqZ%g*jnVO=UH6(NhVEqjPi9 zkGoe_s)5Ubxb)z|$0v_1_kQ_j!uf(rUK{^L;8K@-C`fKgT^hW-aCl_J|9x^#8gi-9 zKtiQZ7tJ1;gXaC~&|DT`oz-aX3%1U>m%q33;@j?(ShoIc& zGB*%syzdM@%ypKWtOt|z@Bml7^r^%3D0P$Bz+^TzUv~$>rw-FF%#vp6-mK<)4<=PV z4fs+EVr~bAUABCW?q+9KT@uD=6#u=z_Zg&ax*lBBaiEUF$$GS=V<_6{xR7@ospEq0 zIBIn)3~He|4zKRGGe`y}4i&ejE)HFMAPAOU#dxRclphGI+=K?zouTUtb>*;1)pO?V z;`76Y#{8eXbd^kXSTeU25+R|DjRxN1F zAd}a$C?eveDpJo=5F9F|fz=MO(+~&*>xHamz8-SJWv3<~`>9DvQ|e8@JbI)2^kk~4 z6xsDa4A??;YJQT&rzYz~?$t_ZL|4|VXQAD!K)&-3yXkka7=n z%(m1bA)XcP>vMht$*@M0ypDe8n?8&PwMde%Hwc$OF-YA|EeH=sw1fbzFUVXC0`3g| zD5~9?pnf?JK67ltImqVKA$xW$46GcI`3PBohy9Qk; zuSYdN-`y7)Xi>EmSk1y@JpkW&DCULPmB0GS$0p~ql_x*&)G2jh@|9f)=EV`~My+KKP;z0=oNNBRi&^<^&G5G{;!Ibo>rT$cuY z0$xye1&LxIix6fyDyX9z>9B)zn0MF0u>i*Pf?8xQz7LAPZix-7a3%#}VZC6isx#sM zkTNK(_8PII-soFMi&I{jq{umx6o8B&O2tMAbR=}S)CWAz0hmC2Ah33{gg8@KxRi#d zYpPmd(=^SxP9p%Uf$0SX(8!HvLmMfL@Lq#(oyw5nYH5X@Ani1Q_;oHP zglB?S8FZLnoa=zR9uuSiFhL(V6O4E{=UeTsVnT&+XVUT5zKQs8)n%74{%^>>iNP!h zd&JSjza(V{4<~o22M5Jk$Ucy{dquzG@OBg1o#CLwamQO!C}*UrTQ&r)b-Pe!;5)-| z-8g5wn|dy@vYi`3s_E__3=1L`9+qMG&xfr?F@0xv1O2J`OZMiwSjql>>>?%-8Ht|^ z%z(-?++Pn6>NHq*y}RF!>bf)&4G)An2rJydDDjmY%?q+^Db&kSne}Yh>@@?vU85eh zV4%!>Vgke>8){}ZFU5z!za=Yu3k78odRa?AdTtvdkr$0{4DmNj{e{41=6RR9YuIM+?U22*&=Z^K>Zj2Fo?li-ZP=}aKte7m_5d#^%-l4!@{2Or; zEf<+g`=J=vAiKq)1=)5dEdhW>NFT>#u4Ibk@(XDJka_>fJ+(rd1+^me3t6g;3fW_m zS@FJFF;0u91D3aBH{V4lkcITZRk1&E0oY_1W*Y1|GGvurc8@vEvvOpGY#6rR7{(sk zh3uP(d2Xy}meds9{P3^1Nyv~T_w^NkCL+~wP9l>=0p+2Tv4O%7SL>1Ml5BVGj##)d z;*ulwEg&c<*8=(7AnQ$oA*7<+k0+aYxuy?kS#l(g2|c8&7nkNc%^pOnvdWwLA{c?2 zPm5XawAkg5M&e!ugHs~0$(+Y6cn(NE_H%#q&x8Arc3Ay9L(Dtstp#b%Nbi|?ISrqy zSCH^`_k}kqRB1V_oUZkNWiK%_Nt~`xQ6^$OX@(og=CoMlk z>#HaM+0$Mqfsl0Ff%P{mon6ftlO(OKXsVQ9tVe!AwRwtz=gQhKgl{!m(eLbB8oFNew{;vdg4Ba85N_<{rYQ~M|o zIx0TmWBEY%fIe`|@Zn0&qpjxACr>?)!_zD!hi8>& zdil7$-9MClYyfyt4U$bglh2Rrd-}6K8Se+IW@P#(j0`{XY__rz&ZnV$y+TrI;rWr_ zr@0nai5x!o)WN76znM808NHxgTJup_&~Ap3)0&A$#vF<1QjmoizAbf6JtzxjUzoGt zy1rH_6mVU^awPw`f;lVR0<|pAzz&QkuHGO&$VKvQl-+Qv_Xz%cy_|QGmNeTdGTBT6 zpsQ{#A!PBkD5;YZEB^IUStH)cf2wG=gy;h;n9{N-D zIN6vv7-a2LJ5Dy4<7RdShl|-|XIR-#z>4{fQ1M;Q*%Z!3TW&S7kzBGy6&iV4u=-1@ zPGicxZmkJT#jR(^36?d4hoL42^g^g&c9FKAac$cdeA~F)7J#=!4=S%#X^*JYp?iw4 z$LkeWpAZ!vdj5%e-^lX2;0aex)YN}D?L`Xpuq- z=Bx3@UC-1iY2QfjbgfScP*6c17*bHdHDKVm5_TXGQwoc(#U?Fl;tfKuM(LomB>MIN zAx0#-F&bWr(V-4T!#Q_4)Ooi?=P`<`2gqL#VxV9cf>xV=1eV~6yF#aIYF#!#fkVXn z5UoRVKp_=r$$~GnX(Fjyj-qxJLZBbeaplp;qxdN@}Y$cFw zE7l@Oej!)K_Hb4_kSP1BE9j`+W#?ATD6gu@-H@5!i}wav)~{eRL%BQ9?l6QPWp}&7 zIYuA3;)D^1$vO%nNmAD4bv`V?h_bIg`GkMRPJZIA$w3RB;WcoFWjx6T2?6Rd5^U)3 z(=-=uW=uqIF3@csdZC7p{-@s1i(@!b;iXT-=_NKZVC>HK8s$d_fkHHf5G!O7#AV5w zeKcK@C&f6-;R#{tmF&6CIj#NFdecgFF8?0q`c_uBR7rdIP^yS&s6{j`tq{-DSmwl+ zkSeMuRaB+Ficb|)@>EeZPOAtkH~Z`+siJBtRa9-IiUL~gI+6v=DXaCeyVL+*>Yor= zpwEBT&Ii40@EVad&;C7mHuJ1izTVU4Df_bFco<)$ArUE5=xRu0>B%QLUQJur?P002Ep>;fLXF5cy&Y`E|MX5`C9i zjsL^9F!G*O<3ES>m>nf&a~4Ff!1SSBk?}7ZKP@ZB1gGj3#54m){cR~$1@m5^3Vxz0 z2U7KmUgEf)(xiR=+Z&x*S^kj(UzA%OJfJ*!v6{$YiTeVt)v~nOUkIHG!WJgi;E|`t zNR(Yo@fTO(oZCCeol5$V%G_yO!ZzWPWd`o<>svGEYTpgp3W)Jps42mZat>-YqnRtyAN&FoAj^NlX+rUhOi1LWOE+RYp)`GA)*|lg29dT(~j9gH69$$m*(R zrQ0jnYmvPXS4+67q$jt!YzLf#NBt6_^-Z$wYIXX?Fj0SP8~_nAx*PQcX+n~#mgEDG z^$&$ii*Kt}?y7BIOdIK|#A_3;L9RFR3BgTS(uKC|AxAu50%B-E6X-4x$w(ERO6@sk!mNbVEB7 zSp6%-Nj^lVpo12^ppwJ&L96W5w7x-Vdf3K{X^4_OBI9Mo(f5n_oYJ|UO!_Fy=PzdT z<}i9y&l+P`apctL+6IU$zSTCuqu?}5*zi8=>0oU`TCtG03gCOlh_m9Bd|!P-Zny}^ zmfz8b{{$bzYeR0rh_ek}k8Ni`f%y%28)|Cl2F_Ge&}wv*v4@@;jbVht*AD2Jz(Rf- z&Q?;$y z$Lft+X1L=$1I>NMxM%vmVe6bTzyq_uT@baNyFKmq%L)}%=4uIRv#e0=Tl>@go#ExXvVg2LA7A-sP}7DG@OZfD0v>3qm4sZXhC8cpA-SW8*r6mfNz)EntzdR zjO((d+?0BhY@{log)cErP2FykFSa|EVLWrR>?KZVb-1 zF_7Xx`k|~W^&DbikHOD1)3!R+%D9WNHXnF{S0|=^8%c&L^?DRm69}N2G0IE;zZv~_ zR;DH0o>i3C*`96`;EmY^J}AbxD4v6H?ZK`TC#A2>xO~YE>aqq!6_2zyvLfqGRbOJf zrEd42E@@CUYue~X16cs=y~V}uPR?@y$W?))N~e>HDxqpuC+BpPZe;UZGh6ZL6LaEc zJ&f~e0lW^3b1Jb`@=qj=aayhWN_^s}iiW5STES}?LzXA7dP8I%vsap(Ov||foGrB! z;2hulU{?4hAq84SX{YbADxpOe{+GdXqmX~$g5`k>`AZ)86;%gV2f$?^Pm?YH)b&>$ z`Ab3`)iidO&4j1@ED$0$CMcF(uCRgF=Gg234G3rT{&b@pj?BIWlx`zMyLoyl-9Akh%pYwY zIUuVaqz1Aqp)bQf;e{94$ZQJrAq$3M+nmq%HfFadL&}FD*v(*;UHo@{&JK0#HOmpK zvJ43Kv|I~l9n}1uiGF6x;7A4!SoL6k=tf@gblO9hR2+Fy#9$+hnu!^#TZ|ZiR_Z1H z<;)WVVYpepaphT00HR`rEew&j>Jhyw<*jb+VV^al_EYn;Y6A_kF?Q3WET=j}w@Pjp zMpeIMw4X%dU$c*o)`p@dI-}Qi9jjPYIm6YaLCLrX)S^z>Lm{{WnL&$=Y65$h^XBc1 zh(sG1fflWIAsRiNq{0Tmh&%*hnlt7!7uSRAx1p}wi2tT-!V1-_I@ZcpXMM?VqxzRW z^B=!tZYTfh8-&+yu=3Sk{|o0ku&=%Wo<<0**Ta=7fB8?X2a|yS#H>W^%7j=%tlY3v z6yz4o%hDc0HJALf=VWag+dz7n>^!eUFrGn~toJiGNytDtpwSOdGey9))7H4mbg?~p zxK6YbAn`$$#Pc1oPow#C+qA%@Wn({U_5p+*LQ;&t#!cI$pAMmutN?9Kw@a86X5l+b zlWpB3=ZS`fa1Mc#|W;dga)}0)U5t;WesMR zD>-I4CT+}e8CVCiOZ2ubW*503nfKyon5}xuE@|pTyAEa-RYHR<%&6AV^Edl6^SuHSHqIB@CZ z$vpo+@c&hkgHy>vmWH_Gaad&mPL*;T7wP*lv3|6Z=nb?#m)*G-ULDAzY60(q8ugQN*)c}^{YW%hfQZC9^I*<;bphh71jNGi@JbwKb{(V3~8NxBOLnLtIh=6L_MByBp$=Z?n z#k+i_<}8K9bc12Y-q(yu-59p6$eJ*Qvpuaqb|72%Kj}?|0%kJGHsG)4fG>9duW0C0 z>H^H&eCP)FvIh*5+AB<&Tc}I_;T^DEk+b+x4*z@y{;CkCP8WRccEdlP!$-ZkK($c4 z*a3A(ps3RYio4yQE&@suw?EZ_zQ*MncjZ;?N)QVISU}wtJ=4P-mu#}Smx<#b_L{m9 zgm0ECwc$y@#M&l8)RoXt-!$u@j{z?!guB!Z;SeqpH16g|nwVZJnN`ph2=)-pXJ^uz zrdZ^Jv=aW!$$MJ)x_gdg!J?nSGQ5Y8f>dk$L2C=9#&|v-W0<^GwEA?(+qCmWg8pQeUn0H~ zwrxt-rRZmaG;V3=NpNr3W%6~KPl)uajUmFC9euF?s>($`Yt5xbR6hdvoF}uS)x`KZ+BY)&TGNG+> zp_Wjz$iP0td)REC+WsG6!*X%dS}uC&wNIUX!a8V8}5;$!ZH*h4FLY9`zfsw3X z{WT3FyL2c6hfGFF+5`ruFC57i@W~l#Ly_`3ilZ1{q)qAp-y=d%YsEHlznuHBM+T4_ z2SH|2rR{rmn;{H{tP5Bjv4b!FHiVSMRrSz)Xu;Ywd>i5E;1uT;-ThC z-Br43z=%@(kAg^>xq>Iy;c$-aTp{~ELoKBE4XWF@U{C7z!x`eoJN?yGe>G?oQO+Y= z=WT-IeQ9`G#G^ad`UYwoX0Agz8!B@PqjV)HF)AfOx2!JBW19 z(*}%h8@(X=_R-tC2E88-*vV~M;e=cS%27A%Yzz7KGmm9La3{B*(kJV#U7}Z2K=K8+OeUWwmI!e(5{rzu5BpTP{jUpPr>+$K_T?pSj6B< z>oXY6!cu$g4AOgcYYQ^ndsf&MWHFzIx`wa21sUZ%5ROwTnY@lrV3S${Yi~;K2!lj| zwVirNVH84J@#oYf={fdBA=<;PB|juYCD+{t+CP zjAJvgrrFA`e))AXZ06PHG!Hs^&zW8M`=o|^-H;7wY)a39ni4H+7ILU?$P!$(N+L50 zlOCRb4N~S)>htVxxfzlnSFS|!+1L2Gwjv!8X{hV*$O7CwE6HY@6khq@U;i(^|J4oojS)%s zN1X&Xkzq@Y^B7SpZ!JaDE?8s$J)*FCD2jOt48sd> zAs1lr#IqmC2SF-a#lgsLV^zg&a16*Ob3Flu^Lm~&jYuTIkroI3E2H?03IjR77Zrx8 zEZEBthW?ijhGa6|(y7)z=$)s8w3w;K-;R!G1tnK16-*>!623@9ECDU^fkR1Z z%razqL86+*?CEN6d(dP^CzRMM2n4h&60?X1&Rd(=)+JlvXH&k8Mv2lsVALHO_S?<0 z)=p1aFp^D(vR5m$)Mt)NIpYO7lvidXJPk>*TweU#)~!4mGv8_t@$E&vxa2q z$1ZGevQ1`mC!I3%`vAF8wmoTT5&(5A`mHrkI_aQaesT1H?Yr=uzS>e(Im*BxdC7{Z z3@_vEpo{JsJ6b(uA7?G)=Ev&JDCzkOL0o@t5V}I45b@vKgGfoOKe0`fA}C-PRVc17 z#qIMQQBqQgIx+NV^7X5l1ggN?y4_^eO7H|7um#nDe4P??#I*>n)gt|4bVjXufA)Qv z>VMJu@75s1*Zzl!xu{+DBe>=cGMWFm(3QCF3tT~h%Xe4I5y+9yV?R;$(+GbhOL3sc zFN(?L%1VDNaIL?v^+lgi>mp!~)oga2ohQA1k8Ud2i?SCShaYHPYLxv?23b#{r~ODM zgyI9>r7++xH1#2NtY(XV(dQYhJWAx|ZS!IdBGQ&eXH+RUwd9}$t}5AzKYv4Oq_gwL z3JJ(l^L27NtoHDXOg$R%o0znZIn%R{{Q|B16YhC3+AIu>-1T?s8&0PiK1ANOKTGGc zue!zTKO!S+II*M$QR`$j3|1SI&swQ(OcOcSiWo@RJg4Ro|aXAoC_O5gVWV;6giGq%}Ax$flth$t&)X zB?UjW0k+1oDcvM3C%YH12u69=P_$dA5*?;f))IzU=E5YT#IgMxCN{vABh=|&PXci`eZ(k!K(*iLs@k;lmZskGWRVDl{B+t31r}80LRIT z=A->vHc$)CUQ`^(_P(Sz{+E@%%;hXwBYUlpTq}F%l3#1`x+JdWy0q!Pq_DNo-qbB= zLVJy*nR~O)&^s2*3Oa(OXY{2uaDAuD;%=k&@a%IYn`KLs4NKBSC8{{4<-ST63 zXP2@u#ZOy4Wyxk)z?D<>xFkKZ+4L8Gg7#4}JNs*HQiyK-$@;du<>hQFZGCs{m>Gm^ zKf_ji;w7E}@C@9R_J8^(+D$IqHSQN(wqlzxN&TO0=-OrgSW`6pOptjq8C4P(_AY2-jP5Z1=|e0_gHq?6qo>`O;mZ^D6d_1wgTs zxFt)yTPu<)V6D18>zyHb>}4i2%&Ag2l|@sdazCVH(?J;r%f5GoSm}LoP@eo$6KSpP zW{fqfnJlbmiVW{InHy@k8O28`08c=rk&drZRu>Bb!{MF2a0}-m7D5gH3u5-#4pvYc zz&Yn_0NPsCdqQHP0hlW=F_HAKLj`HUg_X07CLduNZJy4V_a{}kH0JRPpjuyTIU|%Z z@#40jr>1GFuund}IOV9FCF^OlLQCNPP*H+dm2n7hCq7EsC~M1V)DneW{vsDJmhq0b2Y@33eE;^h;Mq+Zv5nKSWcFGDe zjPFJEZg&_geEoQ-)ibe56y=<4SDuEVAvL5uhZJh8lrqP5Ru1d3sUgV z#3B#K?4gW&(2|yR@mkU=uro<9R>8-d^zgLPh;FJ`Z2?j{i&@1H(&LB^{W2D5Je2jW zRTnVI$Dr9kny^uY*&+;zuZcOi`R1DBy>Jrh#AXL_Y3T%OQAU7|4zt;<+kf^lA?~7; zc*<@yWYLd^5s28xdp`6MhSlF^jH(7k`QUkGL90W(q!;6BnUFsYn46WHg`QrTA~Vo!Ivx3(~@w`~H^$`^0S?M_qCm8p-IZvxPpB{FITQ6c~i{*o>= z&j&hr3$_pNe34~7cqpHzP1t~$CCs|~^iCI;5(AQQu0wkyejMK<)1tA#Q0)p%OWteOS+)s5o-(mOX+lPP zb#E|NV#Vg5ig(}Tc#GJ_nw?}RAkD1w7!Y66ojYT=tm)wgQ>(-B?4#HYpEygS_`BP6 zS@p@VAiH>>@avZ%VM819*x1Enn&@JrjTy z58Z#>Wgq87GskkxGsuc6)`_mT%VyV<6x;nn^@syDwd!srxfW<-;+9J?GO6+RgC`@f zmf9fOeITb}xx}-UA@Vp0yCnSql&n{=iGo%fW~3dR9L=e9Tn_MLfD{h1;?DD<@!5a_ zQ+>A47SGdwG^42+l@ejf0i5tJnUT|=$g)iOHmH*@jQmQC)v0s4d$V*<(UA6Vut`PG z6pnZpEHShW@6<`FJ|qftcqj7>Exc+4I({x>ycx;8@UW#m`Cu7F2_WGPyv$Cl4i6hy zO^rDPXPL8BE667$SCjsz$r09e% zhpD?tP$TAX-D(=zRrMq*Za$LrfLgaX8rr6c09axBYJL!2HvkAmlZ_|YLbD!#HvA66 z!hp6s+31X-h)g`_LdYZ|k}qSqWG?<%U-5gUtBc9=NY z1VSnja|Sdsu%$@2Y9g=e6fk_s=b|8rsv{W!lq%&zJKj)jZ2{=v%Pt7Yv6O^t9&*Iu zAM4|-9uS|Bw%C+uDj=vn*svuhqHPUo6S$@UD~&YQ9jZyDaMjE*@Nn_dVNmFZuB74X zhfQjN+ui+{ZK~6P@rmu`i?w=sfEto*{YzOJ^;t!ILx34VF2Ed53{_G-5^rAQ3)D9l zAth;{=9J|O(27~Un)S6diYD64(7ptyqMdDl&=3j&qu?RTSTShdhw8cpgJdCP|G7j> zN8Ixn2%c+ccFk!l7`3m%TCXrJ-iN+09_l?{PevNq7PWdQ8|ANRjv8iE0vQz--|0QN zA38$Hp`Oxo>=E6mRkkc$4yAS8E@f9f8?k8*u<-k(gm;sNqFP*D zw)%hj16Du)EN$%_l4qsq?ERBX^&o01J73Hqe!(D{o0KYL+m|L6GA%>XjB{}C2&{}Z z37NZ*VvZ$@)IL-ln&0J{2*kD3p;TTyvg%km;fYN8>KaIaI=1c7Y>>o7B+cFA9|xYK ztF666V*W29P?&?ro>n$mkxmaL{egkDdm{g;kB?(c^vQlqlblbRWMr@u0H0=+QcE>F z+nk)vv|pS4wLcIdz|@Xg7BA3F$d2l#PB$HcKRPc?tplsZ0DUDM!6sAZeJFhrzLIlg zdt}=eNB9XjsW85{B}gPc=NW8~k|ODVY^%-(EN1MG@at_1T{GHViI>T@h2b`n0cq*A zUu@&y)X1t(+F5-Ic~Fjp7%hOQ9MeLB>{e-IZ0j)0HJIx5rLhj1CUzrmuTz~^NtH}j zfWVX_1xc1?gyZv$Il#kF ztkReq(at0~sB!86kMG68B{8mY?TLe|uS*=x_ei>lBRAYdoIwyCXb~szn`JqxLOD~Z zy6QTefy1iP9a<;zvPe&_pVrp-m49XKk-Ma+%bE-6{2DYdx%JPA^`!M2X+7h{NIA}%@Yq+Vq)@L+Tfa|e|CXtAG03LPg~X}X-#u2LoCR@8R4 zw2Cu-LU*JG62mISmC`E7mn`Dhz$*F+z5baEW7wRI6r}Z!V~Tc1J!}68YcG4k_H~h8 zSL*3;2t7Y5a}HA}lM`FSR+SUX&1o0#vJ1kfkQ7T$$J*FwN>w<~b;Y4c`PFl5oa3yb z*5&sEcFIqFQ_~mXhOUMpgAhq8*07RavP>sai4K=J93-#OE7*E~BmIl=ScfibhpaXUltJAASLc`Vf5ME{796h|Bxjs$3Cg zz9f%rPZN%x?=t8-!3VRvK{t~Po6Ml!-B#CRKt zhucV8G9;=zMOw5U>Pa52BYEe1Y8rF$qc33EL6%P|Mza#B04975iRzqNL#Wlc4RUmg z9Af>F_tyt?8(WQ8LVq?vHo`xpcewCJ{&0lQy?qr4Y)JcAw3F+uaKI)B7Sm=$A6T9~e&|YejGZb!^E=L5 zLs34~N4!}QTrpeVaAGjbN%-wHjd0P=NIC2K)IXeWlNW7%0pY~UF764uHaJ2Fk z)(xWb82W_5!v~!oZMSdXBZsoHQ<#xx!UMM`pO3DU z)BXo^lLW?ZNM@}&_^OEk@pC2qNWnq&fG6qcLW!D|MZex>8uTYv`eIZl^^_~Uaka01 z0Hx3UyBw!f!Tr9C=9s#_?wY5)@3Z&PBEj65*6QD*dU9g3Y_N)~5&8 zTF?6EQMCi%s-9=#F!rC<=N0mWzpF1i7V;H7`IZ3Sm;F~U8mxkay{lfU88g82?ES9Y5|~-2RRe+(S|IrIgVdURn?%W zWFFm$L?nDDw!}mJaDybvDv)(8xcHkw9_Kd0Y?|dfZ=d{Z*-oAoQJ97saDgnx*{m&k z2iYGH&?sG##aLJjXS1$#Lo6!>DaA}OqHgRtG?&H4J+|QBMF8tr5LRKn0EGDR^8`G9 z>&qhX8#+(v_X|Wf?y~8eMmSh1=GDw{Gb>E?+W7U$4PtzWr&K(dDmj@1z{c)J{x1UZ ztfcz(2_sNYhM*q7t^DVYBn)Tf5k}MlBC!A4JDA{PjhzxmleJ>T;pt|iwj!Tc5!viq z7~_Qb`bdcaJ!ArUAx{XmyQNc&&@*-o9~B~Waf~XO;;=46c3Wv9ktIkzAlAo4Cx-SS z+$jQm@dvcT`K8{1PD?KHLYOb$%mh?d9iE$ET z9@qXO($%RN=8db_alEDa(UWOu;q6j+MVofaie-R&ZKS;_O)?yqDJs*)6M3oLc8;w4 z+b@3dAARi)zx>H%#4Ny26EEvo))Y1*&jfXH4cgn0PRdMSZh!ap{jaX>f;24OaOMXC zHUj9}b#w@|?IIIv-Awf9QAE;7&It^m^S3J;%RYBWz_P8$o;t@X>LDfpNX^XijXqxQ zSRycL=&sRDmcfe$hNJv?r%2vYQl&JzYFK1>3_ssGt0csBtbf5J>=j3Ma(FttJfqXq zPxJP~yN6&aB$B;I^kJ17UpX~bs=K%4oV&U>Z1}%hy3wBSx+J|6LK^bJ^O~~d-p*A0R!6hG* zAj(A`wjv>9@RT(704F3ZBCQli^>P6J_yw1Jnz@N2XQ&NDacXySa#g-L+n6cOfG_nx z3wODMjxlcGkl4bqQx=?LX(QPh9=5x=4dUKbEU7Piuv7RRnOC65oyi4L4~uIS5e)x@ zrdvpnbSuX=`FolZ1BY27KLd#*@r9o&6h1|T+WPqjJFiEBFrvVeK=#Ggf|ycuNN%c1 z6}fu=vf;$+4!=`bGa0*+hPZc-vCBhsC679?%)AX)Ov06x6S$Unv1J(1NkGlTXpkUb zd}M@Mg0J0-Y$$Asea-zk^{B-3!^1eod{wnVn(A z0R<{WPyKH(+f(DnEJrB?Dowt7yICWgnIak1{$zkS7zpdb_}OPFSMIXs1>f9`w?(1Y z)MGVt3R0(rX*)-U$u!}V5HOY~3}(x`6#bLc7<^YRE5tIxV~KBhR7H*uP-B*%A&iu1 zVW(l+&}`)jATH&u8JZm$B8yehCVMl3Q=OEx0Y%$J67rMB63?wI%d<6q*xfVGb}1F( z|1lz%&1o#}E`)G(Q`y!NM58EkDf^i(ed=P4r@r)Rs74q| zDAug^D0m_6=@FT1t;9}d=*hNDV{x>!D)tiddYxx%+^M7FrcSU0^)-(p0`L<`8oAb- zw{rM`JgzOo2w5#o3f&$rO}M4wm>k3`lw4`IpT(yVZQu~O@&OyUJ%>es9p;Z@twV~b z?rR7LtHAn|y@IKLbs4Cy0`6U1`u(vkc`Crt_1J=xIHZs(|D`B72IvF`WEzxugp$tf z1U9G9`Mdr>b1{VRsVi~M;-&(H{9<*eUpn>Sw>F>JVkTqo-d>OAAcuePdy-W1yK=$a z@BXxR;?R*UF<0hgxeEcKx{colK)I(y=*-g&KnZ>sE`JV9&KB1-A~Re`hQJN2A#9O9 zb@)@AGR>R$Q#(FTGlBS@ggkT|@aq)dJ?sx{5MBwm%0eLNp6{WeEBR2}Rdl5dZ+u{^ zrHwMk%;>tp7apwMRtJifW^H#36Eig@h^YxUoTm+=WlNW6k8PEvve(2fs%lNjl)=n%Bfs17?e1 ztUYnq;dGdD8cWc{>F&r0PRr)O=^v~X^WPxx+}7{Es58%QMz-1tsEwnCV5|kyIm;sj z)VN6kYGXOxm?@JSeqlkA7PuoX&NwKgNG1?AYYt6_`Qn11BN=eQCL3CMt4KEVFD!e~ZE2Ke+tZ9o#Z^~JV%nq0X)F_liVu2Uy zE*?{dWV`b+1R#1VSZ@dof+HuK=Dl&{(PLcK>5T;v>kZPa?G46W)ti}D_E#w81`Syy z&I(g71wyiLzF+@C&R}JuvUOsB>%N(1VhCZ(FHrqZAO>LX(VTpKtr}?7%k3c%-8wyB z-YKa09tOL5*lIm&m2z?(vy_uxE9%xx9V!DeIW-A$34Oy3kJ&VgVlx46MMsLC5Eo&- zbxL-?WnWmuQrb`;8J&2Pl8**#LK2D?3~#K1dPTFd$OML4hw9iS%?Fg>u|URxCod03 zOd4r!CXV9)=*z+x)R5Y&4R~uUT=7$&VG}IaFQL#Zjkh(kw1`<6=*rBs2UuqjxN4Vc zVW1D1v;EK?Ffu;BlKTR!8F$lfEig`ZFu-wgHn1UbD5{rDzLagsnO5XeD++pK-?Rh` zPP$5+B;D7_he`#AXH`V~)Ei!NW}?1@l&E#`!=BJY;4QEC-w6F}650Y3ITQ$8NIoW& zP&lJ)P#=T0`gX{+p|nK#&N?Cq-^@0>9Nz`QsczQbP|bW7M&wQ`tNAW69T@yl{J$2I z{0sQ!j?~Gj5Z$Js1wP0{xlUMI)uEx3YQ|*KAHrTeOyZH`cdI2P1=Zt2yD0PsQmX|< zNjMV&$!uCeloqD5OkM%eI<9I0?aqk zgg}~xBjv-I>8DKTjz;w4@ifrF1hA}KnRzHPBH@&Vz84EP@o$EN&9W`axz0fIDI(o4 zP#KNL`v+XfwLN=yIl3_XI%CW;6MQI$rV%k-tI@S%EZEPrTB`8d8>4I=@qs5I^m9i< zN9F}e0)#d2_3}h?>*GY*G{q!rG5owt(Go6qXD*@5}B&=N#rW2a`TF9BqHx+ukgaJdY zN)?GuE%4(~4#jrM+BwS%EBX6KKYbE+@xi=Db`Y6i+NC3n$u#H#-zNoi$&X+QRgc+{ zTBSWx$qqT3yim$u4Y(ss3-BH&S`>dMvM5o>Y6{m(1kTF|pfV)j6oL7P!0+XSq6$qQ z*H%rN@`<^&R8txezJiN9na_pk_Z4*_zJg(!Z;4*5Ex{aTrz95%lca^K-Zzw9?!enGmyuM4i-(qCb{lysrBv z64edATaI8L2Q1MOy8k!fy;^u*@p6Q>E@3lMji5nmIV}5Tu1l$<_{YLE2?<<l>A zx(JoJz_VEDLJvkD2{K6LaI{{7DzKlA;1kIvyWmB&mE6QQVXSi;+Oa*6Mts;WOS~wstn#OiKobj%6ha(5CRL zO^FM#lF9`$^d5Ghuor%wrx3KOjXrUSQ<*l<6EJ>`T&$0tyfq)ZioPYJpXO$= zl)Ay(l6Gq*X=abcB0(hAwQvRREM23I-wDxrW7KTQ{Y(Uk8XnG@8acwFR=Th zSdo5mi&^>~L>#kdzZ9Bhk|a2g7Lu!)hzv6kJr;oFy!@P7@@v1Rj;n;TX0Np+5t;c< zvggJ_TT9;MWUVpB(nVq+f}oTGh21=^R!?3Oo6kQ5+#-7jP##tTxyyMMUY-h^C1QfE zS*@1GSEpl3hdmK+;uHfW#KReK8;=mc-ie4Bsh0rUizR`zAX{6xXnQucnf zCEI@%*nzuiWgAjL38cH7xZ+s?lj|aRatot1I<$Rm-u7U~Xccq0#sEpK`H);we&cd& zHY}Ix{P03s6(zZ7(k!{o>#UV)DtF7ZyRKaSvOGneW9;a&jXBq7RA8r4#symwxDn+x zT}E1405Y6}#WG#X0$Q`D0$C=@P)CGR$fVx|W2V85Q}xO!d8V#c`kl%D5Fm##6IM#F zfkJZ=B_@bObgL3tBtQXgT6n#C9eg0wxz=i%KgwRGRzgIElj@cxm}2e^=d4JAX6P`R z@d1V4!pPrdjzYfM+puDbaCW1|z(jq$WOn~qLT&yI~Y8|mEWqm4!9P|KdE zRCLZA=B2;G_U_!RuU>R-ooybUn0YjvTY6-AV)1xm+yU@cT>NCHIOT@^=qV}`wNEZ@ z=C{iw8#Znl+`Q$ct=qN_?Rdp2Z%%9VSNRGrb^nHX|C_JuvU3+){GJqQRe1IC>f;sj zs`B~}FMPTSc;SZ}F3}M-Iee5W@gkCP6<)o(cJjK7*HK=M&2;vs-)kbD%6EwFz8*PN z7M(lybl?t;47inl_;KgH`e#M3pjve+Rrd3m=jHUH*>QD>A^+B2Ub*tI&+?OuI_fn3 zLNGi$ynA@h@ZRBl!~2I13?CdmGEw_wL!dckjNv`}ZE$dvNcey?5*z-nV<-o_%}w?c2A1 z-+_Gx_Z`}I$Nu5{yZ7(ezjyz>{rmSH*ne>Uq5XFp7(TH3z@7tp59~Xz|G*umf5SojC3(k(MC`w!f@<>6O3H^{9wzV=-Y9ocnAwXXfsGuQFf zCRNw{(*uU_dmgIQw~=g$HoSR9H7Ufe*!artK0Y%$H~&alEDfL$+sj*ecHVN+)=F>x z(9O5L>VqG6;=5k`nmga{-`(}bd+vYBd*1uL@B98A`0tNCJUTYsIQHno7V8bUJ1H}J z+e{Z4a|?~d#!Peck?DrCJ{?V)&Bxt?jfE2vi;EMpGwFC^W&-_x+gxK|w5bBMHP6I;gK%RjgB?a>Bg~UYB-#nU=(j#;_6s>qH#j-8OH3=0<|7l zT72ByH`8QvOLO3qEzScH9JEi8=(j7-qaeUQ3J@(?2WTCGYnlM5>1=106H}m zx=7cYI01UIOU<>Fk1Z`i$$MuPo9V=2IzIc@O#5PKW}z`UcHA({#>WA?n8HXJxRHg~ zsm4q?Hvz@&d)?b|=C=5ej{&-l`etqf2QURYZEOoj24m>(&zI*S&|%_zi%3N0@}b75 zrYD3E{q9qZG4bxiXfyA=MNPQ6bOO0@S^foTGCbq>?98L?osXY*WOjOD%!r&$&(6-7 ze9VlV$i+gH9zs4&jLtls&dxDV(ccE2^MH{WOU4_EV+#{=$Q-z~**Y>he)Vn1)!6Zg z>G8BBlB(1qkBMhnB0GIc3gi$$a%`%*hqnBUBO*rI$7YuRK07wHwBX*^l(Y3ZHa2?_ z<&lOxxB&TQmljia+>o-Y(X6pibQOQ?jx5c1K~npP6LZrI(b9A41R{V7mm)B7@z5*H zeJE;k;@AXW7f_Is)P9rawSbJKi%lfyga%@Y)}d5hOR_1+%$3YnUz8}DmMyvmAVhk6 zbTNIT0XG&KjEO!TT$pXnj?GTz{1-)Y486j4xFflJgxiavOfG1P(6rG6(Z%M(j5jYi zKj=7@(6{6R?(nAF=VJ?qi4ET@NDi%i+Xmj2h&9EFOLJHQk@TI0f(N1oojY$`bJcEs zZ1jX|$lAIs-CXac8G@Wlu2pd@Zf|WgADdm6YO8=@$UZE1($`G_$!W8P$Ss35F5g>M|4u~9Z=P?KZW*VI!uT|BK-4R8AmJLfxMzD9V7PkrCEGIG!!U!eaB^nZc=FVO!5`oBQ`U!?yF^nZbf@|OZvrG1t5RoYj7 z*XgmhICcc|p5Tm<=h))Doij_*)3-U)2Z}j8BNgws_10S#TmN4D+E?E?cI%jH83%|% zmGD*6<)N#1*K3^Yf>--=6yJ_hlWmzzA6uBPd4uftaa=eYa2Zc;#jo1sUb}nG-hKN= zAF+7kxB`+Bie`vPmQFr)>hbqelZU_JZK?D85zd-e0FiKlV8rQhvH14k>-Me|Oj_1- zeu*YqA8>B+fd?LNlNN1F+6QlwPLHa_7o%;i*KOMde+a&|sN3Z>x1pE&QE!h6 zhAJ`4td!tF>|oEjf3T(X*Q=0UaewG4Dy}%*yY9E@<)4}uwEhqz=j@F~AN9w3_kAmC zzafI{KSaV6^yVtQnD_4c?TYpVp>6*m#Ln4U#TWC=f8^F}->&!d|JK|`P+#&-A#&>v z1#!;aD!!Qa?)$BJ{u7mI|Dj^e*;~aI^R6%13%}>$!u;*}%3gx_mH129r-_*{gc|QB z(d?EiHlM|>oqc4Is7w+>kR@?GF*`nS3`4bOCLGIjf@pZ+=)}x;;}nqDoEGoRA?H;+ z2#Y-|P?qlVM66|VVRWoXOSVqLd#~xPCFw`$`g(n`Ln5x{eG1Vv0};+B3lSeN zJ{5Oe(}mr=E^rpv;ypQVU-M;-=cAWgyg#=DLc&{)H1EE7oS^S9!U4~MtEl)n+P0_& ZcN7b~%wGQ!Eenql6p+o03oFCU{eN9&T9W_( literal 0 HcmV?d00001 From be0117227547d956751d3210b9344ba93e8d0e1f Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 12 Apr 2021 08:51:18 -0500 Subject: [PATCH 097/157] Fixed incorrect use of nonlocal. --- tests/nodeos_high_transaction_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nodeos_high_transaction_test.py b/tests/nodeos_high_transaction_test.py index 1afaff11675..6108af9ffa0 100755 --- a/tests/nodeos_high_transaction_test.py +++ b/tests/nodeos_high_transaction_test.py @@ -162,7 +162,7 @@ lastIrreversibleBlockNum = None def cacheTransIdInBlock(transId, transToBlock, node): - nonlocal lastIrreversibleBlockNum + global lastIrreversibleBlockNum lastPassLIB = None blockWaitTimeout = 60 transTimeDelayed = False From 8fb7c8d2d8ee3b17bc6c059810c1ad78e8a22dd5 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 12 Apr 2021 08:52:01 -0500 Subject: [PATCH 098/157] Cleanup of debug log statement. --- plugins/net_plugin/net_plugin.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 610ee7380ac..b017e992859 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -3679,7 +3679,6 @@ namespace eosio { for(auto& connection : connections) { const auto& participant = connection->participant_name(); if(participant && security_group.is_in_security_group(participant.value())) { - ilog("REM update_security_group participant in: ${part}, participating: ${p}",("part",participant->to_string())("p",connection->is_participating())); if(!connection->is_participating()) { connection->set_participating(true); connection->send_handshake(); From 876464375434775dc3572df5e33b3c3eead4593f Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 12 Apr 2021 08:53:04 -0500 Subject: [PATCH 099/157] Made simple test to verify security group functionality. --- tests/CMakeLists.txt | 2 +- tests/privacy_simple_network.py | 94 +++++++++++++++++++++++++-------- 2 files changed, 73 insertions(+), 23 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7f13b5304d7..e7fc99911c4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -119,7 +119,7 @@ add_test(NAME eosio_blocklog_prune_test COMMAND tests/eosio_blocklog_prune_test. set_property(TEST eosio_blocklog_prune_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME privacy_startup_network COMMAND tests/privacy_startup_network.py -p 1 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST privacy_startup_network PROPERTY LABELS nonparallelizable_tests) -add_test(NAME privacy_simple_network COMMAND tests/privacy_simple_network.py -p 1 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME privacy_simple_network COMMAND tests/privacy_simple_network.py -p 2 -n 3 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST privacy_simple_network PROPERTY LABELS nonparallelizable_tests) # Long running tests diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 5f2b511d1ea..3f347136a95 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -35,7 +35,7 @@ port=args.port pnodes=args.p apiNodes=0 # minimum number of apiNodes that will be used in this test -# bios is also treated as an API node +# bios is also treated as an API node, but doesn't count against totalnodes count minTotalNodes=pnodes+apiNodes totalNodes=args.n if args.n >= minTotalNodes else minTotalNodes if totalNodes > minTotalNodes: @@ -201,27 +201,77 @@ def publishContract(account, file, waitForTransBlock=False): publishContract(cluster.eosioAccount, 'eosio.secgrp', waitForTransBlock=True) - addTrans = [] - action = None - for producerNum in range(pnodes): - if action is None: - action = '[[' - else: - action += ',' - action += '"{}"'.format(Node.participantName(producerNum)) - action += ']]' - - addTrans.append(producers[0].pushMessage(cluster.eosioAccount.name, "add", action, "--permission eosio@active")) - addTrans.append(producers[0].pushMessage(cluster.eosioAccount.name, "publish", "[0]", "--permission eosio@active")) - - highestBlock = None - for trans in addTrans: - Utils.Print("trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) - - for i in range(10): - Utils.Print("\n\n\n\n\nNext Round of Info:") - reportInfo() - time.sleep(5) + participants = [x for x in producers] + nonParticipants = [x for x in apiNodes] + + def security_group(nodeNums): + action = None + for nodeNum in nodeNums: + if action is None: + action = '[[' + else: + action += ',' + action += '"{}"'.format(Node.participantName(nodeNum)) + action += ']]' + + Utils.Print("adding {} to the security group".format(action)) + trans = producers[0].pushMessage(cluster.eosioAccount.name, "add", action, "--permission eosio@active") + Utils.Print("add trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) + trans = producers[0].pushMessage(cluster.eosioAccount.name, "publish", "[0]", "--permission eosio@active") + Utils.Print("publish action trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) + return trans + + def verifyParticipantsTransactionFinalized(transId): + Utils.Print("Verify participants are in sync") + for part in participants: + if part.waitForTransFinalization(transId) == None: + Utils.exitError("Transaction: {}, never finalized".format(trans)) + + def verifyNonParticipants(transId): + Utils.Print("Verify non-participants don't receive blocks") + publishBlock = producers[0].getBlockIdByTransId(transId) + prodLib = producers[0].getBlockNum(blockType=BlockType.lib) + waitForLib = prodLib + 3 * 12 + if producers[0].waitForBlock(waitForLib, blockType=BlockType.lib) == None: + Utils.exitError("Producer did not advance lib the expected amount. Starting lib: {}, exp lib: {}, actual state: {}".format(prodLib, waitForLib, producers[0].getInfo())) + producerHead = producers[0].getBlockNum() + + for nonParticipant in nonParticipants: + nonParticipantPostLIB = nonParticipant.getBlockNum(blockType=BlockType.lib) + assert nonParticipantPostLIB < publishBlock, "Participants not in security group should not have advanced LIB to {}, but it has advanced to {}".format(publishBlock, nonParticipantPostLIB) + nonParticipantHead = nonParticipant.getBlockNum() + assert nonParticipantHead < producerHead, "Participants (that are not producers themselves) should not advance head to {}, but it has advanced to {}".format(producerHead, nonParticipantHead) + + Utils.Print("Add all producers to security group") + publishTrans = security_group([x for x in range(pnodes)]) + publishTransId = Node.getTransId(publishTrans[1]) + verifyParticipantsTransactionFinalized(publishTransId) + verifyNonParticipants(publishTransId) + + while len(nonParticipants) > 0: + toAdd = nonParticipants[0] + participants.append(toAdd) + del nonParticipants[0] + Utils.Print("Take a non-participant and make a participant. Now there are {} participants and {} non-participants".format(len(participants), len(nonParticipants))) + + toAddNum = None + num = 0 + for node in cluster.getNodes(): + if node == toAdd: + toAddNum = num + Utils.Print("TEMP FOUND NODE: {}".format(num)) + break + num += 1 + if toAddNum is None: + assert toAdd == cluster.biosNode + toAddNum = totalNodes + Utils.Print("TEMP FOUND NODE checking bios: {}".format(toAddNum)) + Utils.Print("TEMP toAddNum: {}".format(toAddNum)) + publishTrans = security_group([toAddNum]) + publishTransId = Node.getTransId(publishTrans[1]) + verifyParticipantsTransactionFinalized(publishTransId) + verifyNonParticipants(publishTransId) + testSuccessful=True finally: From 8a34e9d2c5bbe4e3c02015d5054cc3995a9b418b Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 12 Apr 2021 08:53:54 -0500 Subject: [PATCH 100/157] Fixed apply_block functionality for bringing pending security group into the active security group. --- libraries/chain/controller.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index fffeb6e3c7d..a9a8e07d3ea 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1759,6 +1759,9 @@ struct controller_impl { if( !use_bsp_cached ) { bsp->set_trxs_metas( std::move( ab._trx_metas ), !skip_auth_checks ); } + + auto& pbsh = ab._pending_block_header_state; + bsp->set_security_group_info(std::move(pbsh.security_group)); // create completed_block with the existing block_state as we just verified it is the same as assembled_block pending->_block_stage = completed_block{ bsp }; From 99abc09e0e533b6e3832f64589da3e947e387707 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 12 Apr 2021 16:51:53 -0500 Subject: [PATCH 101/157] Additional verification for test. --- unittests/security_group_tests.cpp | 39 +++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/unittests/security_group_tests.cpp b/unittests/security_group_tests.cpp index 85afdc4e1a4..5a7c5da45ad 100644 --- a/unittests/security_group_tests.cpp +++ b/unittests/security_group_tests.cpp @@ -203,6 +203,9 @@ BOOST_AUTO_TEST_CASE(test_snapshot_global_property_object) { } BOOST_AUTO_TEST_CASE(test_participants_change) { + // Need to create a 2 chain version of this test, but will require using a transaction, + // instead of calling add_security_group_participants directly, so the second chain can + // process it as part of the block eosio::testing::tester chain; using namespace eosio::chain::literals; @@ -220,22 +223,56 @@ BOOST_AUTO_TEST_CASE(test_participants_change) { BOOST_TEST(chain.control->proposed_security_group_participants() == new_participants); BOOST_CHECK_EQUAL(chain.control->active_security_group().participants.size() , 0); + BOOST_CHECK(!chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + + { + const auto& cur_security_group = chain.control->active_security_group(); + BOOST_REQUIRE_EQUAL(cur_security_group.version, 0); + } chain.produce_block(); - BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 0); + { + const auto& cur_security_group = chain.control->active_security_group(); + BOOST_REQUIRE_EQUAL(cur_security_group.version, 1); + } + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size(), 0); BOOST_TEST(chain.control->active_security_group().participants == new_participants); BOOST_CHECK(chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); BOOST_CHECK(!chain.control->in_active_security_group(participants_t{"bob"_n, "charlie"_n})); chain.control->remove_security_group_participants({"alice"_n}); BOOST_TEST(chain.control->proposed_security_group_participants() == participants_t{"bob"_n}); + BOOST_CHECK(chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + + { + const auto& cur_security_group = chain.control->active_security_group(); + BOOST_REQUIRE_EQUAL(cur_security_group.version, 1); + } chain.produce_block(); BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 0); BOOST_TEST(chain.control->active_security_group().participants == participants_t{"bob"_n}); BOOST_CHECK(chain.control->in_active_security_group(participants_t{"bob"_n})); + BOOST_CHECK(!chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + + { + const auto& cur_security_group = chain.control->active_security_group(); + BOOST_REQUIRE_EQUAL(cur_security_group.version, 2); + } + + chain.produce_block(); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 0); + BOOST_TEST(chain.control->active_security_group().participants == participants_t{"bob"_n}); + BOOST_CHECK(chain.control->in_active_security_group(participants_t{"bob"_n})); + + { + const auto& cur_security_group = chain.control->active_security_group(); + BOOST_REQUIRE_EQUAL(cur_security_group.version, 2); + } } From 0f777393f76d83e723bd7449a65314c4933b03ad Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 13 Apr 2021 12:50:11 -0500 Subject: [PATCH 102/157] Made setters for proposed_security_group_participants and transaction_hooks and other cleanup. --- libraries/chain/controller.cpp | 10 +++----- .../eosio/chain/global_property_object.hpp | 23 +++++++++++++------ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index a9a8e07d3ea..9a4126ca3de 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1444,9 +1444,7 @@ struct controller_impl { ++bb._pending_block_header_state.security_group.version; bb._pending_block_header_state.security_group.participants = { gpo.proposed_security_group_participants.begin(), - gpo.proposed_security_group_participants.end(), - bb._pending_block_header_state.security_group.participants.key_comp(), - bb._pending_block_header_state.security_group.participants.get_allocator()}; + gpo.proposed_security_group_participants.end()}; db.modify(gpo, [&](auto& gp) { gp.proposed_security_group_block_num = 0; @@ -2282,10 +2280,8 @@ struct controller_impl { db.modify(gpo, [&proposed_participants, cur_block_num](auto& gp) { gp.proposed_security_group_block_num = cur_block_num; - gp.proposed_security_group_participants = {proposed_participants.begin(), - proposed_participants.end(), - gp.proposed_security_group_participants.key_comp(), - gp.proposed_security_group_participants.get_allocator()}; + gp.set_proposed_security_group_participants(proposed_participants.begin(), + proposed_participants.end()); }); return 0; diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index 062ddca05e9..e550a35688c 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -73,6 +73,7 @@ namespace eosio { namespace chain { kv_database_config kv_configuration; wasm_config wasm_configuration; block_num_type proposed_security_group_block_num = 0; + // members that are containers need to be shared_* containers, since this object is stored in a multi-index container shared_set proposed_security_group_participants; shared_vector transaction_hooks; @@ -103,6 +104,18 @@ namespace eosio { namespace chain { kv_configuration = legacy.kv_configuration; wasm_configuration = legacy.wasm_configuration; } + + template + void set_proposed_security_group_participants(Iter begin, Iter end) { + proposed_security_group_participants = {begin, end, + proposed_security_group_participants.key_comp(), + proposed_security_group_participants.get_allocator()}; + } + + template + void set_transaction_hooks(Iter begin, Iter end) { + transaction_hooks = {begin, end, transaction_hooks.get_allocator()}; + } }; @@ -163,13 +176,9 @@ namespace eosio { namespace chain { std::visit( [&gpo](auto& ext) { gpo.proposed_security_group_block_num = ext.proposed_security_group_block_num; - gpo.proposed_security_group_participants = {ext.proposed_security_group_participants.begin(), - ext.proposed_security_group_participants.end(), - gpo.proposed_security_group_participants.key_comp(), - gpo.proposed_security_group_participants.get_allocator()}; - gpo.transaction_hooks = {ext.transaction_hooks.begin(), - ext.transaction_hooks.end(), - gpo.transaction_hooks.get_allocator()}; + gpo.set_proposed_security_group_participants(ext.proposed_security_group_participants.begin(), + ext.proposed_security_group_participants.end()); + gpo.set_transaction_hooks(ext.transaction_hooks.begin(), ext.transaction_hooks.end()); }, extension); } From 85ac2aae5f52c68b6b1d764b997bb020f3c496f8 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 13 Apr 2021 12:53:01 -0500 Subject: [PATCH 103/157] Moved common methods to Cluster and more cleanup. --- tests/Cluster.py | 55 +++++++++++++++++ tests/Node.py | 6 +- tests/privacy_simple_network.py | 105 +++----------------------------- 3 files changed, 70 insertions(+), 96 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index b3a81794639..fb879ab2688 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -1822,3 +1822,58 @@ def stripValues(lowestMaxes,greaterThan): while len(lowestMaxes)>0 and compareCommon(blockLogs, blockNameExtensions, first, lowestMaxes[0]): first=lowestMaxes[0]+1 lowestMaxes=stripValues(lowestMaxes,lowestMaxes[0]) + + def getAllNodes(self): + nodes = [] + nodes.extend(self.getNodes()) + if self.biosNode is not None: + nodes.append(self.biosNode) + return nodes + + def reportInfo(self): + Utils.Print("\n\n\n*****************************") + Utils.Print("All Nodes current info:") + for node in self.getAllNodes(): + Utils.Print("Info: {}".format(json.dumps(node.getInfo(), indent=4, sort_keys=True))) + Utils.Print("\n*****************************") + + def verifyInSync(self, sourceNodeNum=0, specificNodes=None): + assert isinstance(sourceNodeNum, int) + desc = "provided " if specificNodes else "" + + if specificNodes is None: + specificNodes = self.getAllNodes() + assert sourceNodeNum < len(specificNodes) + Utils.Print("Ensure all {}nodes are in-sync".format(desc)) + source = specificNodes[sourceNodeNum] + lib = source.getInfo()["last_irreversible_block_num"] + headBlockNum = source.getBlockNum() + headBlock = source.getBlock(headBlockNum) + Utils.Print("headBlock: {}".format(json.dumps(headBlock, indent=4, sort_keys=True))) + headBlockId = headBlock["id"] + error = None + for node in specificNodes: + if node.waitForBlock(headBlockNum, reportInterval = 1) is None: + error = "Node failed to get block number {}. Current node info: {}".format(headBlockNum, json.dumps(node.getInfo(), indent=4, sort_keys=True)) + break + + if node.waitForNextBlock() is None: + error = "Node failed to advance head. Current node info: {}".format(json.dumps(node.getInfo(), indent=4, sort_keys=True)) + break + + if node.getBlock(headBlockId) is None: + error = "Producer node has block number: {}, but it is not id: {}. Block: {}".format(headBlockNum, headBlockId, json.dumps(node.getBlock(headBlockNum), indent=4, sort_keys=True)) + break + + if node.waitForBlock(lib, blockType=BlockType.lib) is None: + error = "Producer node is failing to advance its lib ({}) with producer {} ({})".format(node.getInfo()["last_irreversible_block_num"], producerNum, lib) + break + + Utils.Print("Ensure all nodes are advancing lib") + if node.waitForBlock(lib + 1, blockType=BlockType.lib, reportInterval = 1) == None: + error = "Producer node failed to advance lib ahead one block to: {}".format(lib + 1) + break + + if error: + self.reportInfo() + Utils.errorExit(error) diff --git a/tests/Node.py b/tests/Node.py index 17a53ee2ac2..61c69991f36 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -230,6 +230,10 @@ def getBlock(self, blockNumOrId, silentErrors=False, exitOnError=False): msg="(block %s=%s)" % (numOrId, blockNumOrId) return self.processCleosCmd(cmd, cmdDesc, silentErrors=silentErrors, exitOnError=exitOnError, exitMsg=msg) + def getHeadOrLib(self, blockType=BlockType.head, silentErrors=False, exitOnError=False): + blockNum = self.getBlockNum(blockType=blockType) + return self.getBlock(blockNum, silentErrors=silentErrors, exitOnError=exitOnError) + def isBlockPresent(self, blockNum, blockType=BlockType.head): """Does node have head_block_num/last_irreversible_block_num >= blockNum""" assert isinstance(blockNum, int) @@ -491,7 +495,7 @@ def waitForTransFinalization(self, transId, timeout=None): def waitForNextBlock(self, timeout=WaitSpec.default(), blockType=BlockType.head): num=self.getBlockNum(blockType=blockType) if isinstance(timeout, WaitSpec): - timeout = timeout.seconds(num, num+1) + timeout.convert(num, num+1) lam = lambda: self.getHeadBlockNum() > num ret=Utils.waitForTruth(lam, timeout) return ret diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 3f347136a95..aaf436bbb09 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -31,7 +31,7 @@ from core_symbol import CORE_SYMBOL args = TestHelper.parse_args({"--port","-p","-n","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run" - ,"--sanity-test","--wallet-port"}) + ,"--wallet-port"}) port=args.port pnodes=args.p apiNodes=0 # minimum number of apiNodes that will be used in this test @@ -47,7 +47,6 @@ dontKill=args.leave_running onlyBios=False killAll=args.clean_run -sanityTest=args.sanity_test walletPort=args.wallet_port cluster=Cluster(host=TestHelper.LOCAL_HOST, port=port, walletd=True) @@ -55,7 +54,6 @@ testSuccessful=False killEosInstances=not dontKill killWallet=not dontKill -dontBootstrap=sanityTest # intent is to limit the scope of the sanity test to just verifying that nodes can be started WalletdName=Utils.EosWalletName ClientName="cleos" @@ -72,108 +70,29 @@ cluster.cleanup() Print("Stand up cluster") - if cluster.launch(pnodes=pnodes, totalNodes=totalNodes, prodCount=1, onlyBios=False, dontBootstrap=dontBootstrap, configSecurityGroup=True) is False: + if cluster.launch(pnodes=pnodes, totalNodes=totalNodes, prodCount=1, onlyBios=False, configSecurityGroup=True) is False: cmdError("launcher") errorExit("Failed to stand up eos cluster.") Print("Validating system accounts after bootstrap") cluster.validateAccounts(None) - def getNodes(): - global cluster - nodes = [cluster.biosNode] - nodes.extend(cluster.getNodes()) - return nodes - -# cluster.biosNode.kill(signal.SIGTERM) - - def reportInfo(): - Utils.Print("\n\n\n\n\nInfo:") - for node in getNodes(): - Utils.Print("Info: {}".format(json.dumps(node.getInfo(), indent=4, sort_keys=True))) - Utils.Print("\n\n\n\n\nCheck after KILL:") Utils.Print("\n\n\n\n\nNext Round of Info:") - reportInfo() + cluster.reportInfo() producers = [cluster.getNode(x) for x in range(pnodes) ] apiNodes = [cluster.getNode(x) for x in range(pnodes, totalNodes)] apiNodes.append(cluster.biosNode) - def createAccount(newAcc): - producers[0].createInitializeAccount(newAcc, cluster.eosioAccount) - ignWallet = cluster.walletMgr.create("ignition") # will actually just look up the wallet - cluster.walletMgr.importKey(newAcc, ignWallet) - - # numAccounts = 4 - # testAccounts = Cluster.createAccountKeys(numAccounts) - # accountPrefix = "testaccount" - # for i in range(numAccounts): - # testAccount = testAccounts[i] - # testAccount.name = accountPrefix + str(i + 1) - # createAccount(testAccount) - - blockProducer = None - - def verifyInSync(producerNum): - Utils.Print("Ensure all nodes are in-sync") - lib = producers[producerNum].getInfo()["last_irreversible_block_num"] - headBlockNum = producers[producerNum].getBlockNum() - headBlock = producers[producerNum].getBlock(headBlockNum) - global blockProducer - if blockProducer is None: - blockProducer = headBlock["producer"] - Utils.Print("headBlock: {}".format(json.dumps(headBlock, indent=4, sort_keys=True))) - headBlockId = headBlock["id"] - error = None - for prod in producers: - if prod == producers[producerNum]: - continue - - if prod.waitForBlock(headBlockNum, reportInterval = 1) is None: - error = "Producer node failed to get block number {}. Current node info: {}".format(headBlockNum, json.dumps(prod.getInfo(), indent=4, sort_keys=True)) - break - - if prod.getBlock(headBlockId) is None: - error = "Producer node has block number: {}, but it is not id: {}. Block: {}".format(headBlockNum, headBlockId, json.dumps(prod.getBlock(headBlockNum), indent=4, sort_keys=True)) - break - - if prod.waitForBlock(lib, blockType=BlockType.lib) is None: - error = "Producer node is failing to advance its lib ({}) with producer {} ({})".format(prod.getInfo()["last_irreversible_block_num"], producerNum, lib) - break + blockProducer = producers[0].getHeadOrLib()["producer"] - if error: - reportInfo() - Utils.exitError(error) - - for node in apiNodes: - if node.waitForBlock(headBlockNum, reportInterval = 1) == None: - error = "API node failed to get block number {}".format(headBlockNum) - break - - if node.getBlock(headBlockId) is None: - error = "API node has block number: {}, but it is not id: {}. Block: {}".format(headBlockNum, headBlockId, json.dumps(node.getBlock(headBlockNum), indent=4, sort_keys=True)) - break - - if node.waitForBlock(lib, blockType=BlockType.lib) == None: - error = "API node is failing to advance its lib ({}) with producer {} ({})".format(node.getInfo()["last_irreversible_block_num"], producerNum, lib) - break - - Utils.Print("Ensure all nodes are in-sync") - if node.waitForBlock(lib + 1, blockType=BlockType.lib, reportInterval = 1) == None: - error = "Producer node failed to advance lib ahead one block to: {}".format(lib + 1) - break - - if error: - reportInfo() - Utils.exitError(error) - - verifyInSync(producerNum=0) + cluster.verifyInSync() featureDict = producers[0].getSupportedProtocolFeatureDict() Utils.Print("feature dict: {}".format(json.dumps(featureDict, indent=4, sort_keys=True))) - reportInfo() + cluster.reportInfo() Utils.Print("Activating SECURITY_GROUP Feature") #Utils.Print("act feature dict: {}".format(json.dumps(producers[0].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) @@ -186,17 +105,13 @@ def verifyInSync(producerNum): break Utils.Print("SECURITY_GROUP Feature activated") - reportInfo() + cluster.reportInfo() assert producers[0].containsFeatures([feature]), "{} feature was not activated".format(feature) - if sanityTest: - testSuccessful=True - exit(0) - def publishContract(account, file, waitForTransBlock=False): Print("Publish contract") - reportInfo() + cluster.reportInfo() return producers[0].publishContract(account, "unittests/test-contracts/security_group_test/", "{}.wasm".format(file), "{}.abi".format(file), waitForTransBlock=waitForTransBlock) publishContract(cluster.eosioAccount, 'eosio.secgrp', waitForTransBlock=True) @@ -225,7 +140,7 @@ def verifyParticipantsTransactionFinalized(transId): Utils.Print("Verify participants are in sync") for part in participants: if part.waitForTransFinalization(transId) == None: - Utils.exitError("Transaction: {}, never finalized".format(trans)) + Utils.errorExit("Transaction: {}, never finalized".format(trans)) def verifyNonParticipants(transId): Utils.Print("Verify non-participants don't receive blocks") @@ -233,7 +148,7 @@ def verifyNonParticipants(transId): prodLib = producers[0].getBlockNum(blockType=BlockType.lib) waitForLib = prodLib + 3 * 12 if producers[0].waitForBlock(waitForLib, blockType=BlockType.lib) == None: - Utils.exitError("Producer did not advance lib the expected amount. Starting lib: {}, exp lib: {}, actual state: {}".format(prodLib, waitForLib, producers[0].getInfo())) + Utils.errorExit("Producer did not advance lib the expected amount. Starting lib: {}, exp lib: {}, actual state: {}".format(prodLib, waitForLib, producers[0].getInfo())) producerHead = producers[0].getBlockNum() for nonParticipant in nonParticipants: From c825ceb77c051bed601b0611d7973ec3995ff3a9 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 13 Apr 2021 14:53:09 -0500 Subject: [PATCH 104/157] Fixing resume from state. --- pipeline.jsonc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline.jsonc b/pipeline.jsonc index 76fdb578a7c..d79afd29cdf 100644 --- a/pipeline.jsonc +++ b/pipeline.jsonc @@ -52,7 +52,7 @@ "test": [ { - "commit": "fa9581c9c50b8123d3f86055e8b9cafa3ef36f5b" + "commit": "85ac2aae5f52c68b6b1d764b997bb020f3c496f8" } ] } From ba88c843bdf3c4f219f1c35cb498aeb1126bbca7 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 13 Apr 2021 16:09:45 -0500 Subject: [PATCH 105/157] Cleanup. --- tests/privacy_simple_network.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index aaf436bbb09..a1d10c9c1be 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -174,14 +174,11 @@ def verifyNonParticipants(transId): for node in cluster.getNodes(): if node == toAdd: toAddNum = num - Utils.Print("TEMP FOUND NODE: {}".format(num)) break num += 1 if toAddNum is None: assert toAdd == cluster.biosNode toAddNum = totalNodes - Utils.Print("TEMP FOUND NODE checking bios: {}".format(toAddNum)) - Utils.Print("TEMP toAddNum: {}".format(toAddNum)) publishTrans = security_group([toAddNum]) publishTransId = Node.getTransId(publishTrans[1]) verifyParticipantsTransactionFinalized(publishTransId) From 83d46ab230ef2237c0c0b1bc2c4af16e4d8b3bed Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 13 Apr 2021 17:58:33 -0500 Subject: [PATCH 106/157] Changed prodCount for single producers to ensure that LIB trails enough behind HEAD. --- tests/privacy_simple_network.py | 5 ++++- tests/privacy_startup_network.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index a1d10c9c1be..b55b5f96b2c 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -70,7 +70,10 @@ cluster.cleanup() Print("Stand up cluster") - if cluster.launch(pnodes=pnodes, totalNodes=totalNodes, prodCount=1, onlyBios=False, configSecurityGroup=True) is False: + # adjust prodCount to ensure that lib trails more than 1 block behind head + prodCount = 1 if pnodes > 1 else 2 + + if cluster.launch(pnodes=pnodes, totalNodes=totalNodes, prodCount=prodCount, onlyBios=False, configSecurityGroup=True) is False: cmdError("launcher") errorExit("Failed to stand up eos cluster.") diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index ed9aa478548..d2f676b8822 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -78,7 +78,10 @@ topo[pairedRelayNodeNum] = apiNodeNums Utils.Print("topo: {}".format(json.dumps(topo, indent=4, sort_keys=True))) - if cluster.launch(pnodes=pnodes, totalNodes=totalNodes, prodCount=1, onlyBios=False, dontBootstrap=dontBootstrap, configSecurityGroup=True, topo=topo) is False: + # adjust prodCount to ensure that lib trails more than 1 block behind head + prodCount = 1 if pnodes > 1 else 2 + + if cluster.launch(pnodes=pnodes, totalNodes=totalNodes, prodCount=prodCount, onlyBios=False, dontBootstrap=dontBootstrap, configSecurityGroup=True, topo=topo) is False: cmdError("launcher") errorExit("Failed to stand up eos cluster.") From bb668ecd90aab6170780adacf574e1b0b0d1bfbc Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 14 Apr 2021 14:08:24 -0500 Subject: [PATCH 107/157] Moved parseProducers from Cluster to Node and made member function and added getParticipantNum method. --- tests/Cluster.py | 29 +++++++++-------------- tests/Node.py | 21 ++++++++++++++++ tests/nodeos_forked_chain_test.py | 2 +- tests/nodeos_high_transaction_test.py | 2 +- tests/nodeos_short_fork_take_over_test.py | 2 +- tests/nodeos_voting_test.py | 2 +- 6 files changed, 36 insertions(+), 22 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index fb879ab2688..8e0f6a07e9b 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -538,6 +538,8 @@ def connectGroup(group, producerNodes, bridgeNodes) : if onlyBios: self.nodes=[biosNode] + self.totalNodes = totalNodes + # ensure cluster node are inter-connected by ensuring everyone has block 1 Utils.Print("Cluster viability smoke test. Validate every cluster node has block 1. ") if not self.waitOnClusterBlockNumSync(1): @@ -1032,24 +1034,6 @@ def parseProducerKeys(configFile, nodeName): return producerKeys - @staticmethod - def parseProducers(nodeNum): - """Parse node config file for producers.""" - - configFile=Utils.getNodeConfigDir(nodeNum, "config.ini") - if Utils.Debug: Utils.Print("Parsing config file %s" % configFile) - configStr=None - with open(configFile, 'r') as f: - configStr=f.read() - - pattern=r"^\s*producer-name\s*=\W*(\w+)\W*$" - producerMatches=re.findall(pattern, configStr, re.MULTILINE) - if producerMatches is None: - if Utils.Debug: Utils.Print("Failed to find producers.") - return None - - return producerMatches - @staticmethod def parseClusterKeys(totalNodes): """Parse cluster config file. Updates producer keys data members.""" @@ -1877,3 +1861,12 @@ def verifyInSync(self, sourceNodeNum=0, specificNodes=None): if error: self.reportInfo() Utils.errorExit(error) + + def getParticipantNum(self, nodeToIdentify): + num = 0 + for node in self.nodes: + if node == nodeToIdentify: + return num + num += 1 + assert nodeToIdentify == self.biosNode + return self.totalNodes diff --git a/tests/Node.py b/tests/Node.py index 61c69991f36..7a1cbc3bc04 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1687,3 +1687,24 @@ def waitForIrreversibleBlockProducedBy(self, producer, startBlockNum=0, retry=10 retry = retry - 1 startBlockNum = latestBlockNum + 1 return False + + @staticmethod + def parseProducers(nodeNum): + """Parse node config file for producers.""" + + configFile=Utils.getNodeConfigDir(nodeNum, "config.ini") + if Utils.Debug: Utils.Print("Parsing config file %s" % configFile) + configStr=None + with open(configFile, 'r') as f: + configStr=f.read() + + pattern=r"^\s*producer-name\s*=\W*(\w+)\W*$" + producerMatches=re.findall(pattern, configStr, re.MULTILINE) + if producerMatches is None: + if Utils.Debug: Utils.Print("Failed to find producers.") + return None + + return producerMatches + + def getProducers(self): + return Node.parseProducers(self.nodeId) \ No newline at end of file diff --git a/tests/nodeos_forked_chain_test.py b/tests/nodeos_forked_chain_test.py index 06ae59646a9..9baa3b1a7e5 100755 --- a/tests/nodeos_forked_chain_test.py +++ b/tests/nodeos_forked_chain_test.py @@ -207,7 +207,7 @@ def getMinHeadAndLib(prodNodes): producers=[] for i in range(0, totalNodes): node=cluster.getNode(i) - node.producers=Cluster.parseProducers(i) + node.producers=node.getProducers() numProducers=len(node.producers) Print("node has producers=%s" % (node.producers)) if numProducers==0: diff --git a/tests/nodeos_high_transaction_test.py b/tests/nodeos_high_transaction_test.py index 6108af9ffa0..8107e973ce5 100755 --- a/tests/nodeos_high_transaction_test.py +++ b/tests/nodeos_high_transaction_test.py @@ -116,7 +116,7 @@ allNodes=cluster.getNodes() for i in range(0, totalNodes): node=allNodes[i] - nodeProducers=Cluster.parseProducers(i) + nodeProducers=node.getProducers() numProducers=len(nodeProducers) Print("node has producers=%s" % (nodeProducers)) if numProducers==0: diff --git a/tests/nodeos_short_fork_take_over_test.py b/tests/nodeos_short_fork_take_over_test.py index 29aa223aee2..f09b860fb74 100755 --- a/tests/nodeos_short_fork_take_over_test.py +++ b/tests/nodeos_short_fork_take_over_test.py @@ -170,7 +170,7 @@ def getMinHeadAndLib(prodNodes): producers=[] for i in range(0, totalNodes): node=cluster.getNode(i) - node.producers=Cluster.parseProducers(i) + node.producers=node.getProducers() numProducers=len(node.producers) Print("node has producers=%s" % (node.producers)) if numProducers==0: diff --git a/tests/nodeos_voting_test.py b/tests/nodeos_voting_test.py index a3c157e8027..ae951ef096b 100755 --- a/tests/nodeos_voting_test.py +++ b/tests/nodeos_voting_test.py @@ -202,7 +202,7 @@ def verifyProductionRounds(trans, node, prodsActive, rounds): for i in range(0, totalNodes): node=cluster.getNode(i) - node.producers=Cluster.parseProducers(i) + node.producers=node.getProducers() for prod in node.producers: trans=node.regproducer(cluster.defProducerAccounts[prod], "http::/mysite.com", 0, waitForTransBlock=False, exitOnError=True) From 21ca57efc64c30a630d763777a4f9c3e012a5433 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 14 Apr 2021 15:04:32 -0500 Subject: [PATCH 108/157] Added more senarios to test. --- tests/privacy_simple_network.py | 135 +++++++++++++++++++++++--------- 1 file changed, 97 insertions(+), 38 deletions(-) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index b55b5f96b2c..1295d5a7e66 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -80,7 +80,6 @@ Print("Validating system accounts after bootstrap") cluster.validateAccounts(None) - Utils.Print("\n\n\n\n\nCheck after KILL:") Utils.Print("\n\n\n\n\nNext Round of Info:") cluster.reportInfo() @@ -88,29 +87,33 @@ apiNodes = [cluster.getNode(x) for x in range(pnodes, totalNodes)] apiNodes.append(cluster.biosNode) - blockProducer = producers[0].getHeadOrLib()["producer"] + featureProdNum = 0 + blockProducer = producers[featureProdNum].getHeadOrLib()["producer"] + while blockProducer not in producers[featureProdNum].getProducers(): + featureProdNum += 1 + assert featureProdNum < pnodes, "Checked nodes {} through {} but could not find producer: {}".format(0, featureProdNum - 1, blockProducer) cluster.verifyInSync() - featureDict = producers[0].getSupportedProtocolFeatureDict() + featureDict = producers[featureProdNum].getSupportedProtocolFeatureDict() Utils.Print("feature dict: {}".format(json.dumps(featureDict, indent=4, sort_keys=True))) cluster.reportInfo() Utils.Print("Activating SECURITY_GROUP Feature") - #Utils.Print("act feature dict: {}".format(json.dumps(producers[0].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) + Utils.Print("act feature dict: {}".format(json.dumps(producers[featureProdNum].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) timeout = ( pnodes * 12 / 2 ) * 2 # (number of producers * blocks produced / 0.5 blocks per second) * 2 rounds - for producer in producers: - producers[0].waitUntilBeginningOfProdTurn(blockProducer, timeout=timeout) + for tryNum in range(3): # try 3 times to set the security group feature + producers[featureProdNum].waitUntilBeginningOfProdTurn(blockProducer, timeout=timeout) feature = "SECURITY_GROUP" - producers[0].activateFeatures([feature]) - if producers[0].containsFeatures([feature]): + producers[featureProdNum].activateFeatures([feature]) + if producers[featureProdNum].containsFeatures([feature]): break Utils.Print("SECURITY_GROUP Feature activated") cluster.reportInfo() - assert producers[0].containsFeatures([feature]), "{} feature was not activated".format(feature) + assert producers[featureProdNum].containsFeatures([feature]), "{} feature was not activated".format(feature) def publishContract(account, file, waitForTransBlock=False): Print("Publish contract") @@ -122,19 +125,32 @@ def publishContract(account, file, waitForTransBlock=False): participants = [x for x in producers] nonParticipants = [x for x in apiNodes] - def security_group(nodeNums): - action = None - for nodeNum in nodeNums: - if action is None: - action = '[[' - else: - action += ',' - action += '"{}"'.format(Node.participantName(nodeNum)) - action += ']]' - - Utils.Print("adding {} to the security group".format(action)) - trans = producers[0].pushMessage(cluster.eosioAccount.name, "add", action, "--permission eosio@active") - Utils.Print("add trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) + def security_group(addNodeNums=[], removeNodeNums=[]): + def createAction(nodeNums): + action = None + for nodeNum in nodeNums: + if action is None: + action = '[[' + else: + action += ',' + action += '"{}"'.format(Node.participantName(nodeNum)) + if action: + action += ']]' + return action + + addAction = createAction(addNodeNums) + removeAction = createAction(removeNodeNums) + + if addAction: + Utils.Print("adding {} to the security group".format(addAction)) + trans = producers[0].pushMessage(cluster.eosioAccount.name, "add", addAction, "--permission eosio@active") + Utils.Print("add trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) + + if removeAction: + Utils.Print("removing {} from the security group".format(removeAction)) + trans = producers[0].pushMessage(cluster.eosioAccount.name, "remove", removeAction, "--permission eosio@active") + Utils.Print("remove trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) + trans = producers[0].pushMessage(cluster.eosioAccount.name, "publish", "[0]", "--permission eosio@active") Utils.Print("publish action trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) return trans @@ -160,33 +176,76 @@ def verifyNonParticipants(transId): nonParticipantHead = nonParticipant.getBlockNum() assert nonParticipantHead < producerHead, "Participants (that are not producers themselves) should not advance head to {}, but it has advanced to {}".format(producerHead, nonParticipantHead) + def verifySecurityGroup(publishTransPair): + publishTransId = Node.getTransId(publishTransPair[1]) + verifyParticipantsTransactionFinalized(publishTransId) + verifyNonParticipants(publishTransId) + Utils.Print("Add all producers to security group") publishTrans = security_group([x for x in range(pnodes)]) - publishTransId = Node.getTransId(publishTrans[1]) - verifyParticipantsTransactionFinalized(publishTransId) - verifyNonParticipants(publishTransId) + verifySecurityGroup(publishTrans) + + cluster.reportInfo() + + # one by one add each nonParticipant to the security group while len(nonParticipants) > 0: toAdd = nonParticipants[0] participants.append(toAdd) del nonParticipants[0] Utils.Print("Take a non-participant and make a participant. Now there are {} participants and {} non-participants".format(len(participants), len(nonParticipants))) - toAddNum = None - num = 0 - for node in cluster.getNodes(): - if node == toAdd: - toAddNum = num - break - num += 1 - if toAddNum is None: - assert toAdd == cluster.biosNode - toAddNum = totalNodes + toAddNum = cluster.getParticipantNum(toAdd) publishTrans = security_group([toAddNum]) - publishTransId = Node.getTransId(publishTrans[1]) - verifyParticipantsTransactionFinalized(publishTransId) - verifyNonParticipants(publishTransId) + verifySecurityGroup(publishTrans) + cluster.reportInfo() + + + # one by one remove each (original) nonParticipant from the security group + while len(participants) > pnodes: + toRemove = participants[-1] + # popping off back of participants and need to push on the front of nonParticipants + nonParticipants.insert(0, toRemove) + del participants[-1] + Utils.Print("Take a participant and make a non-participant. Now there are {} participants and {} non-participants".format(len(participants), len(nonParticipants))) + toRemoveNum = cluster.getParticipantNum(toRemove) + publishTrans = security_group(removeNodeNums=[toRemoveNum]) + verifySecurityGroup(publishTrans) + cluster.reportInfo() + + + # if we have more than 1 api node, we will add and remove all those nodes in bulk, if not it is just a repeat of the above test + if totalNodes > pnodes + 1: + # add all the api nodes to security group at once + toAdd = [] + for apiNode in nonParticipants: + participantNum = cluster.getParticipantNum(apiNode) + toAdd.append(participantNum) + participants.extend(nonParticipants) + nonParticipants = [] + + Utils.Print("Add all api nodes to security group") + publishTrans = security_group(addNodeNums=toAdd) + verifySecurityGroup(publishTrans) + + cluster.reportInfo() + + + # remove all the api nodes from the security group at once + toRemove = [] + # index pnodes and following are moving to nonParticipants, so participants has everything before that + nonParticipants = participants[pnodes:] + participants = participants[:pnodes] + for apiNode in nonParticipants: + participantNum = cluster.getParticipantNum(apiNode) + toRemove.append(participantNum) + + Utils.Print("Remove all api nodes from security group") + publishTrans = security_group(removeNodeNums=toRemove) + verifySecurityGroup(publishTrans) + + cluster.reportInfo() testSuccessful=True finally: From e8a4e3042bc7a3b44d641f008fe334497f443ac8 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 15 Apr 2021 13:17:12 -0500 Subject: [PATCH 109/157] Cleaned up unused parameter. --- tests/Node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Node.py b/tests/Node.py index 7a1cbc3bc04..9e94dda08db 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1200,7 +1200,7 @@ def getBlockProducerByNum(self, blockNum, timeout=None, waitForBlock=True, exitO block=self.getBlock(blockNum, exitOnError=exitOnError) return Node.getBlockAttribute(block, "producer", blockNum, exitOnError=exitOnError) - def getBlockProducer(self, timeout=None, waitForBlock=True, exitOnError=True, blockType=BlockType.head): + def getBlockProducer(self, timeout=None, exitOnError=True, blockType=BlockType.head): blockNum=self.getBlockNum(blockType=blockType) block=self.getBlock(blockNum, exitOnError=exitOnError, blockType=blockType) return Node.getBlockAttribute(block, "producer", blockNum, exitOnError=exitOnError) From 23f8510dfed89f0d205fe4106587c1a609f9527d Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 15 Apr 2021 13:40:51 -0500 Subject: [PATCH 110/157] Added method for identifying the node for the provided producer. --- tests/Cluster.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/Cluster.py b/tests/Cluster.py index 8e0f6a07e9b..0536f1b057f 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -1870,3 +1870,14 @@ def getParticipantNum(self, nodeToIdentify): num += 1 assert nodeToIdentify == self.biosNode return self.totalNodes + + def getProducingNodeIndex(self, blockProducer): + featureProdNum = 0 + while featureProdNum < pnodes: + if blockProducer in self.nodes[featureProdNum].getProducers(): + return featureProdNum + + featureProdNum += 1 + + assert blockProducer in self.biosNode.getProducers(), "Checked all nodes but could not find producer: {}".format(blockProducer) + return "bios" From 27433a9696ab567b7653387a34ba64323a4b7e68 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 16 Apr 2021 10:43:52 -0500 Subject: [PATCH 111/157] Improvement for activating features and allowing more control for block header state methods. --- tests/Node.py | 71 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index 9e94dda08db..dc716e1b368 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1453,7 +1453,8 @@ def isDesiredProdTurn(): return beginningOfProdTurnHead # Require producer_api_plugin - def activateFeatures(self, features): + def activateFeatures(self, features, blocksToAdvance=2): + assert blocksToAdvance >= 0 featureDigests = [] for feature in features: protocolFeatureDigestDict = self.getSupportedProtocolFeatureDict() @@ -1465,16 +1466,44 @@ def activateFeatures(self, features): self.scheduleProtocolFeatureActivations(featureDigests) # Wait for the next block to be produced so the scheduled protocol feature is activated - assert self.waitForHeadToAdvance(blocksToAdvance=2), print("ERROR: TIMEOUT WAITING FOR activating features: {}".format(",".join(features))) + assert self.waitForHeadToAdvance(blocksToAdvance=blocksToAdvance), print("ERROR: TIMEOUT WAITING FOR activating features: {}".format(",".join(features))) + + def activateAndVerifyFeatures(self, features): + self.activateFeatures(features, blocksToAdvance=0) + headBlockNum = self.getBlockNum() + blockNum = headBlockNum + producers = {} + lastProducer = None + while True: + block = self.getBlock(blockNum) + blockHeaderState = self.getBlockHeaderState(blockNum) + if self.containsFeatures(features, blockHeaderState): + return + + producer = block["producer"] + producers[producer] += 1 + assert lastProducer != producer or producers[producer] == 1, \ + "We have already cycled through a complete cycle, so feature should have been set by now. \ + Initial block num: {}, looking at block num: {}".format(headBlockNum, blockNum) + + # feature should be in block for this node's producers, if it is at least 2 blocks after we sent the activate + minBlocksForGuarantee = 2 + assert producer not in self.getProducers() or blockNum - headBlockNum < minBlocksForGuarantee, \ + "It is {} blocks past the block when we activated the features and block num: {} was produced by this \ + node, so features should have been set." + self.waitForBlock(blockNum + 1) + blockNum = self.getBlockNum() + + # Require producer_api_plugin def activatePreactivateFeature(self): return self.activateFeatures(["PREACTIVATE_FEATURE"]) - def containsFeatures(self, features): + def containsFeatures(self, features, blockHeaderState=None): protocolFeatureDict = self.getSupportedProtocolFeatureDict() - blockHeaderState = self.getLatestBlockHeaderState() - assert blockHeaderState, "blockHeaderState should not be empty" + if blockHeaderState is None: + blockHeaderState = self.getLatestBlockHeaderState() for feature in features: featureDigest = protocolFeatureDict[feature]["feature_digest"] assert featureDigest, "{}'s Digest should not be empty".format(feature) @@ -1520,20 +1549,26 @@ def preactivateAllBuiltinProtocolFeature(self): def getLatestBlockHeaderState(self): headBlockNum = self.getHeadBlockNum() - for i in range(10): - cmdDesc = "get block {} --header-state".format(headBlockNum) - latestBlockHeaderState = self.processCleosCmd(cmdDesc, cmdDesc) - Utils.Print("block num: {}, block state: {}, head: {}".format(headBlockNum, latestBlockHeaderState, self.getHeadBlockNum())) - if latestBlockHeaderState: - return latestBlockHeaderState - time.sleep(1) - return None - - def getActivatedProtocolFeatures(self): - latestBlockHeaderState = self.getLatestBlockHeaderState() - if "activated_protocol_features" not in latestBlockHeaderState or "protocol_features" not in latestBlockHeaderState["activated_protocol_features"]: + return self.getBlockHeaderState(headBlockNum) + + def getBlockHeaderState(self, blockNum, errorOnNone=True): + cmdDesc = "get block {} --header-state".format(blockNum) + blockHeaderState = self.processCleosCmd(cmdDesc, cmdDesc) + if blockHeaderState is None and errorOnNone: + info = self.getInfo() + lib = info["last_irreversible_block_num"] + head = info["head_block_num"] + assert head == lib + 1, "getLatestBlockHeaderState failed to retrieve the latest block. This should be investigated." + Utils.errorExit("Called getLatestBlockHeaderState, which can only retrieve blocks in reversible database, but the test setup only has one producer so there" + + " is only 1 block in the reversible database. Test should be redesigned to aquire this information via another interface.") + return blockHeaderState + + def getActivatedProtocolFeatures(self, blockHeaderState=None): + if blockHeaderState is None: + blockHeaderState = self.getLatestBlockHeaderState() + if "activated_protocol_features" not in blockHeaderState or "protocol_features" not in blockHeaderState["activated_protocol_features"]: Utils.errorExit("getLatestBlockHeaderState did not return expected output, should contain [\"activated_protocol_features\"][\"protocol_features\"]: {}".format(latestBlockHeaderState)) - return latestBlockHeaderState["activated_protocol_features"]["protocol_features"] + return blockHeaderState["activated_protocol_features"]["protocol_features"] def modifyBuiltinPFSubjRestrictions(self, featureCodename, subjectiveRestriction={}): jsonPath = os.path.join(Utils.getNodeConfigDir(self.nodeId), From b9d27b8a0407e9dd8afc40fb9595227314190431 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 16 Apr 2021 11:32:40 -0500 Subject: [PATCH 112/157] Added scenarios of adding and removing participants and covers scenario #2. --- tests/privacy_simple_network.py | 107 ++++++++++++++++++++------------ 1 file changed, 69 insertions(+), 38 deletions(-) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 1295d5a7e66..72c0c0b4fab 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -87,33 +87,16 @@ apiNodes = [cluster.getNode(x) for x in range(pnodes, totalNodes)] apiNodes.append(cluster.biosNode) - featureProdNum = 0 - blockProducer = producers[featureProdNum].getHeadOrLib()["producer"] - while blockProducer not in producers[featureProdNum].getProducers(): - featureProdNum += 1 - assert featureProdNum < pnodes, "Checked nodes {} through {} but could not find producer: {}".format(0, featureProdNum - 1, blockProducer) - + feature = "SECURITY_GROUP" + Utils.Print("Activating {} Feature".format(feature)) + producers[0].activateAndVerifyFeatures({feature}) cluster.verifyInSync() - featureDict = producers[featureProdNum].getSupportedProtocolFeatureDict() + featureDict = producers[0].getSupportedProtocolFeatureDict() Utils.Print("feature dict: {}".format(json.dumps(featureDict, indent=4, sort_keys=True))) + Utils.Print("{} Feature activated".format(feature)) cluster.reportInfo() - Utils.Print("Activating SECURITY_GROUP Feature") - - Utils.Print("act feature dict: {}".format(json.dumps(producers[featureProdNum].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) - timeout = ( pnodes * 12 / 2 ) * 2 # (number of producers * blocks produced / 0.5 blocks per second) * 2 rounds - for tryNum in range(3): # try 3 times to set the security group feature - producers[featureProdNum].waitUntilBeginningOfProdTurn(blockProducer, timeout=timeout) - feature = "SECURITY_GROUP" - producers[featureProdNum].activateFeatures([feature]) - if producers[featureProdNum].containsFeatures([feature]): - break - - Utils.Print("SECURITY_GROUP Feature activated") - cluster.reportInfo() - - assert producers[featureProdNum].containsFeatures([feature]), "{} feature was not activated".format(feature) def publishContract(account, file, waitForTransBlock=False): Print("Publish contract") @@ -125,6 +108,8 @@ def publishContract(account, file, waitForTransBlock=False): participants = [x for x in producers] nonParticipants = [x for x in apiNodes] + # this is passed to limit the number of add/remove table entries are processed, but using it here to keep from getting duplicate transactions + publishProcessNum = 20 def security_group(addNodeNums=[], removeNodeNums=[]): def createAction(nodeNums): action = None @@ -151,7 +136,9 @@ def createAction(nodeNums): trans = producers[0].pushMessage(cluster.eosioAccount.name, "remove", removeAction, "--permission eosio@active") Utils.Print("remove trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) - trans = producers[0].pushMessage(cluster.eosioAccount.name, "publish", "[0]", "--permission eosio@active") + global publishProcessNum + publishProcessNum += 1 + trans = producers[0].pushMessage(cluster.eosioAccount.name, "publish", "[{}]".format(publishProcessNum), "--permission eosio@active") Utils.Print("publish action trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) return trans @@ -181,6 +168,31 @@ def verifySecurityGroup(publishTransPair): verifyParticipantsTransactionFinalized(publishTransId) verifyNonParticipants(publishTransId) + def moveToParticipants(): + movedNode = nonParticipants[0] + participants.append(movedNode) + del nonParticipants[0] + return movedNode + + def moveToNonParticipants(): + movedNode = participants[-1] + # popping off back of participants and need to push on the front of nonParticipants + nonParticipants.insert(0, movedNode) + del participants[-1] + return movedNode + + def addToSg(): + node = moveToParticipants() + Utils.Print("Take a non-participant and make a participant. Now there are {} participants and {} non-participants".format(len(participants), len(nonParticipants))) + toAddNum = cluster.getParticipantNum(node) + return security_group([toAddNum]) + + def remFromSg(): + node = moveToNonParticipants() + Utils.Print("Take a participant and make a non-participant. Now there are {} participants and {} non-participants".format(len(participants), len(nonParticipants))) + toRemoveNum = cluster.getParticipantNum(node) + return security_group(removeNodeNums=[toRemoveNum]) + Utils.Print("Add all producers to security group") publishTrans = security_group([x for x in range(pnodes)]) verifySecurityGroup(publishTrans) @@ -190,27 +202,14 @@ def verifySecurityGroup(publishTransPair): # one by one add each nonParticipant to the security group while len(nonParticipants) > 0: - toAdd = nonParticipants[0] - participants.append(toAdd) - del nonParticipants[0] - Utils.Print("Take a non-participant and make a participant. Now there are {} participants and {} non-participants".format(len(participants), len(nonParticipants))) - - toAddNum = cluster.getParticipantNum(toAdd) - publishTrans = security_group([toAddNum]) + publishTrans = addToSg() verifySecurityGroup(publishTrans) cluster.reportInfo() # one by one remove each (original) nonParticipant from the security group while len(participants) > pnodes: - toRemove = participants[-1] - # popping off back of participants and need to push on the front of nonParticipants - nonParticipants.insert(0, toRemove) - del participants[-1] - Utils.Print("Take a participant and make a non-participant. Now there are {} participants and {} non-participants".format(len(participants), len(nonParticipants))) - - toRemoveNum = cluster.getParticipantNum(toRemove) - publishTrans = security_group(removeNodeNums=[toRemoveNum]) + publishTrans = remFromSg() verifySecurityGroup(publishTrans) cluster.reportInfo() @@ -232,6 +231,38 @@ def verifySecurityGroup(publishTransPair): cluster.reportInfo() + # alternate adding/removing participants to ensure the security group doesn't change + initialBlockNum = None + blockNum = None + def is_done(): + # want to ensure that we can identify the range of libs the security group was changed in + return blockNum - initialBlockNum > 12 + + done = False + # keep adding and removing nodes till we are done + while not done: + if blockNum: + participants[0].waitForNextBlock() + + while not done and len(participants) > pnodes: + publishTrans = remFromSg() + Utils.Print("publishTrans: {}".format(json.dumps(publishTrans, indent=2))) + blockNum = Node.getTransBlockNum(publishTrans[1]) + if initialBlockNum is None: + initialBlockNum = blockNum + lastBlockNum = blockNum + done = is_done() + + while not done and len(nonParticipants) > 0: + publishTrans = addToSg() + blockNum = Node.getTransBlockNum(publishTrans[1]) + done = is_done() + + Utils.Print("First adjustment to security group was in block num: {}, verifying no changes till block num: {} is finalized".format(initialBlockNum, blockNum)) + verifySecurityGroup(publishTrans) + + cluster.reportInfo() + # remove all the api nodes from the security group at once toRemove = [] # index pnodes and following are moving to nonParticipants, so participants has everything before that From e2182ce595bdd1970f8a70e55f26da9bc40796e9 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 16 Apr 2021 11:45:25 -0500 Subject: [PATCH 113/157] Adding more non-producing nodes for the test. --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e7fc99911c4..7e673e289f6 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -119,7 +119,7 @@ add_test(NAME eosio_blocklog_prune_test COMMAND tests/eosio_blocklog_prune_test. set_property(TEST eosio_blocklog_prune_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME privacy_startup_network COMMAND tests/privacy_startup_network.py -p 1 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST privacy_startup_network PROPERTY LABELS nonparallelizable_tests) -add_test(NAME privacy_simple_network COMMAND tests/privacy_simple_network.py -p 2 -n 3 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME privacy_simple_network COMMAND tests/privacy_simple_network.py -p 2 -n 4 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST privacy_simple_network PROPERTY LABELS nonparallelizable_tests) # Long running tests From fafcfaa644460b6d705018940e59a6fca28b30a4 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 16 Apr 2021 13:15:35 -0500 Subject: [PATCH 114/157] Making logic more straightforward. --- tests/privacy_simple_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 72c0c0b4fab..9a41d9d8178 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -215,7 +215,7 @@ def remFromSg(): # if we have more than 1 api node, we will add and remove all those nodes in bulk, if not it is just a repeat of the above test - if totalNodes > pnodes + 1: + if len(apiNodes) > 1: # add all the api nodes to security group at once toAdd = [] for apiNode in nonParticipants: From 853c53b22fc12002cf1857a6aa500fd8449c6f8d Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 16 Apr 2021 16:17:32 -0500 Subject: [PATCH 115/157] Peer Review comment changes. --- tests/Node.py | 8 ++------ tests/privacy_simple_network.py | 14 +++----------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index dc716e1b368..ce8d2abd9e4 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1473,7 +1473,6 @@ def activateAndVerifyFeatures(self, features): headBlockNum = self.getBlockNum() blockNum = headBlockNum producers = {} - lastProducer = None while True: block = self.getBlock(blockNum) blockHeaderState = self.getBlockHeaderState(blockNum) @@ -1482,15 +1481,12 @@ def activateAndVerifyFeatures(self, features): producer = block["producer"] producers[producer] += 1 - assert lastProducer != producer or producers[producer] == 1, \ - "We have already cycled through a complete cycle, so feature should have been set by now. \ - Initial block num: {}, looking at block num: {}".format(headBlockNum, blockNum) # feature should be in block for this node's producers, if it is at least 2 blocks after we sent the activate minBlocksForGuarantee = 2 assert producer not in self.getProducers() or blockNum - headBlockNum < minBlocksForGuarantee, \ - "It is {} blocks past the block when we activated the features and block num: {} was produced by this \ - node, so features should have been set." + "It is {} blocks past the block when we activated the features and block num {} was produced by this \ + node, so features should have been set.".format(blockNum - headBlockNum, blockNum) self.waitForBlock(blockNum + 1) blockNum = self.getBlockNum() diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 9a41d9d8178..9e3c0e50750 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -112,16 +112,8 @@ def publishContract(account, file, waitForTransBlock=False): publishProcessNum = 20 def security_group(addNodeNums=[], removeNodeNums=[]): def createAction(nodeNums): - action = None - for nodeNum in nodeNums: - if action is None: - action = '[[' - else: - action += ',' - action += '"{}"'.format(Node.participantName(nodeNum)) - if action: - action += ']]' - return action + return None if len(nodeNums) else \ + "[[{}]]".format(','.join(['"[]"'.format(Node.participantName(nodeNum)) for nodeNum in nodeNums])) addAction = createAction(addNodeNums) removeAction = createAction(removeNodeNums) @@ -246,7 +238,7 @@ def is_done(): while not done and len(participants) > pnodes: publishTrans = remFromSg() - Utils.Print("publishTrans: {}".format(json.dumps(publishTrans, indent=2))) + Utils.Print("publishTrans: {}".format(json.dumps(publishTrans, indent=4))) blockNum = Node.getTransBlockNum(publishTrans[1]) if initialBlockNum is None: initialBlockNum = blockNum From 185975b5aade825aaa9d3820e139a8a2c533768e Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 16 Apr 2021 16:36:25 -0500 Subject: [PATCH 116/157] Missed committing change. --- tests/Cluster.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index 0536f1b057f..d82633de75a 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -1864,10 +1864,9 @@ def verifyInSync(self, sourceNodeNum=0, specificNodes=None): def getParticipantNum(self, nodeToIdentify): num = 0 - for node in self.nodes: + for num, node in zip(len(self.nodes), self.nodes): if node == nodeToIdentify: return num - num += 1 assert nodeToIdentify == self.biosNode return self.totalNodes From c55a0a6f266d194f9b9e57f37b154acf5a0c1d5b Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 16 Apr 2021 16:56:27 -0500 Subject: [PATCH 117/157] Cleanup --- tests/Cluster.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index d82633de75a..6f7638b62cf 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -1863,7 +1863,6 @@ def verifyInSync(self, sourceNodeNum=0, specificNodes=None): Utils.errorExit(error) def getParticipantNum(self, nodeToIdentify): - num = 0 for num, node in zip(len(self.nodes), self.nodes): if node == nodeToIdentify: return num From 518bb8d1d84eca6255a351e6448e6ced186113e5 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 16 Apr 2021 19:41:06 -0500 Subject: [PATCH 118/157] Fixed reversed logic for createAction. --- tests/privacy_simple_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 9e3c0e50750..748b7513278 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -112,7 +112,7 @@ def publishContract(account, file, waitForTransBlock=False): publishProcessNum = 20 def security_group(addNodeNums=[], removeNodeNums=[]): def createAction(nodeNums): - return None if len(nodeNums) else \ + return None if len(nodeNums) == 0 else \ "[[{}]]".format(','.join(['"[]"'.format(Node.participantName(nodeNum)) for nodeNum in nodeNums])) addAction = createAction(addNodeNums) From f2f963d7ff1b736fcc6aabc5abc0b3d61ab3d895 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Fri, 16 Apr 2021 21:48:37 -0400 Subject: [PATCH 119/157] tls integration tests added --- tests/CMakeLists.txt | 3 ++ tests/Cluster.py | 101 +++++++++++++++++++++------------- tests/Node.py | 2 +- tests/privacy_tls_test.py | 110 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 38 deletions(-) create mode 100755 tests/privacy_tls_test.py diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7e673e289f6..f25ea34d56b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -63,6 +63,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/blockvault_tests.py ${CMAKE_CURRENT_B configure_file(${CMAKE_CURRENT_SOURCE_DIR}/generate-certificates.sh ${CMAKE_CURRENT_BINARY_DIR}/generate-certificates.sh COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_startup_network.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_startup_network.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_simple_network.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_simple_network.py COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_tls_test.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_tls_test.py COPYONLY) #To run plugin_test with all log from blockchain displayed, put --verbose after --, i.e. plugin_test -- --verbose add_test(NAME plugin_test COMMAND plugin_test --report_level=detailed --color_output) @@ -121,6 +122,8 @@ add_test(NAME privacy_startup_network COMMAND tests/privacy_startup_network.py - set_property(TEST privacy_startup_network PROPERTY LABELS nonparallelizable_tests) add_test(NAME privacy_simple_network COMMAND tests/privacy_simple_network.py -p 2 -n 4 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST privacy_simple_network PROPERTY LABELS nonparallelizable_tests) +add_test(NAME privacy_tls_test COMMAND tests/privacy_tls_test.py -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST privacy_tls_test PROPERTY LABELS nonparallelizable_tests) # Long running tests add_test(NAME nodeos_sanity_lr_test COMMAND tests/nodeos_run_test.py -v --sanity-test --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/tests/Cluster.py b/tests/Cluster.py index 0536f1b057f..ba2d4948b2a 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -159,6 +159,46 @@ def setAlternateVersionLabels(self, file): self.alternateVersionLabels[label]=path if Utils.Debug: Utils.Print("Version label \"%s\" maps to \"%s\"" % (label, path)) + @staticmethod + def generateCertificates(directoryName, certNumber): + """ + Generates TLS certificates + directoryName: directory where to save certificates, it will be created in Utils.ConfigDir + """ + #path is relative + privacyDir=os.path.join(Utils.ConfigDir, directoryName) + if not os.path.isdir(privacyDir): + if Utils.Debug: Utils.Print("creating dir {} in dir: {}".format(privacyDir, os.getcwd())) + os.mkdir(privacyDir) + + old_cwd=os.getcwd() + os.chdir(privacyDir) + if Utils.Debug: Utils.Print("change to dir: {}".format(os.getcwd())) + genCertScript=os.path.join(old_cwd, "tests", "generate-certificates.sh") + cmd = "{} --days 1 --CA-org Block.one --CA-CN test-domain --org-mask node{{NUMBER}} --cn-mask test-domain{{NUMBER}} --group-size {} --use-RSA".format(genCertScript, certNumber) + rtn = Utils.runCmdReturnStr(cmd, silentErrors=False) + + with open("generate.log", 'w') as f: + f.write("executed cmd: {}".format(cmd)) + f.write("=========================") + f.write("OUTPUT") + f.write("=========================") + f.write("{}".format(rtn)) + os.chdir(old_cwd) + if Utils.Debug: Utils.Print("changed back to dir: {}".format(os.getcwd())) + + @staticmethod + def getPrivacyArguments(privacyDir, index): + """ + Generates TLS arguments for nodeos + """ + privacyDir=os.path.join(Utils.ConfigDir, privacyDir) + participantName = Node.participantName(index) + certAuth = os.path.join(privacyDir, "CA_cert.pem") + nodeCert = os.path.join(privacyDir, "{}.crt".format(participantName)) + nodeKey = os.path.join(privacyDir, "{}_key.pem".format(participantName)) + return "--p2p-tls-own-certificate-file {} --p2p-tls-private-key-file {} --p2p-tls-security-group-ca-file {}".format(nodeCert, nodeKey, certAuth) + # launch local nodes and set self.nodes # pylint: disable=too-many-locals # pylint: disable=too-many-return-statements @@ -167,7 +207,7 @@ def setAlternateVersionLabels(self, file): def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="mesh", delay=1, onlyBios=False, dontBootstrap=False, totalProducers=None, sharedProducers=0, extraNodeosArgs="", useBiosBootFile=True, specificExtraNodeosArgs=None, onlySetProds=False, pfSetupPolicy=PFSetupPolicy.FULL, alternateVersionLabelsFile=None, associatedNodeLabels=None, loadSystemContract=True, manualProducerNodeConf={}, - configSecurityGroup=False): + configSecurityGroup=False, printInfo=False): """Launch cluster. pnodes: producer nodes count unstartedNodes: non-producer nodes that are configured into the launch, but not started. Should be included in totalNodes. @@ -191,7 +231,12 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me loadSystemContract: indicate whether the eosio.system contract should be loaded (setting this to False causes useBiosBootFile to be treated as False) manualProducerNodeConf: additional producer public keys which is not automatically generated by launcher configSecurityGroup: configure the network for TLS and setup a certificate authority so the security group can be used + printInfo: prints information about cluster """ + if printInfo: + Utils.Print("SERVER: {}".format(self.host)) + Utils.Print("PORT: {}".format(self.port)) + assert(isinstance(topo, (str,dict))) assert PFSetupPolicy.isValid(pfSetupPolicy) if alternateVersionLabelsFile is not None: @@ -221,41 +266,17 @@ def insertSpecificExtraNodeosArgs(node, insertStr): specificExtraNodeosArgs[node] = arg + " " + insertStr if configSecurityGroup: - privacyDir=os.path.join(Utils.ConfigDir, "privacy") - if not os.path.isdir(privacyDir): - if Utils.Debug: Utils.Print("creating dir {} in dir: {}".format(privacyDir, os.getcwd())) - os.mkdir(privacyDir) - original=os.getcwd() - os.chdir(privacyDir) - if Utils.Debug: Utils.Print("change to dir: {}".format(os.getcwd())) - genCertScript=os.path.join(original, "tests", "generate-certificates.sh") - totalNodesInNetwork = totalNodes + 1 # account for bios node - cmd = "{} --days 1 --CA-org Block.one --CA-CN test-domain --org-mask node{{NUMBER}} --cn-mask test-domain{{NUMBER}} --group-size {} --use-RSA".format(genCertScript, totalNodesInNetwork) - rtn=Utils.runCmdReturnStr(cmd, silentErrors=False) - with open("generate.log", 'w') as f: - f.write("executed cmd: {}".format(cmd)) - f.write("=========================") - f.write("OUTPUT") - f.write("=========================") - f.write("{}".format(rtn)) - os.chdir(original) - if Utils.Debug: Utils.Print("changed back to dir: {}".format(os.getcwd())) + Cluster.generateCertificates("privacy", totalNodes + 1) + if specificExtraNodeosArgs is None: specificExtraNodeosArgs = {} - - certAuth = os.path.join(privacyDir, "CA_cert.pem") - def getArguments(number): - participantName = Node.participantName(number) - nodeCert = os.path.join(privacyDir, "{}.crt".format(participantName)) - nodeKey = os.path.join(privacyDir, "{}_key.pem".format(participantName)) - return "--p2p-tls-own-certificate-file {} --p2p-tls-private-key-file {} --p2p-tls-security-group-ca-file {}".format(nodeCert, nodeKey, certAuth) - + for node in range(totalNodes): - arguments = getArguments(node) + arguments = Cluster.getPrivacyArguments("privacy", node+1) if Utils.Debug: Utils.Print("adding arguments: {}".format(arguments)) insertSpecificExtraNodeosArgs(node, arguments) - arguments = getArguments(totalNodes) + arguments = Cluster.getPrivacyArguments("privacy", totalNodes+1) if Utils.Debug: Utils.Print("adding arguments: {}".format(arguments)) biosNodeNum = -1 insertSpecificExtraNodeosArgs(biosNodeNum, arguments) @@ -702,16 +723,16 @@ def doNodesHaveBlockNum(nodes, targetBlockNum, blockType, printCount): return ret @staticmethod - def getClientVersion(verbose=False): + def getClientVersion(): """Returns client version (string)""" - p = re.compile(r'^Build version:\s(\w+)\n$') + p = re.compile(r'^v?(.+)\n$') try: cmd="%s version client" % (Utils.EosClientPath) - if verbose: Utils.Print("cmd: %s" % (cmd)) + if Utils.Debug: Utils.Print("cmd: %s" % (cmd)) response=Utils.checkOutput(cmd.split()) assert(response) assert(isinstance(response, str)) - if verbose: Utils.Print("response: <%s>" % (response)) + if Utils.Debug: Utils.Print("response: <%s>" % (response)) m=p.match(response) if m is None: Utils.Print("ERROR: client version regex mismatch") @@ -1573,7 +1594,7 @@ def dumpErrorDetails(self): if self.useBiosBootFile: Cluster.dumpErrorDetailImpl(Cluster.__bootlog) - def killall(self, kill=True, silent=True, allInstances=False): + def killall(self, kill=True, silent=True, allInstances=False, cleanup=False): """Kill cluster nodeos instances. allInstances will kill all nodeos instances running on the system.""" signalNum=9 if kill else 15 cmd="%s -k %d" % (Utils.EosLauncherPath, signalNum) @@ -1595,6 +1616,9 @@ def killall(self, kill=True, silent=True, allInstances=False): os.kill(node.pid, signal.SIGKILL) except OSError as _: pass + + if cleanup: + self.cleanup() def bounce(self, nodes, silent=True): """Bounces nodeos instances as indicated by parameter nodes. @@ -1814,10 +1838,13 @@ def getAllNodes(self): nodes.append(self.biosNode) return nodes - def reportInfo(self): + def reportInfo(self, nodes=None): Utils.Print("\n\n\n*****************************") Utils.Print("All Nodes current info:") - for node in self.getAllNodes(): + if nodes is None: + nodes = self.getAllNodes() + assert isinstance(nodes, list) + for node in nodes: Utils.Print("Info: {}".format(json.dumps(node.getInfo(), indent=4, sort_keys=True))) Utils.Print("\n*****************************") diff --git a/tests/Node.py b/tests/Node.py index dc716e1b368..6f663317cc8 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1642,7 +1642,7 @@ def normalizeNumber(number): assert(len(ret) <= 13) return ret - return "node{}".format(normalizeNumber(nodeNumber + 1)) + return "node{}".format(normalizeNumber(nodeNumber)) def analyzeProduction(self, specificBlockNum=None, thresholdMs=500): dataDir=Utils.getNodeDataDir(self.nodeId) diff --git a/tests/privacy_tls_test.py b/tests/privacy_tls_test.py new file mode 100755 index 00000000000..070ffe5cf16 --- /dev/null +++ b/tests/privacy_tls_test.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +from testUtils import Account +from testUtils import Utils +from Cluster import Cluster +from WalletMgr import WalletMgr +from Node import BlockType +from Node import Node +from Node import ReturnType +from TestHelper import TestHelper + +import decimal +import re +import signal +import json +import os + +def makeRootCertArgs(privacyDir): + privacyDir=os.path.join(Utils.ConfigDir, privacyDir) + certAuth = os.path.join(privacyDir, "CA_cert.pem") + nodeCert = os.path.join(privacyDir, "CA_cert.pem") + nodeKey = os.path.join(privacyDir, "CA_key.pem") + return "--p2p-tls-own-certificate-file {} --p2p-tls-private-key-file {} --p2p-tls-security-group-ca-file {}".format(nodeCert, nodeKey, certAuth) + +def makeWrongPrivateKeyArgs(index): + participantName = Node.participantName(index) + certAuth = os.path.join(os.path.join(Utils.ConfigDir, "privacy1"), "CA_cert.pem") + nodeCert = os.path.join(os.path.join(Utils.ConfigDir, "privacy1"), "{}.crt".format(participantName)) + nodeKey = os.path.join(os.path.join(Utils.ConfigDir, "privacy2"), "{}_key.pem".format(participantName)) + return "--p2p-tls-own-certificate-file {} --p2p-tls-private-key-file {} --p2p-tls-security-group-ca-file {}".format(nodeCert, nodeKey, certAuth) + + +############################################################### +# privacy_tls_network +# +# General test for TLS peer to peer connections with different certificates or without certificates at all +# +############################################################### + +Print=Utils.Print + +args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v"}) + +pnodes=1 #always one as we testing just connection between peers so we don't need bios setup and need just eosio producer +totalNodes=8 #3 valid and 5 invalid cases, exclusing bios +dumpErrorDetails=args.dump_error_details +keepLogs=args.keep_logs +Utils.Debug=args.v + +testSuccessful=False +cluster=Cluster(host=TestHelper.LOCAL_HOST, port=TestHelper.DEFAULT_PORT, walletd=True) +try: + TestHelper.printSystemInfo("BEGIN") + cluster.killall(allInstances=True, cleanup=True) + + #this is for producer and 1 valid node + Cluster.generateCertificates("privacy1", 2) + #those are certificates for invalid cases + Cluster.generateCertificates("privacy2", 2) + + specificExtraNodeosArgs = {} + + #VALID CASES + #producer node + specificExtraNodeosArgs[-1] = Cluster.getPrivacyArguments("privacy1", 1) + #valid network member + specificExtraNodeosArgs[0] = Cluster.getPrivacyArguments("privacy1", 2) + #testing duplicate + specificExtraNodeosArgs[1] = Cluster.getPrivacyArguments("privacy1", 2) + #valid root certificate used as participant + specificExtraNodeosArgs[2] = makeRootCertArgs("privacy1") + + #INVALID CASES + #certificate out of group with same name #1 + specificExtraNodeosArgs[3] = Cluster.getPrivacyArguments("privacy2", 1) + #certificate out of group with same name #2 + specificExtraNodeosArgs[4] = Cluster.getPrivacyArguments("privacy2", 2) + #using invalid self-signed certificate + specificExtraNodeosArgs[5] = makeRootCertArgs("privacy2") + #valid CA and certificate but invalid password + specificExtraNodeosArgs[6] = makeWrongPrivateKeyArgs(2) + #no TLS arguments at all + specificExtraNodeosArgs[7] = "" + + if not cluster.launch(pnodes=pnodes, totalNodes=totalNodes, dontBootstrap=True, configSecurityGroup=False, specificExtraNodeosArgs=specificExtraNodeosArgs, printInfo=True): + Utils.cmdError("launcher") + Utils.errorExit("Failed to stand up eos cluster.") + + cluster.biosNode.waitForLibToAdvance() + + validNodes = [cluster.getNode(x) for x in range(3)] + [cluster.biosNode] + cluster.verifyInSync(specificNodes=validNodes) + if Utils.Debug: + Print("*****************************") + Print(" Valid nodes ") + Print("*****************************") + cluster.reportInfo(validNodes) + + invalidNodes = [cluster.getNode(x) for x in range(3, totalNodes)] + for node in invalidNodes: + assert node.getHeadBlockNum() == 1 + if Utils.Debug: + Print("*****************************") + Print(" Invalid nodes ") + Print("*****************************") + cluster.reportInfo(invalidNodes) + + testSuccessful=True +finally: + TestHelper.shutdown(cluster, cluster.walletMgr, testSuccessful, True, True, keepLogs, True, dumpErrorDetails) \ No newline at end of file From 7c2ebcbe1727cee73f2f15f7f20535cd478b53dd Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Fri, 16 Apr 2021 23:35:05 -0400 Subject: [PATCH 120/157] tests fix --- tests/Cluster.py | 6 +++--- tests/privacy_simple_network.py | 4 ++-- tests/privacy_tls_test.py | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index ba2d4948b2a..894f2974c6d 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -193,7 +193,7 @@ def getPrivacyArguments(privacyDir, index): Generates TLS arguments for nodeos """ privacyDir=os.path.join(Utils.ConfigDir, privacyDir) - participantName = Node.participantName(index) + participantName = Node.participantName(index+1) certAuth = os.path.join(privacyDir, "CA_cert.pem") nodeCert = os.path.join(privacyDir, "{}.crt".format(participantName)) nodeKey = os.path.join(privacyDir, "{}_key.pem".format(participantName)) @@ -272,11 +272,11 @@ def insertSpecificExtraNodeosArgs(node, insertStr): specificExtraNodeosArgs = {} for node in range(totalNodes): - arguments = Cluster.getPrivacyArguments("privacy", node+1) + arguments = Cluster.getPrivacyArguments("privacy", node) if Utils.Debug: Utils.Print("adding arguments: {}".format(arguments)) insertSpecificExtraNodeosArgs(node, arguments) - arguments = Cluster.getPrivacyArguments("privacy", totalNodes+1) + arguments = Cluster.getPrivacyArguments("privacy", totalNodes) if Utils.Debug: Utils.Print("adding arguments: {}".format(arguments)) biosNodeNum = -1 insertSpecificExtraNodeosArgs(biosNodeNum, arguments) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 9a41d9d8178..8cc92b62842 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -113,12 +113,12 @@ def publishContract(account, file, waitForTransBlock=False): def security_group(addNodeNums=[], removeNodeNums=[]): def createAction(nodeNums): action = None - for nodeNum in nodeNums: + for nodeIndex in nodeNums: if action is None: action = '[[' else: action += ',' - action += '"{}"'.format(Node.participantName(nodeNum)) + action += '"{}"'.format(Node.participantName(nodeIndex+1)) if action: action += ']]' return action diff --git a/tests/privacy_tls_test.py b/tests/privacy_tls_test.py index 070ffe5cf16..91696a39347 100755 --- a/tests/privacy_tls_test.py +++ b/tests/privacy_tls_test.py @@ -23,7 +23,7 @@ def makeRootCertArgs(privacyDir): return "--p2p-tls-own-certificate-file {} --p2p-tls-private-key-file {} --p2p-tls-security-group-ca-file {}".format(nodeCert, nodeKey, certAuth) def makeWrongPrivateKeyArgs(index): - participantName = Node.participantName(index) + participantName = Node.participantName(index+1) certAuth = os.path.join(os.path.join(Utils.ConfigDir, "privacy1"), "CA_cert.pem") nodeCert = os.path.join(os.path.join(Utils.ConfigDir, "privacy1"), "{}.crt".format(participantName)) nodeKey = os.path.join(os.path.join(Utils.ConfigDir, "privacy2"), "{}_key.pem".format(participantName)) @@ -62,23 +62,23 @@ def makeWrongPrivateKeyArgs(index): #VALID CASES #producer node - specificExtraNodeosArgs[-1] = Cluster.getPrivacyArguments("privacy1", 1) + specificExtraNodeosArgs[-1] = Cluster.getPrivacyArguments("privacy1", 0) #valid network member - specificExtraNodeosArgs[0] = Cluster.getPrivacyArguments("privacy1", 2) + specificExtraNodeosArgs[0] = Cluster.getPrivacyArguments("privacy1", 1) #testing duplicate - specificExtraNodeosArgs[1] = Cluster.getPrivacyArguments("privacy1", 2) + specificExtraNodeosArgs[1] = Cluster.getPrivacyArguments("privacy1", 1) #valid root certificate used as participant specificExtraNodeosArgs[2] = makeRootCertArgs("privacy1") #INVALID CASES #certificate out of group with same name #1 - specificExtraNodeosArgs[3] = Cluster.getPrivacyArguments("privacy2", 1) + specificExtraNodeosArgs[3] = Cluster.getPrivacyArguments("privacy2", 0) #certificate out of group with same name #2 - specificExtraNodeosArgs[4] = Cluster.getPrivacyArguments("privacy2", 2) + specificExtraNodeosArgs[4] = Cluster.getPrivacyArguments("privacy2", 1) #using invalid self-signed certificate specificExtraNodeosArgs[5] = makeRootCertArgs("privacy2") #valid CA and certificate but invalid password - specificExtraNodeosArgs[6] = makeWrongPrivateKeyArgs(2) + specificExtraNodeosArgs[6] = makeWrongPrivateKeyArgs(1) #no TLS arguments at all specificExtraNodeosArgs[7] = "" From 3c0a2445c54991f071ac70cac1f20ca83d46879d Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Sat, 17 Apr 2021 00:36:15 -0500 Subject: [PATCH 121/157] Fixing more errors. --- tests/privacy_simple_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 748b7513278..3388d40260b 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -113,7 +113,7 @@ def publishContract(account, file, waitForTransBlock=False): def security_group(addNodeNums=[], removeNodeNums=[]): def createAction(nodeNums): return None if len(nodeNums) == 0 else \ - "[[{}]]".format(','.join(['"[]"'.format(Node.participantName(nodeNum)) for nodeNum in nodeNums])) + "[[{}]]".format(','.join(['"{}"'.format(Node.participantName(nodeNum)) for nodeNum in nodeNums])) addAction = createAction(addNodeNums) removeAction = createAction(removeNodeNums) From 3029fe0fca25c64e95033d632ac3aa745a181b60 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Sat, 17 Apr 2021 08:51:40 -0500 Subject: [PATCH 122/157] Fix another error. --- tests/Cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index 6f7638b62cf..77c8c37fd9d 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -1863,7 +1863,7 @@ def verifyInSync(self, sourceNodeNum=0, specificNodes=None): Utils.errorExit(error) def getParticipantNum(self, nodeToIdentify): - for num, node in zip(len(self.nodes), self.nodes): + for num, node in zip(range(len(self.nodes)), self.nodes): if node == nodeToIdentify: return num assert nodeToIdentify == self.biosNode From 353314edd5b3682612e58c5f110d8b0375ce1641 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 19 Apr 2021 13:33:57 -0500 Subject: [PATCH 123/157] Reduced the number of API nodes --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7e673e289f6..e7fc99911c4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -119,7 +119,7 @@ add_test(NAME eosio_blocklog_prune_test COMMAND tests/eosio_blocklog_prune_test. set_property(TEST eosio_blocklog_prune_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME privacy_startup_network COMMAND tests/privacy_startup_network.py -p 1 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST privacy_startup_network PROPERTY LABELS nonparallelizable_tests) -add_test(NAME privacy_simple_network COMMAND tests/privacy_simple_network.py -p 2 -n 4 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME privacy_simple_network COMMAND tests/privacy_simple_network.py -p 2 -n 3 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST privacy_simple_network PROPERTY LABELS nonparallelizable_tests) # Long running tests From 146b9e3d3dba16ab977df2af380c210fe6ed30dd Mon Sep 17 00:00:00 2001 From: Farhad Shahabi Date: Mon, 19 Apr 2021 17:51:49 -0400 Subject: [PATCH 124/157] Update tester.cpp Added push_action_no_produce method, pushes action without producing block --- libraries/testing/tester.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index 3641ebdf9d1..51f9df2f417 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -632,6 +632,20 @@ namespace eosio { namespace testing { return success(); } + transaction_trace_ptr base_tester::push_action_no_produce(action&& act, uint64_t authorizer) { + signed_transaction trx; + if (authorizer) { + act.authorization = vector{{account_name(authorizer), config::active_name}}; + } + trx.actions.emplace_back(std::move(act)); + set_transaction_headers(trx); + if (authorizer) { + trx.sign(get_private_key(account_name(authorizer), "active"), control->get_chain_id()); + } + + return push_transaction(trx); + } + transaction_trace_ptr base_tester::push_action( const account_name& code, const action_name& acttype, const account_name& actor, From 402f5574619fbb5d3c56c151101df2b3cd06fddb Mon Sep 17 00:00:00 2001 From: Farhad Shahabi Date: Mon, 19 Apr 2021 17:53:16 -0400 Subject: [PATCH 125/157] Update security_group_tests.cpp added test case: "test_participants_change_modified", create/remove member through push_action --- unittests/security_group_tests.cpp | 57 ++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/unittests/security_group_tests.cpp b/unittests/security_group_tests.cpp index 5a7c5da45ad..2f945859ddd 100644 --- a/unittests/security_group_tests.cpp +++ b/unittests/security_group_tests.cpp @@ -223,6 +223,7 @@ BOOST_AUTO_TEST_CASE(test_participants_change) { BOOST_TEST(chain.control->proposed_security_group_participants() == new_participants); BOOST_CHECK_EQUAL(chain.control->active_security_group().participants.size() , 0); +<<<<<<< Updated upstream BOOST_CHECK(!chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); { @@ -230,6 +231,9 @@ BOOST_AUTO_TEST_CASE(test_participants_change) { BOOST_REQUIRE_EQUAL(cur_security_group.version, 0); } +======= + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 2); +>>>>>>> Stashed changes chain.produce_block(); { @@ -367,6 +371,59 @@ std::vector participants_payload( participants_t names ) { return ds.storage(); } +BOOST_AUTO_TEST_CASE(test_participants_change_modified) { + eosio::testing::tester chain; + using namespace eosio::chain::literals; + + chain.create_accounts( {"alice"_n,"bob"_n,"charlie"_n} ); + chain.produce_block(); + + { + const auto& cur_security_group = chain.control->active_security_group(); + BOOST_REQUIRE_EQUAL(cur_security_group.version, 0); + BOOST_REQUIRE_EQUAL(cur_security_group.participants.size(), 0); + } + + chain.create_accounts({ "addmember"_n, "rmmember"_n }); + + chain.produce_block(); + + chain.set_code( "addmember"_n, add_security_group_participants_wast ); + chain.set_code( "rmmember"_n, remove_security_group_participants_wast ); + + chain.produce_block(); + + chain.push_action( "eosio"_n, "setpriv"_n, "eosio"_n, fc::mutable_variant_object()("account", "addmember"_n)("is_priv", 1)); + chain.push_action( "eosio"_n, "setpriv"_n, "eosio"_n, fc::mutable_variant_object()("account", "rmmember"_n)("is_priv", 1)); + + chain.produce_block(); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 0); + BOOST_CHECK_EQUAL(chain.control->active_security_group().participants.size(), 0); + + BOOST_TEST_REQUIRE(chain.push_action_no_produce( eosio::chain::action({}, "addmember"_n, {}, participants_payload({"alice"_n})), "addmember"_n.to_uint64_t())); + BOOST_TEST_REQUIRE(chain.push_action_no_produce( eosio::chain::action({}, "addmember"_n, {}, participants_payload({"bob"_n})), "addmember"_n.to_uint64_t())); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 2); + BOOST_CHECK_EQUAL(chain.control->active_security_group().participants.size(), 0); + + chain.produce_block(); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 0); + BOOST_CHECK(chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + + + BOOST_TEST_REQUIRE(chain.push_action_no_produce( eosio::chain::action({}, "rmmember"_n, {}, participants_payload({"alice"_n})), "rmmember"_n.to_uint64_t())); + BOOST_TEST(chain.control->proposed_security_group_participants() == participants_t{"bob"_n}); + BOOST_CHECK(chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + + chain.produce_block(); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 0); + BOOST_TEST(chain.control->active_security_group().participants == participants_t{"bob"_n}); + BOOST_CHECK(chain.control->in_active_security_group(participants_t{"bob"_n})); +} + BOOST_AUTO_TEST_CASE(test_security_group_intrinsic) { eosio::testing::tester chain1; From be6c5658c48cd1cce638772220d27e7e48b269a1 Mon Sep 17 00:00:00 2001 From: Farhad Shahabi Date: Mon, 19 Apr 2021 17:54:18 -0400 Subject: [PATCH 126/157] Update tester.hpp added push_action_no_produce() method. --- libraries/testing/include/eosio/testing/tester.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index 724a831cc9b..b57cbdfa246 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -225,6 +225,7 @@ namespace eosio { namespace testing { const variant_object& data, uint32_t expiration = DEFAULT_EXPIRATION_DELTA, uint32_t delay_sec = 0 ); + transaction_trace_ptr push_action_no_produce(action&& act, uint64_t authorizer); action get_action( account_name code, action_name acttype, vector auths, From b8f74c542ab7423191e349808e2cd353caf6652e Mon Sep 17 00:00:00 2001 From: Farhad Shahabi Date: Mon, 19 Apr 2021 18:43:52 -0400 Subject: [PATCH 127/157] Update security_group_tests.cpp --- unittests/security_group_tests.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/unittests/security_group_tests.cpp b/unittests/security_group_tests.cpp index 2f945859ddd..dd2ec005735 100644 --- a/unittests/security_group_tests.cpp +++ b/unittests/security_group_tests.cpp @@ -223,7 +223,6 @@ BOOST_AUTO_TEST_CASE(test_participants_change) { BOOST_TEST(chain.control->proposed_security_group_participants() == new_participants); BOOST_CHECK_EQUAL(chain.control->active_security_group().participants.size() , 0); -<<<<<<< Updated upstream BOOST_CHECK(!chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); { @@ -231,9 +230,6 @@ BOOST_AUTO_TEST_CASE(test_participants_change) { BOOST_REQUIRE_EQUAL(cur_security_group.version, 0); } -======= - BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 2); ->>>>>>> Stashed changes chain.produce_block(); { From 4b1076b3ea0f22b827118604ba109d9a5d71a7d5 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 20 Apr 2021 15:33:46 -0500 Subject: [PATCH 128/157] Added retrieving the participant name from the command line for a Node. --- tests/Cluster.py | 21 +++++++++++++++++++-- tests/Node.py | 15 ++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index 77c8c37fd9d..40a57a7966c 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -1483,6 +1483,21 @@ def discoverLocalNodes(self, totalNodes, timeout=None): if Utils.Debug: Utils.Print("Found %d nodes" % (len(nodes))) return nodes + @staticmethod + def extractParticipant(pgrepStr): + pattern = r"\s--p2p-tls-own-certificate-file(?:\s+.*?/|\s+)(node[1-5](?:.[1-5])*).crt" + m = re.search(pattern, pgrepStr, re.MULTILINE) + if m is not None: + Utils.Print("FOUND participant: {}, pgrepStr: {}".format(m.group(1), pgrepStr)) + return m.group(1) + + pattern = r"\s--p2p-tls-own-certificate-file" + m = re.search(pattern, pgrepStr, re.MULTILINE) + if m is not None: + Utils.Print("FOUND participant start: {}".format(m.group(0))) + + return None + # Populate a node matched to actual running instance def discoverLocalNode(self, nodeNum, psOut=None, timeout=None): if psOut is None: @@ -1495,7 +1510,8 @@ def discoverLocalNode(self, nodeNum, psOut=None, timeout=None): if m is None: Utils.Print("ERROR: Failed to find %s pid. Pattern %s" % (Utils.EosServerName, pattern)) return None - instance=Node(self.host, self.port + nodeNum, nodeNum, pid=int(m.group(1)), cmd=m.group(2), walletMgr=self.walletMgr) + participant = Cluster.extractParticipant(m.group(2)) + instance=Node(self.host, self.port + nodeNum, nodeNum, pid=int(m.group(1)), cmd=m.group(2), walletMgr=self.walletMgr, participant=participant) if Utils.Debug: Utils.Print("Node>", instance) return instance @@ -1508,7 +1524,8 @@ def discoverBiosNode(self, timeout=None): Utils.Print("ERROR: Failed to find %s pid. Pattern %s" % (Utils.EosServerName, pattern)) return None else: - return Node(Cluster.__BiosHost, Cluster.__BiosPort, "bios", pid=int(m.group(1)), cmd=m.group(2), walletMgr=self.walletMgr) + participant = Cluster.extractParticipant(m.group(2)) + return Node(Cluster.__BiosHost, Cluster.__BiosPort, "bios", pid=int(m.group(1)), cmd=m.group(2), walletMgr=self.walletMgr, participant=participant) # Kills a percentange of Eos instances starting from the tail and update eosInstanceInfos state def killSomeEosInstances(self, killCount, killSignalStr=Utils.SigKillTag): diff --git a/tests/Node.py b/tests/Node.py index ce8d2abd9e4..1621955596a 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -29,7 +29,7 @@ class Node(object): # pylint: disable=too-many-instance-attributes # pylint: disable=too-many-arguments - def __init__(self, host, port, nodeId, pid=None, cmd=None, walletMgr=None): + def __init__(self, host, port, nodeId, pid=None, cmd=None, walletMgr=None, participant=None): self.host=host self.port=port self.pid=pid @@ -48,6 +48,8 @@ def __init__(self, host, port, nodeId, pid=None, cmd=None, walletMgr=None): self.transCache={} self.walletMgr=walletMgr self.missingTransaction=False + self.participant=participant + if participant is not None: Utils.Print("Creating participant: {}".format(participant)) self.popenProc=None # initial process is started by launcher, this will only be set on relaunch def eosClientArgs(self): @@ -55,7 +57,11 @@ def eosClientArgs(self): return self.endpointArgs + walletArgs + " " + Utils.MiscEosClientArgs def __str__(self): - return "Host: %s, Port:%d, NodeNum:%s, Pid:%s" % (self.host, self.port, self.nodeId, self.pid) + participantStr = ", Participant: {}".format(self.participant) if self.participant else "" + return "Host: {}, Port:{}, NodeNum: {}, Pid: {}{}".format(self.host, self.port, self.nodeId, self.pid, participantStr) + + def __eq__(self, obj): + return isinstance(obj, Node) and str(self) == str(obj) @staticmethod def validateTransaction(trans): @@ -1738,4 +1744,7 @@ def parseProducers(nodeNum): return producerMatches def getProducers(self): - return Node.parseProducers(self.nodeId) \ No newline at end of file + return Node.parseProducers(self.nodeId) + + def getParticipant(self): + return self.participant From 457a22ce4a72b9bb944d8a78e22c20b4055371da Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 20 Apr 2021 15:34:53 -0500 Subject: [PATCH 129/157] Improved logging for Security Group changes. --- plugins/net_plugin/net_plugin.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index b017e992859..4d56b9247f7 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -3676,6 +3676,7 @@ namespace eosio { // update connections // + fc_dlog( logger, "SecurityGroup changed to version: ${v}", ("v", security_group.current_version()) ); for(auto& connection : connections) { const auto& participant = connection->participant_name(); if(participant && security_group.is_in_security_group(participant.value())) { From abf570c1511e1588654973c2e2f60939cf695e7e Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 20 Apr 2021 15:36:27 -0500 Subject: [PATCH 130/157] Added SecurityGroup helper python class. --- tests/CMakeLists.txt | 1 + tests/Cluster.py | 13 +++++++ tests/SecurityGroup.py | 82 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 tests/SecurityGroup.py diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e7fc99911c4..296b605e889 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -17,6 +17,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/testUtils.py ${CMAKE_CURRENT_BINARY_D configure_file(${CMAKE_CURRENT_SOURCE_DIR}/WalletMgr.py ${CMAKE_CURRENT_BINARY_DIR}/WalletMgr.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Node.py ${CMAKE_CURRENT_BINARY_DIR}/Node.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Cluster.py ${CMAKE_CURRENT_BINARY_DIR}/Cluster.py COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/SecurityGroup.py ${CMAKE_CURRENT_BINARY_DIR}/SecurityGroup.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/TestHelper.py ${CMAKE_CURRENT_BINARY_DIR}/TestHelper.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/p2p_tests/dawn_515/test.sh ${CMAKE_CURRENT_BINARY_DIR}/p2p_tests/dawn_515/test.sh COPYONLY) diff --git a/tests/Cluster.py b/tests/Cluster.py index 40a57a7966c..56276c04584 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -18,6 +18,7 @@ from testUtils import BlockLogAction from Node import BlockType from Node import Node +from SecurityGroup import SecurityGroup from WalletMgr import WalletMgr # Protocol Feature Setup Policy @@ -121,6 +122,8 @@ def __init__(self, walletd=False, localCluster=True, host="localhost", port=8888 self.alternateVersionLabels=Cluster.__defaultAlternateVersionLabels() self.biosNode = None + self.securityGroupEnabled = False + def setChainStrategy(self, chainSyncStrategy=Utils.SyncReplayTag): self.__chainSyncStrategy=self.__chainSyncStrategies.get(chainSyncStrategy) @@ -221,6 +224,7 @@ def insertSpecificExtraNodeosArgs(node, insertStr): specificExtraNodeosArgs[node] = arg + " " + insertStr if configSecurityGroup: + self.securityGroupEnabled = True privacyDir=os.path.join(Utils.ConfigDir, "privacy") if not os.path.isdir(privacyDir): if Utils.Debug: Utils.Print("creating dir {} in dir: {}".format(privacyDir, os.getcwd())) @@ -1896,3 +1900,12 @@ def getProducingNodeIndex(self, blockProducer): assert blockProducer in self.biosNode.getProducers(), "Checked all nodes but could not find producer: {}".format(blockProducer) return "bios" + + def getSecurityGroup(self, require=True): + assert not require or self.securityGroupEnabled, "Need to launch Cluster with configSecurityGroup=True to create a SecurityGroup" + if self.securityGroupEnabled: + for node in self.getAllNodes(): + Utils.Print("Creating securityGroup with participant: {}".format(node.getParticipant())) + return SecurityGroup(self.getAllNodes(), self.eosioAccount, defaultNode=self.nodes[0]) + + return None \ No newline at end of file diff --git a/tests/SecurityGroup.py b/tests/SecurityGroup.py new file mode 100644 index 00000000000..68f10fe0030 --- /dev/null +++ b/tests/SecurityGroup.py @@ -0,0 +1,82 @@ +import copy +import json +import Node + +from testUtils import Utils + +# pylint: disable=too-many-public-methods +class SecurityGroup(object): + + # pylint: disable=too-many-instance-attributes + # pylint: disable=too-many-arguments + def __init__(self, nonParticipants, contractAccount, defaultNode=None, minAddRemEntriesToPublish=100): + self.participants = [] + self.contractAccount = contractAccount + assert len(nonParticipants) > 0 + self.nonParticipants = copy.deepcopy(nonParticipants) + if Utils.Debug: Utils.Print("Creating SecurityGroup with the following nonParticipants: []".format(SecurityGroup.createAction(self.nonParticipants))) + self.defaultNode = defaultNode if defaultNode else nonParticipants[0] + self.publishProcessNum = minAddRemEntriesToPublish + Utils.Print("Publish contract") + self.contractTrans = self.defaultNode.publishContract(self.contractAccount, "unittests/test-contracts/security_group_test/", "eosio.secgrp.wasm", "eosio.secgrp.abi", waitForTransBlock=True) + + @staticmethod + def __lessThan(lhsNode, rhsNode): + assert lhsNode != rhsNode + return lhsNode.getParticipant() < rhsNode.getParticipant() + + def __consistentStorage(self): + intersect = list(set([x.getParticipant() for x in self.participants]) & set([x.getParticipant() for x in self.nonParticipants])) + if len(intersect) != 0: + Utils.exitError("Values: {}, are in the nonParticipants list: {}, and the participants list: {}".format(", ".join(intersect), SecurityGroup.createAction(self.nonParticipants), SecurityGroup.createAction(self.participants))) + + def __addParticipants(self, nodes): + if Utils.Debug: Utils.Print("Moving the following: {}, from the nonParticipants list: {}, to the participants list: {}".format(SecurityGroup.createAction(nodes), SecurityGroup.createAction(self.nonParticipants), SecurityGroup.createAction(self.participants))) + for node in nodes: + if node not in self.nonParticipants: + for nonParticipant in self.nonParticipants: + match = "" if node == nonParticipant else " not" + Utils.Print("node: '{}' does{} match '{}'".format(node, match, nonParticipant)) + assert node in self.nonParticipants, "Cannot remove {} from nonParticipants list: {}".format(node, SecurityGroup.createAction(self.nonParticipants)) + self.nonParticipants.remove(node) + + self.participants.extend(nodes) + if Utils.Debug: Utils.Print("nonParticipants list: {}, to the participants list: {}".format(SecurityGroup.createAction(self.nonParticipants), SecurityGroup.createAction(self.participants))) + self.__consistentStorage() + + def __remParticipants(self, nodes): + if Utils.Debug: Utils.Print("Moving the following: {}, from the participants list: {}, to the non-participants list: {}".format(SecurityGroup.createAction(nodes), SecurityGroup.createAction(self.participants), SecurityGroup.createAction(self.nonParticipants))) + for node in nodes: + self.participants.remove(node) + + self.nonParticipants.extend(nodes) + if Utils.Debug: Utils.Print("participants list: {}, to the non-participants list: {}".format(SecurityGroup.createAction(self.participants), SecurityGroup.createAction(self.nonParticipants))) + self.__consistentStorage() + + @staticmethod + def createAction(nodes): + return None if len(nodes) == 0 else \ + "[[{}]]".format(','.join(['"{}"'.format(node.getParticipant()) for node in nodes])) + + def securityGroup(self, addNodes=[], removeNodes=[]): + + addAction = SecurityGroup.createAction(addNodes) + # doing deep copy in case the passed in list IS participants or nonParticipants lists, which will be adjusted + self.__addParticipants(copy.deepcopy(addNodes)) + removeAction = SecurityGroup.createAction(removeNodes) + self.__remParticipants(copy.deepcopy(removeNodes)) + + if addAction: + Utils.Print("adding {} to the security group".format(addAction)) + trans = self.defaultNode.pushMessage(self.contractAccount.name, "add", addAction, "--permission eosio@active") + Utils.Print("add trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) + + if removeAction: + Utils.Print("removing {} from the security group".format(removeAction)) + trans = self.defaultNode.pushMessage(self.contractAccount.name, "remove", removeAction, "--permission eosio@active") + Utils.Print("remove trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) + + self.publishProcessNum += 1 + trans = self.defaultNode.pushMessage(self.contractAccount.name, "publish", "[{}]".format(self.publishProcessNum), "--permission eosio@active") + Utils.Print("publish action trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) + return trans From 9e74dea7312889c79acd814884458680e6df9ca3 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 20 Apr 2021 15:37:11 -0500 Subject: [PATCH 131/157] Initial conversion to using SecurityGroup. --- tests/privacy_simple_network.py | 129 +++++++++----------------------- 1 file changed, 37 insertions(+), 92 deletions(-) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 3388d40260b..1c429b0238f 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -9,7 +9,6 @@ from Node import ReturnType from TestHelper import TestHelper -import copy import decimal import re import signal @@ -86,6 +85,8 @@ producers = [cluster.getNode(x) for x in range(pnodes) ] apiNodes = [cluster.getNode(x) for x in range(pnodes, totalNodes)] apiNodes.append(cluster.biosNode) + Utils.Print("producer participants: [{}]".format(", ".join([x.getParticipant() for x in producers]))) + Utils.Print("api participants: [{}]".format(", ".join([x.getParticipant() for x in apiNodes]))) feature = "SECURITY_GROUP" Utils.Print("Activating {} Feature".format(feature)) @@ -98,58 +99,24 @@ Utils.Print("{} Feature activated".format(feature)) cluster.reportInfo() - def publishContract(account, file, waitForTransBlock=False): - Print("Publish contract") - cluster.reportInfo() - return producers[0].publishContract(account, "unittests/test-contracts/security_group_test/", "{}.wasm".format(file), "{}.abi".format(file), waitForTransBlock=waitForTransBlock) - - publishContract(cluster.eosioAccount, 'eosio.secgrp', waitForTransBlock=True) - - participants = [x for x in producers] - nonParticipants = [x for x in apiNodes] - - # this is passed to limit the number of add/remove table entries are processed, but using it here to keep from getting duplicate transactions - publishProcessNum = 20 - def security_group(addNodeNums=[], removeNodeNums=[]): - def createAction(nodeNums): - return None if len(nodeNums) == 0 else \ - "[[{}]]".format(','.join(['"{}"'.format(Node.participantName(nodeNum)) for nodeNum in nodeNums])) - - addAction = createAction(addNodeNums) - removeAction = createAction(removeNodeNums) - - if addAction: - Utils.Print("adding {} to the security group".format(addAction)) - trans = producers[0].pushMessage(cluster.eosioAccount.name, "add", addAction, "--permission eosio@active") - Utils.Print("add trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) - - if removeAction: - Utils.Print("removing {} from the security group".format(removeAction)) - trans = producers[0].pushMessage(cluster.eosioAccount.name, "remove", removeAction, "--permission eosio@active") - Utils.Print("remove trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) - - global publishProcessNum - publishProcessNum += 1 - trans = producers[0].pushMessage(cluster.eosioAccount.name, "publish", "[{}]".format(publishProcessNum), "--permission eosio@active") - Utils.Print("publish action trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) - return trans + securityGroup = cluster.getSecurityGroup() def verifyParticipantsTransactionFinalized(transId): Utils.Print("Verify participants are in sync") - for part in participants: + for part in securityGroup.participants: if part.waitForTransFinalization(transId) == None: Utils.errorExit("Transaction: {}, never finalized".format(trans)) def verifyNonParticipants(transId): Utils.Print("Verify non-participants don't receive blocks") - publishBlock = producers[0].getBlockIdByTransId(transId) - prodLib = producers[0].getBlockNum(blockType=BlockType.lib) + publishBlock = securityGroup.defaultNode.getBlockIdByTransId(transId) + prodLib = securityGroup.defaultNode.getBlockNum(blockType=BlockType.lib) waitForLib = prodLib + 3 * 12 - if producers[0].waitForBlock(waitForLib, blockType=BlockType.lib) == None: - Utils.errorExit("Producer did not advance lib the expected amount. Starting lib: {}, exp lib: {}, actual state: {}".format(prodLib, waitForLib, producers[0].getInfo())) - producerHead = producers[0].getBlockNum() + if securityGroup.defaultNode.waitForBlock(waitForLib, blockType=BlockType.lib) == None: + Utils.errorExit("Producer did not advance lib the expected amount. Starting lib: {}, exp lib: {}, actual state: {}".format(prodLib, waitForLib, securityGroup.defaultNode.getInfo())) + producerHead = securityGroup.defaultNode.getBlockNum() - for nonParticipant in nonParticipants: + for nonParticipant in securityGroup.nonParticipants: nonParticipantPostLIB = nonParticipant.getBlockNum(blockType=BlockType.lib) assert nonParticipantPostLIB < publishBlock, "Participants not in security group should not have advanced LIB to {}, but it has advanced to {}".format(publishBlock, nonParticipantPostLIB) nonParticipantHead = nonParticipant.getBlockNum() @@ -160,47 +127,35 @@ def verifySecurityGroup(publishTransPair): verifyParticipantsTransactionFinalized(publishTransId) verifyNonParticipants(publishTransId) - def moveToParticipants(): - movedNode = nonParticipants[0] - participants.append(movedNode) - del nonParticipants[0] - return movedNode - - def moveToNonParticipants(): - movedNode = participants[-1] - # popping off back of participants and need to push on the front of nonParticipants - nonParticipants.insert(0, movedNode) - del participants[-1] - return movedNode - def addToSg(): - node = moveToParticipants() - Utils.Print("Take a non-participant and make a participant. Now there are {} participants and {} non-participants".format(len(participants), len(nonParticipants))) - toAddNum = cluster.getParticipantNum(node) - return security_group([toAddNum]) + trans = securityGroup.securityGroup([securityGroup.nonParticipants[0]]) + Utils.Print("Take a non-participant and make a participant. Now there are {} participants and {} non-participants".format(len(securityGroup.participants), len(securityGroup.nonParticipants))) + return trans def remFromSg(): - node = moveToNonParticipants() - Utils.Print("Take a participant and make a non-participant. Now there are {} participants and {} non-participants".format(len(participants), len(nonParticipants))) - toRemoveNum = cluster.getParticipantNum(node) - return security_group(removeNodeNums=[toRemoveNum]) + trans = securityGroup.securityGroup(removeNodes=[securityGroup.participants[-1]]) + Utils.Print("Take a participant and make a non-participant. Now there are {} participants and {} non-participants".format(len(securityGroup.participants), len(securityGroup.nonParticipants))) + return trans Utils.Print("Add all producers to security group") - publishTrans = security_group([x for x in range(pnodes)]) + publishTrans = securityGroup.securityGroup([cluster.getNodes()[x] for x in range(pnodes)]) verifySecurityGroup(publishTrans) cluster.reportInfo() - + Utils.Print("One by one, add each API Node to the security group") # one by one add each nonParticipant to the security group - while len(nonParticipants) > 0: + while len(securityGroup.nonParticipants) > 0: + Utils.Print("TEST - Add one") publishTrans = addToSg() verifySecurityGroup(publishTrans) cluster.reportInfo() + Utils.Print("One by one, remove each API Node from the security group") # one by one remove each (original) nonParticipant from the security group - while len(participants) > pnodes: + while len(securityGroup.participants) > pnodes: + Utils.Print("TEST - Remove one") publishTrans = remFromSg() verifySecurityGroup(publishTrans) cluster.reportInfo() @@ -208,16 +163,8 @@ def remFromSg(): # if we have more than 1 api node, we will add and remove all those nodes in bulk, if not it is just a repeat of the above test if len(apiNodes) > 1: - # add all the api nodes to security group at once - toAdd = [] - for apiNode in nonParticipants: - participantNum = cluster.getParticipantNum(apiNode) - toAdd.append(participantNum) - participants.extend(nonParticipants) - nonParticipants = [] - - Utils.Print("Add all api nodes to security group") - publishTrans = security_group(addNodeNums=toAdd) + Utils.Print("Add all api nodes to security group at the same time") + publishTrans = securityGroup.securityGroup(addNodes=securityGroup.nonParticipants) verifySecurityGroup(publishTrans) cluster.reportInfo() @@ -234,9 +181,9 @@ def is_done(): # keep adding and removing nodes till we are done while not done: if blockNum: - participants[0].waitForNextBlock() + securityGroup.defaultNode.waitForNextBlock() - while not done and len(participants) > pnodes: + while not done and len(securityGroup.participants) > pnodes: publishTrans = remFromSg() Utils.Print("publishTrans: {}".format(json.dumps(publishTrans, indent=4))) blockNum = Node.getTransBlockNum(publishTrans[1]) @@ -245,7 +192,7 @@ def is_done(): lastBlockNum = blockNum done = is_done() - while not done and len(nonParticipants) > 0: + while not done and len(securityGroup.nonParticipants) > 0: publishTrans = addToSg() blockNum = Node.getTransBlockNum(publishTrans[1]) done = is_done() @@ -255,17 +202,15 @@ def is_done(): cluster.reportInfo() - # remove all the api nodes from the security group at once - toRemove = [] - # index pnodes and following are moving to nonParticipants, so participants has everything before that - nonParticipants = participants[pnodes:] - participants = participants[:pnodes] - for apiNode in nonParticipants: - participantNum = cluster.getParticipantNum(apiNode) - toRemove.append(participantNum) - - Utils.Print("Remove all api nodes from security group") - publishTrans = security_group(removeNodeNums=toRemove) + if len(securityGroup.nonParticipants) > 0: + Utils.Print("Add all remaining non-participants to security group at the same time, so all api nodes can be removed as one group") + publishTrans = securityGroup.securityGroup(addNodes=securityGroup.nonParticipants) + verifySecurityGroup(publishTrans) + + cluster.reportInfo() + + Utils.Print("Remove all api nodes from security group at the same time") + publishTrans = securityGroup.securityGroup(removeNodes=apiNodes) verifySecurityGroup(publishTrans) cluster.reportInfo() From 523bf9e55ef80ccf78e9d10332cacfbae3f2d1a0 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 21 Apr 2021 09:34:51 -0500 Subject: [PATCH 132/157] Moved more code into SecurityGroup. --- tests/SecurityGroup.py | 53 +++++++++++++++++++++-- tests/privacy_simple_network.py | 77 ++++++++------------------------- 2 files changed, 68 insertions(+), 62 deletions(-) diff --git a/tests/SecurityGroup.py b/tests/SecurityGroup.py index 68f10fe0030..3a4434a8e0f 100644 --- a/tests/SecurityGroup.py +++ b/tests/SecurityGroup.py @@ -1,7 +1,9 @@ import copy import json -import Node +from Node import BlockType +from Node import Node +from Node import ReturnType from testUtils import Utils # pylint: disable=too-many-public-methods @@ -19,6 +21,7 @@ def __init__(self, nonParticipants, contractAccount, defaultNode=None, minAddRem self.publishProcessNum = minAddRemEntriesToPublish Utils.Print("Publish contract") self.contractTrans = self.defaultNode.publishContract(self.contractAccount, "unittests/test-contracts/security_group_test/", "eosio.secgrp.wasm", "eosio.secgrp.abi", waitForTransBlock=True) + self.publishTrans = None @staticmethod def __lessThan(lhsNode, rhsNode): @@ -58,7 +61,7 @@ def createAction(nodes): return None if len(nodes) == 0 else \ "[[{}]]".format(','.join(['"{}"'.format(node.getParticipant()) for node in nodes])) - def securityGroup(self, addNodes=[], removeNodes=[]): + def editSecurityGroup(self, addNodes=[], removeNodes=[]): addAction = SecurityGroup.createAction(addNodes) # doing deep copy in case the passed in list IS participants or nonParticipants lists, which will be adjusted @@ -77,6 +80,48 @@ def securityGroup(self, addNodes=[], removeNodes=[]): Utils.Print("remove trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) self.publishProcessNum += 1 - trans = self.defaultNode.pushMessage(self.contractAccount.name, "publish", "[{}]".format(self.publishProcessNum), "--permission eosio@active") - Utils.Print("publish action trans: {}".format(json.dumps(trans, indent=4, sort_keys=True))) + self.publishTrans = self.defaultNode.pushMessage(self.contractAccount.name, "publish", "[{}]".format(self.publishProcessNum), "--permission eosio@active")[1] + Utils.Print("publish action trans: {}".format(json.dumps(self.publishTrans, indent=4, sort_keys=True))) + return self.publishTrans + + def verifyParticipantsTransactionFinalized(self, transId = None): + Utils.Print("Verify participants are in sync") + if transId is None: + transId = Node.getTransId(self.publishTrans) + for part in self.participants: + if part.waitForTransFinalization(transId) == None: + Utils.errorExit("Transaction: {}, never finalized".format(trans)) + + def verifyNonParticipants(self, transId = None): + Utils.Print("Verify non-participants don't receive blocks") + if transId is None: + transId = Node.getTransId(self.publishTrans) + publishBlock = self.defaultNode.getBlockIdByTransId(transId) + prodLib = self.defaultNode.getBlockNum(blockType=BlockType.lib) + waitForLib = prodLib + 3 * 12 + if self.defaultNode.waitForBlock(waitForLib, blockType=BlockType.lib) == None: + Utils.errorExit("Producer did not advance lib the expected amount. Starting lib: {}, exp lib: {}, actual state: {}".format(prodLib, waitForLib, self.defaultNode.getInfo())) + producerHead = self.defaultNode.getBlockNum() + + for nonParticipant in self.nonParticipants: + nonParticipantPostLIB = nonParticipant.getBlockNum(blockType=BlockType.lib) + assert nonParticipantPostLIB < publishBlock, "Participants not in security group should not have advanced LIB to {}, but it has advanced to {}".format(publishBlock, nonParticipantPostLIB) + nonParticipantHead = nonParticipant.getBlockNum() + assert nonParticipantHead < producerHead, "Participants (that are not producers themselves) should not advance head to {}, but it has advanced to {}".format(producerHead, nonParticipantHead) + + def verifySecurityGroup(self, publishTrans = None): + if publishTrans is None: + publishTrans = self.publishTrans + publishTransId = Node.getTransId(publishTrans) + self.verifyParticipantsTransactionFinalized(publishTransId) + self.verifyNonParticipants(publishTransId) + + def addToSecurityGroup(self): + trans = self.editSecurityGroup([self.nonParticipants[0]]) + Utils.Print("Take a non-participant and make a participant. Now there are {} participants and {} non-participants".format(len(self.participants), len(self.nonParticipants))) + return trans + + def remFromSecurityGroup(self): + trans = self.editSecurityGroup(removeNodes=[self.participants[-1]]) + Utils.Print("Take a participant and make a non-participant. Now there are {} participants and {} non-participants".format(len(self.participants), len(self.nonParticipants))) return trans diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 1c429b0238f..52ff1e6bfa2 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -101,71 +101,33 @@ securityGroup = cluster.getSecurityGroup() - def verifyParticipantsTransactionFinalized(transId): - Utils.Print("Verify participants are in sync") - for part in securityGroup.participants: - if part.waitForTransFinalization(transId) == None: - Utils.errorExit("Transaction: {}, never finalized".format(trans)) - - def verifyNonParticipants(transId): - Utils.Print("Verify non-participants don't receive blocks") - publishBlock = securityGroup.defaultNode.getBlockIdByTransId(transId) - prodLib = securityGroup.defaultNode.getBlockNum(blockType=BlockType.lib) - waitForLib = prodLib + 3 * 12 - if securityGroup.defaultNode.waitForBlock(waitForLib, blockType=BlockType.lib) == None: - Utils.errorExit("Producer did not advance lib the expected amount. Starting lib: {}, exp lib: {}, actual state: {}".format(prodLib, waitForLib, securityGroup.defaultNode.getInfo())) - producerHead = securityGroup.defaultNode.getBlockNum() - - for nonParticipant in securityGroup.nonParticipants: - nonParticipantPostLIB = nonParticipant.getBlockNum(blockType=BlockType.lib) - assert nonParticipantPostLIB < publishBlock, "Participants not in security group should not have advanced LIB to {}, but it has advanced to {}".format(publishBlock, nonParticipantPostLIB) - nonParticipantHead = nonParticipant.getBlockNum() - assert nonParticipantHead < producerHead, "Participants (that are not producers themselves) should not advance head to {}, but it has advanced to {}".format(producerHead, nonParticipantHead) - - def verifySecurityGroup(publishTransPair): - publishTransId = Node.getTransId(publishTransPair[1]) - verifyParticipantsTransactionFinalized(publishTransId) - verifyNonParticipants(publishTransId) - - def addToSg(): - trans = securityGroup.securityGroup([securityGroup.nonParticipants[0]]) - Utils.Print("Take a non-participant and make a participant. Now there are {} participants and {} non-participants".format(len(securityGroup.participants), len(securityGroup.nonParticipants))) - return trans - - def remFromSg(): - trans = securityGroup.securityGroup(removeNodes=[securityGroup.participants[-1]]) - Utils.Print("Take a participant and make a non-participant. Now there are {} participants and {} non-participants".format(len(securityGroup.participants), len(securityGroup.nonParticipants))) - return trans - Utils.Print("Add all producers to security group") - publishTrans = securityGroup.securityGroup([cluster.getNodes()[x] for x in range(pnodes)]) - verifySecurityGroup(publishTrans) + securityGroup.editSecurityGroup([cluster.getNodes()[x] for x in range(pnodes)]) + securityGroup.verifySecurityGroup() cluster.reportInfo() Utils.Print("One by one, add each API Node to the security group") # one by one add each nonParticipant to the security group while len(securityGroup.nonParticipants) > 0: - Utils.Print("TEST - Add one") - publishTrans = addToSg() - verifySecurityGroup(publishTrans) + securityGroup.addToSecurityGroup() + securityGroup.verifySecurityGroup() cluster.reportInfo() Utils.Print("One by one, remove each API Node from the security group") # one by one remove each (original) nonParticipant from the security group while len(securityGroup.participants) > pnodes: - Utils.Print("TEST - Remove one") - publishTrans = remFromSg() - verifySecurityGroup(publishTrans) + securityGroup.remFromSecurityGroup() + securityGroup.verifySecurityGroup() cluster.reportInfo() # if we have more than 1 api node, we will add and remove all those nodes in bulk, if not it is just a repeat of the above test if len(apiNodes) > 1: Utils.Print("Add all api nodes to security group at the same time") - publishTrans = securityGroup.securityGroup(addNodes=securityGroup.nonParticipants) - verifySecurityGroup(publishTrans) + securityGroup.editSecurityGroup(addNodes=securityGroup.nonParticipants) + securityGroup.verifySecurityGroup() cluster.reportInfo() @@ -180,38 +142,37 @@ def is_done(): done = False # keep adding and removing nodes till we are done while not done: - if blockNum: - securityGroup.defaultNode.waitForNextBlock() + # waiting for a block change to prevent duplicate publish transactions + securityGroup.defaultNode.waitForNextBlock() while not done and len(securityGroup.participants) > pnodes: - publishTrans = remFromSg() - Utils.Print("publishTrans: {}".format(json.dumps(publishTrans, indent=4))) - blockNum = Node.getTransBlockNum(publishTrans[1]) + publishTrans = securityGroup.remFromSecurityGroup() + blockNum = Node.getTransBlockNum(publishTrans) if initialBlockNum is None: initialBlockNum = blockNum lastBlockNum = blockNum done = is_done() while not done and len(securityGroup.nonParticipants) > 0: - publishTrans = addToSg() - blockNum = Node.getTransBlockNum(publishTrans[1]) + publishTrans = securityGroup.addToSecurityGroup() + blockNum = Node.getTransBlockNum(publishTrans) done = is_done() Utils.Print("First adjustment to security group was in block num: {}, verifying no changes till block num: {} is finalized".format(initialBlockNum, blockNum)) - verifySecurityGroup(publishTrans) + securityGroup.verifySecurityGroup() cluster.reportInfo() if len(securityGroup.nonParticipants) > 0: Utils.Print("Add all remaining non-participants to security group at the same time, so all api nodes can be removed as one group") - publishTrans = securityGroup.securityGroup(addNodes=securityGroup.nonParticipants) - verifySecurityGroup(publishTrans) + securityGroup.editSecurityGroup(addNodes=securityGroup.nonParticipants) + securityGroup.verifySecurityGroup() cluster.reportInfo() Utils.Print("Remove all api nodes from security group at the same time") - publishTrans = securityGroup.securityGroup(removeNodes=apiNodes) - verifySecurityGroup(publishTrans) + securityGroup.editSecurityGroup(removeNodes=apiNodes) + securityGroup.verifySecurityGroup() cluster.reportInfo() From ac8ebb0f977bb39fe9f67c8c06bf56ed4887a32c Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 21 Apr 2021 11:41:58 -0500 Subject: [PATCH 133/157] Added block delay to ensure repeating a duplicate remove transactin does not occur. --- tests/privacy_simple_network.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 52ff1e6bfa2..20a19c2f22d 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -115,10 +115,11 @@ cluster.reportInfo() + removeTrans = None Utils.Print("One by one, remove each API Node from the security group") # one by one remove each (original) nonParticipant from the security group while len(securityGroup.participants) > pnodes: - securityGroup.remFromSecurityGroup() + removeTrans = securityGroup.remFromSecurityGroup() securityGroup.verifySecurityGroup() cluster.reportInfo() @@ -131,6 +132,9 @@ cluster.reportInfo() + # waiting for a block to change (2 blocks since transaction indication could be 1 behind) to prevent duplicate remove transactions + removeBlockNum = Node.getTransBlockNum(removeTrans) + securityGroup.defaultNode.waitForBlock(removeBlockNum + 2) # alternate adding/removing participants to ensure the security group doesn't change initialBlockNum = None From 22cb7806ef001e746597e751772eaa9178c05042 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 21 Apr 2021 14:05:52 -0500 Subject: [PATCH 134/157] Pulled activation of SECURITY_GROUP feature into SecurityGroup and PR cleanup. --- tests/SecurityGroup.py | 92 +++++++++++++++++++++------------ tests/privacy_simple_network.py | 20 ++----- 2 files changed, 65 insertions(+), 47 deletions(-) diff --git a/tests/SecurityGroup.py b/tests/SecurityGroup.py index 3a4434a8e0f..822242f2385 100644 --- a/tests/SecurityGroup.py +++ b/tests/SecurityGroup.py @@ -11,63 +11,84 @@ class SecurityGroup(object): # pylint: disable=too-many-instance-attributes # pylint: disable=too-many-arguments - def __init__(self, nonParticipants, contractAccount, defaultNode=None, minAddRemEntriesToPublish=100): + def __init__(self, nonParticipants, contractAccount, defaultNode=None, minAddRemEntriesToPublish=100, activateAndPublish=True): self.participants = [] self.contractAccount = contractAccount assert len(nonParticipants) > 0 - self.nonParticipants = copy.deepcopy(nonParticipants) + self.nonParticipants = copy.copy(nonParticipants) if Utils.Debug: Utils.Print("Creating SecurityGroup with the following nonParticipants: []".format(SecurityGroup.createAction(self.nonParticipants))) self.defaultNode = defaultNode if defaultNode else nonParticipants[0] self.publishProcessNum = minAddRemEntriesToPublish - Utils.Print("Publish contract") - self.contractTrans = self.defaultNode.publishContract(self.contractAccount, "unittests/test-contracts/security_group_test/", "eosio.secgrp.wasm", "eosio.secgrp.abi", waitForTransBlock=True) + if activateAndPublish: + SecurityGroup.activateFeature(self.defaultNode) + + self.contractTrans = SecurityGroup.publishContract(self.defaultNode, self.contractAccount) + else: + self.contractTrans = None + self.publishTrans = None + # activate the SECURITY_GROUP feature @staticmethod - def __lessThan(lhsNode, rhsNode): - assert lhsNode != rhsNode - return lhsNode.getParticipant() < rhsNode.getParticipant() + def activateFeature(node): + feature = "SECURITY_GROUP" + Utils.Print("Activating {} Feature".format(feature)) + node.activateAndVerifyFeatures({feature}) + + featureDict = node.getSupportedProtocolFeatureDict() + Utils.Print("feature dict: {}".format(json.dumps(featureDict, indent=4, sort_keys=True))) - def __consistentStorage(self): - intersect = list(set([x.getParticipant() for x in self.participants]) & set([x.getParticipant() for x in self.nonParticipants])) - if len(intersect) != 0: - Utils.exitError("Values: {}, are in the nonParticipants list: {}, and the participants list: {}".format(", ".join(intersect), SecurityGroup.createAction(self.nonParticipants), SecurityGroup.createAction(self.participants))) + Utils.Print("{} Feature activated".format(feature)) + + # publish the eosio.secgrp contract + @staticmethod + def publishContract(node, account): + Utils.Print("Publish contract") + return node.publishContract(account, "unittests/test-contracts/security_group_test/", "eosio.secgrp.wasm", "eosio.secgrp.abi", waitForTransBlock=True) + # move the provided nodes from the nonParticipants list to the participants list def __addParticipants(self, nodes): + if len(nodes) == 0: + return + if Utils.Debug: Utils.Print("Moving the following: {}, from the nonParticipants list: {}, to the participants list: {}".format(SecurityGroup.createAction(nodes), SecurityGroup.createAction(self.nonParticipants), SecurityGroup.createAction(self.participants))) for node in nodes: - if node not in self.nonParticipants: - for nonParticipant in self.nonParticipants: - match = "" if node == nonParticipant else " not" - Utils.Print("node: '{}' does{} match '{}'".format(node, match, nonParticipant)) assert node in self.nonParticipants, "Cannot remove {} from nonParticipants list: {}".format(node, SecurityGroup.createAction(self.nonParticipants)) self.nonParticipants.remove(node) self.participants.extend(nodes) - if Utils.Debug: Utils.Print("nonParticipants list: {}, to the participants list: {}".format(SecurityGroup.createAction(self.nonParticipants), SecurityGroup.createAction(self.participants))) - self.__consistentStorage() + # move the provided nodes from the participants list to the nonParticipants list def __remParticipants(self, nodes): + if len(nodes) == 0: + return + if Utils.Debug: Utils.Print("Moving the following: {}, from the participants list: {}, to the non-participants list: {}".format(SecurityGroup.createAction(nodes), SecurityGroup.createAction(self.participants), SecurityGroup.createAction(self.nonParticipants))) for node in nodes: self.participants.remove(node) self.nonParticipants.extend(nodes) - if Utils.Debug: Utils.Print("participants list: {}, to the non-participants list: {}".format(SecurityGroup.createAction(self.participants), SecurityGroup.createAction(self.nonParticipants))) - self.__consistentStorage() + # create the action payload for an add or remove action @staticmethod def createAction(nodes): return None if len(nodes) == 0 else \ "[[{}]]".format(','.join(['"{}"'.format(node.getParticipant()) for node in nodes])) + # sends actions to add/remove the provided nodes to/from the network's security group def editSecurityGroup(self, addNodes=[], removeNodes=[]): + def copyIfNeeded(nodes): + # doing deep copy in case the passed in list IS one of our lists, which will be adjusted + if nodes is self.participants or nodes is self.nonParticipants: + return copy.copy(nodes) + return nodes + addAction = SecurityGroup.createAction(addNodes) - # doing deep copy in case the passed in list IS participants or nonParticipants lists, which will be adjusted - self.__addParticipants(copy.deepcopy(addNodes)) + self.__addParticipants(copyIfNeeded(addNodes)) + removeAction = SecurityGroup.createAction(removeNodes) - self.__remParticipants(copy.deepcopy(removeNodes)) + self.__remParticipants(copyIfNeeded(removeNodes)) if addAction: Utils.Print("adding {} to the security group".format(addAction)) @@ -84,31 +105,36 @@ def editSecurityGroup(self, addNodes=[], removeNodes=[]): Utils.Print("publish action trans: {}".format(json.dumps(self.publishTrans, indent=4, sort_keys=True))) return self.publishTrans - def verifyParticipantsTransactionFinalized(self, transId = None): + # verify that the transaction ID is found, and finalized, in every node in the participants list + def verifyParticipantsTransactionFinalized(self, transId): Utils.Print("Verify participants are in sync") - if transId is None: - transId = Node.getTransId(self.publishTrans) + assert transId for part in self.participants: if part.waitForTransFinalization(transId) == None: Utils.errorExit("Transaction: {}, never finalized".format(trans)) - def verifyNonParticipants(self, transId = None): + # verify that the block for the transaction ID is never finalized in nonParticipants + def verifyNonParticipants(self, transId): Utils.Print("Verify non-participants don't receive blocks") - if transId is None: - transId = Node.getTransId(self.publishTrans) + assert transId publishBlock = self.defaultNode.getBlockIdByTransId(transId) + + # first ensure that enough time has passed that the nonParticipant is not just trailing behind prodLib = self.defaultNode.getBlockNum(blockType=BlockType.lib) waitForLib = prodLib + 3 * 12 if self.defaultNode.waitForBlock(waitForLib, blockType=BlockType.lib) == None: Utils.errorExit("Producer did not advance lib the expected amount. Starting lib: {}, exp lib: {}, actual state: {}".format(prodLib, waitForLib, self.defaultNode.getInfo())) producerHead = self.defaultNode.getBlockNum() + # verify each nonParticipant in the list has not advanced its lib to the publish block, since the block that would cause it to become finalized would + # never have been forwarded to a nonParticipant for nonParticipant in self.nonParticipants: nonParticipantPostLIB = nonParticipant.getBlockNum(blockType=BlockType.lib) assert nonParticipantPostLIB < publishBlock, "Participants not in security group should not have advanced LIB to {}, but it has advanced to {}".format(publishBlock, nonParticipantPostLIB) nonParticipantHead = nonParticipant.getBlockNum() assert nonParticipantHead < producerHead, "Participants (that are not producers themselves) should not advance head to {}, but it has advanced to {}".format(producerHead, nonParticipantHead) + # verify that the participants' and nonParticipants' nodes are consistent based on the publish transaction def verifySecurityGroup(self, publishTrans = None): if publishTrans is None: publishTrans = self.publishTrans @@ -116,12 +142,14 @@ def verifySecurityGroup(self, publishTrans = None): self.verifyParticipantsTransactionFinalized(publishTransId) self.verifyNonParticipants(publishTransId) - def addToSecurityGroup(self): - trans = self.editSecurityGroup([self.nonParticipants[0]]) + def moveToSecurityGroup(self, index = 0): + assert abs(index) < len(self.nonParticipants) + trans = self.editSecurityGroup([self.nonParticipants[index]]) Utils.Print("Take a non-participant and make a participant. Now there are {} participants and {} non-participants".format(len(self.participants), len(self.nonParticipants))) return trans - def remFromSecurityGroup(self): - trans = self.editSecurityGroup(removeNodes=[self.participants[-1]]) + def removeFromSecurityGroup(self, index = -1): + assert abs(index) < len(self.participants) + trans = self.editSecurityGroup(removeNodes=[self.participants[index]]) Utils.Print("Take a participant and make a non-participant. Now there are {} participants and {} non-participants".format(len(self.participants), len(self.nonParticipants))) return trans diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 20a19c2f22d..f0b7f39d563 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -88,18 +88,8 @@ Utils.Print("producer participants: [{}]".format(", ".join([x.getParticipant() for x in producers]))) Utils.Print("api participants: [{}]".format(", ".join([x.getParticipant() for x in apiNodes]))) - feature = "SECURITY_GROUP" - Utils.Print("Activating {} Feature".format(feature)) - producers[0].activateAndVerifyFeatures({feature}) - cluster.verifyInSync() - - featureDict = producers[0].getSupportedProtocolFeatureDict() - Utils.Print("feature dict: {}".format(json.dumps(featureDict, indent=4, sort_keys=True))) - - Utils.Print("{} Feature activated".format(feature)) - cluster.reportInfo() - securityGroup = cluster.getSecurityGroup() + cluster.reportInfo() Utils.Print("Add all producers to security group") securityGroup.editSecurityGroup([cluster.getNodes()[x] for x in range(pnodes)]) @@ -110,7 +100,7 @@ Utils.Print("One by one, add each API Node to the security group") # one by one add each nonParticipant to the security group while len(securityGroup.nonParticipants) > 0: - securityGroup.addToSecurityGroup() + securityGroup.moveToSecurityGroup() securityGroup.verifySecurityGroup() cluster.reportInfo() @@ -119,7 +109,7 @@ Utils.Print("One by one, remove each API Node from the security group") # one by one remove each (original) nonParticipant from the security group while len(securityGroup.participants) > pnodes: - removeTrans = securityGroup.remFromSecurityGroup() + removeTrans = securityGroup.removeFromSecurityGroup() securityGroup.verifySecurityGroup() cluster.reportInfo() @@ -150,7 +140,7 @@ def is_done(): securityGroup.defaultNode.waitForNextBlock() while not done and len(securityGroup.participants) > pnodes: - publishTrans = securityGroup.remFromSecurityGroup() + publishTrans = securityGroup.removeFromSecurityGroup() blockNum = Node.getTransBlockNum(publishTrans) if initialBlockNum is None: initialBlockNum = blockNum @@ -158,7 +148,7 @@ def is_done(): done = is_done() while not done and len(securityGroup.nonParticipants) > 0: - publishTrans = securityGroup.addToSecurityGroup() + publishTrans = securityGroup.moveToSecurityGroup() blockNum = Node.getTransBlockNum(publishTrans) done = is_done() From 6225f30a475ee36c412469648b9f1410698b00cd Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 22 Apr 2021 08:30:18 -0500 Subject: [PATCH 135/157] More cleanup. --- tests/SecurityGroup.py | 5 +++-- tests/privacy_simple_network.py | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/SecurityGroup.py b/tests/SecurityGroup.py index 822242f2385..bac5a24ab1c 100644 --- a/tests/SecurityGroup.py +++ b/tests/SecurityGroup.py @@ -43,8 +43,9 @@ def activateFeature(node): # publish the eosio.secgrp contract @staticmethod def publishContract(node, account): - Utils.Print("Publish contract") - return node.publishContract(account, "unittests/test-contracts/security_group_test/", "eosio.secgrp.wasm", "eosio.secgrp.abi", waitForTransBlock=True) + contract = "eosio.secgrp" + Utils.Print("Publish {} contract".format(contract)) + return node.publishContract(account, "unittests/test-contracts/security_group_test/", "{}.wasm".format(contract), "{}.abi".format(contract), waitForTransBlock=True) # move the provided nodes from the nonParticipants list to the participants list def __addParticipants(self, nodes): diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index f0b7f39d563..3d7d198aa35 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -144,7 +144,6 @@ def is_done(): blockNum = Node.getTransBlockNum(publishTrans) if initialBlockNum is None: initialBlockNum = blockNum - lastBlockNum = blockNum done = is_done() while not done and len(securityGroup.nonParticipants) > 0: From d617d9c3e1e109006212555952dc08c8ee4a9f2d Mon Sep 17 00:00:00 2001 From: Farhad Shahabi Date: Thu, 22 Apr 2021 12:44:18 -0400 Subject: [PATCH 136/157] Update security_group_tests.cpp --- unittests/security_group_tests.cpp | 84 ++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/unittests/security_group_tests.cpp b/unittests/security_group_tests.cpp index dd2ec005735..04a2c1fd983 100644 --- a/unittests/security_group_tests.cpp +++ b/unittests/security_group_tests.cpp @@ -420,6 +420,90 @@ BOOST_AUTO_TEST_CASE(test_participants_change_modified) { BOOST_CHECK(chain.control->in_active_security_group(participants_t{"bob"_n})); } +BOOST_AUTO_TEST_CASE(test_participants_change_2_chains) { + // chain2 trails chain a block behind since it reads the records of last finalized block from chain + eosio::testing::tester chain; + eosio::testing::tester chain2; + using namespace eosio::chain::literals; + + chain.create_accounts( {"alice"_n,"bob"_n,"charlie"_n} ); + + chain2.push_block(chain.produce_block()); + + { + const auto& cur_security_group = chain.control->active_security_group(); + BOOST_REQUIRE_EQUAL(cur_security_group.version, 0); + BOOST_REQUIRE_EQUAL(cur_security_group.participants.size(), 0); + } + + chain.create_accounts({ "addmember"_n, "rmmember"_n }); + + chain2.push_block(chain.produce_block()); + + chain.set_code( "addmember"_n, add_security_group_participants_wast ); + chain.set_code( "rmmember"_n, remove_security_group_participants_wast ); + + chain2.push_block(chain.produce_block()); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 0); + BOOST_CHECK_EQUAL(chain.control->active_security_group().participants.size(), 0); + BOOST_CHECK_EQUAL(chain2.control->proposed_security_group_participants().size(), 0); + BOOST_CHECK_EQUAL(chain2.control->active_security_group().participants.size(), 0); + + chain.push_action( "eosio"_n, "setpriv"_n, "eosio"_n, fc::mutable_variant_object()("account", "addmember"_n)("is_priv", 1)); + chain.push_action( "eosio"_n, "setpriv"_n, "eosio"_n, fc::mutable_variant_object()("account", "rmmember"_n)("is_priv", 1)); + + chain2.push_block(chain.produce_block()); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size(), 0); + BOOST_CHECK_EQUAL(chain.control->active_security_group().participants.size(), 0); + BOOST_CHECK_EQUAL(chain2.control->proposed_security_group_participants().size(), 0); + BOOST_CHECK_EQUAL(chain2.control->active_security_group().participants.size(), 0); + + BOOST_TEST_REQUIRE(chain.push_action_no_produce( eosio::chain::action({}, "addmember"_n, {}, participants_payload({"alice"_n})), "addmember"_n.to_uint64_t())); + BOOST_TEST_REQUIRE(chain.push_action_no_produce( eosio::chain::action({}, "addmember"_n, {}, participants_payload({"bob"_n})), "addmember"_n.to_uint64_t())); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size(), 2); + BOOST_CHECK_EQUAL(chain.control->active_security_group().participants.size(), 0); + BOOST_CHECK_EQUAL(chain2.control->proposed_security_group_participants().size(), 0); + BOOST_CHECK_EQUAL(chain2.control->active_security_group().participants.size(), 0); + + chain2.push_block(chain.produce_block()); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 0); + BOOST_CHECK(chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + BOOST_CHECK_EQUAL(chain2.control->proposed_security_group_participants().size(), 2); + BOOST_CHECK_EQUAL(chain2.control->active_security_group().participants.size(), 0); + + chain2.push_block(chain.produce_block()); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 0); + BOOST_CHECK(chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + BOOST_CHECK_EQUAL(chain2.control->proposed_security_group_participants().size(), 0); // + BOOST_CHECK(chain2.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + + BOOST_TEST_REQUIRE(chain.push_action_no_produce(eosio::chain::action({}, "rmmember"_n, {}, participants_payload({"alice"_n})), "rmmember"_n.to_uint64_t())); + + BOOST_TEST(chain.control->proposed_security_group_participants() == participants_t{"bob"_n}); + BOOST_CHECK(chain.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + BOOST_CHECK_EQUAL(chain2.control->proposed_security_group_participants().size(), 0); // + BOOST_CHECK(chain2.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + + chain2.push_block(chain.produce_block()); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 0); + BOOST_TEST(chain.control->active_security_group().participants == participants_t{"bob"_n}); + BOOST_TEST(chain2.control->proposed_security_group_participants() == participants_t{"bob"_n}); + BOOST_CHECK(chain2.control->in_active_security_group(participants_t({"alice"_n, "bob"_n}))); + + chain2.push_block(chain.produce_block()); + + BOOST_CHECK_EQUAL(chain.control->proposed_security_group_participants().size() , 0); + BOOST_TEST(chain.control->active_security_group().participants == participants_t{"bob"_n}); + BOOST_CHECK_EQUAL(chain2.control->proposed_security_group_participants().size() , 0); + BOOST_TEST(chain2.control->active_security_group().participants == participants_t{"bob"_n}); +} + BOOST_AUTO_TEST_CASE(test_security_group_intrinsic) { eosio::testing::tester chain1; From 1ec27f1ccf8a7119bac1dc456a623d9448dba2c8 Mon Sep 17 00:00:00 2001 From: Farhad Shahabi Date: Thu, 22 Apr 2021 17:55:03 -0400 Subject: [PATCH 137/157] Update security_group_tests.cpp --- unittests/security_group_tests.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/unittests/security_group_tests.cpp b/unittests/security_group_tests.cpp index 04a2c1fd983..3072c83c963 100644 --- a/unittests/security_group_tests.cpp +++ b/unittests/security_group_tests.cpp @@ -421,7 +421,9 @@ BOOST_AUTO_TEST_CASE(test_participants_change_modified) { } BOOST_AUTO_TEST_CASE(test_participants_change_2_chains) { - // chain2 trails chain a block behind since it reads the records of last finalized block from chain + /* Note: + * because produce_block calls start_block, and push_block does not, chain2 will trail behind chain + */ eosio::testing::tester chain; eosio::testing::tester chain2; using namespace eosio::chain::literals; From 31312a766169eb7743bbd5afd8221464063846de Mon Sep 17 00:00:00 2001 From: Farhad Shahabi Date: Thu, 22 Apr 2021 18:20:50 -0400 Subject: [PATCH 138/157] Update security_group_tests.cpp --- unittests/security_group_tests.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unittests/security_group_tests.cpp b/unittests/security_group_tests.cpp index 3072c83c963..4aeb98df909 100644 --- a/unittests/security_group_tests.cpp +++ b/unittests/security_group_tests.cpp @@ -422,7 +422,8 @@ BOOST_AUTO_TEST_CASE(test_participants_change_modified) { BOOST_AUTO_TEST_CASE(test_participants_change_2_chains) { /* Note: - * because produce_block calls start_block, and push_block does not, chain2 will trail behind chain + * because produce_block calls start_block after finalizing the produced block, + * and push_block does not, chain2 will trail behind chain */ eosio::testing::tester chain; eosio::testing::tester chain2; From 40adba0a4104b890ea9e364d0821d548e44ae22b Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 22 Apr 2021 20:55:48 -0500 Subject: [PATCH 139/157] Finished Test Case #1. --- tests/CMakeLists.txt | 2 +- tests/SecurityGroup.py | 3 +- tests/privacy_startup_network.py | 93 ++++++++++---------------------- 3 files changed, 31 insertions(+), 67 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 61a7c9c84ab..42bf6f3b8ef 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -119,7 +119,7 @@ add_test(NAME light_validation_sync_test COMMAND tests/light_validation_sync_tes set_property(TEST light_validation_sync_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME eosio_blocklog_prune_test COMMAND tests/eosio_blocklog_prune_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST eosio_blocklog_prune_test PROPERTY LABELS nonparallelizable_tests) -add_test(NAME privacy_startup_network COMMAND tests/privacy_startup_network.py -p 1 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME privacy_startup_network COMMAND tests/privacy_startup_network.py -p 2 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST privacy_startup_network PROPERTY LABELS nonparallelizable_tests) add_test(NAME privacy_simple_network COMMAND tests/privacy_simple_network.py -p 2 -n 3 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST privacy_simple_network PROPERTY LABELS nonparallelizable_tests) diff --git a/tests/SecurityGroup.py b/tests/SecurityGroup.py index bac5a24ab1c..3f8d02d33a6 100644 --- a/tests/SecurityGroup.py +++ b/tests/SecurityGroup.py @@ -15,7 +15,8 @@ def __init__(self, nonParticipants, contractAccount, defaultNode=None, minAddRem self.participants = [] self.contractAccount = contractAccount assert len(nonParticipants) > 0 - self.nonParticipants = copy.copy(nonParticipants) + # copy over all the running processes + self.nonParticipants = [x for x in nonParticipants if x.pid] if Utils.Debug: Utils.Print("Creating SecurityGroup with the following nonParticipants: []".format(SecurityGroup.createAction(self.nonParticipants))) self.defaultNode = defaultNode if defaultNode else nonParticipants[0] self.publishProcessNum = minAddRemEntriesToPublish diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index d2f676b8822..1554e568e4c 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -9,6 +9,7 @@ from Node import ReturnType from TestHelper import TestHelper +import copy import decimal import re import signal @@ -36,8 +37,10 @@ apiNodes=2 # minimum number of apiNodes that will be used in this test minTotalNodes=pnodes+relayNodes+apiNodes totalNodes=args.n if args.n >= minTotalNodes else minTotalNodes -if totalNodes > minTotalNodes: +if totalNodes >= minTotalNodes: apiNodes += totalNodes - minTotalNodes +else: + Utils.Print("Requested {} total nodes, but since the minumum number of API nodes is {}, there will be {} total nodes".format(args.n, apiNodes, totalNodes)) Utils.Debug=args.v dumpErrorDetails=args.dump_error_details @@ -74,8 +77,12 @@ apiNodeNums = [x for x in range(firstApiNodeNum, totalNodes)] for producerNum in range(pnodes): pairedRelayNodeNum = pnodes + producerNum + # p2p connection between producer and relay topo[producerNum] = [pairedRelayNodeNum] - topo[pairedRelayNodeNum] = apiNodeNums + # p2p connections between relays + topo[pairedRelayNodeNum] = [x + producerNum for x in range(pnodes) if x != producerNum] + # p2p connections between relay and all api nodes + topo[pairedRelayNodeNum].extend(apiNodeNums) Utils.Print("topo: {}".format(json.dumps(topo, indent=4, sort_keys=True))) # adjust prodCount to ensure that lib trails more than 1 block behind head @@ -94,69 +101,25 @@ relays = [cluster.getNode(pnodes + x) for x in range(pnodes) ] apiNodes = [cluster.getNode(x) for x in apiNodeNums] - def createAccount(newAcc): - producers[0].createInitializeAccount(newAcc, cluster.eosioAccount) - ignWallet = cluster.walletMgr.create("ignition") # will actually just look up the wallet - cluster.walletMgr.importKey(newAcc, ignWallet) - - numAccounts = 4 - testAccounts = Cluster.createAccountKeys(numAccounts) - accountPrefix = "testaccount" - for i in range(numAccounts): - testAccount = testAccounts[i] - testAccount.name = accountPrefix + str(i + 1) - createAccount(testAccount) - - blockProducer = None - - def verifyInSync(producerNum): - Utils.Print("Ensure all nodes are in-sync") - lib = producers[producerNum].getInfo()["last_irreversible_block_num"] - headBlockNum = producers[producerNum].getBlockNum() - headBlock = producers[producerNum].getBlock(headBlockNum) - global blockProducer - if blockProducer is None: - blockProducer = headBlock["producer"] - Utils.Print("headBlock: {}".format(json.dumps(headBlock, indent=4, sort_keys=True))) - headBlockId = headBlock["id"] - for prod in producers: - if prod == producers[producerNum]: - continue - - assert prod.waitForBlock(headBlockNum, timeout = 10, reportInterval = 1) != None, "Producer node failed to get block number {}".format(headBlockNum) - prod.getBlock(headBlockId) # if it isn't there it will throw an exception - assert prod.waitForBlock(lib, blockType=BlockType.lib), \ - "Producer node is failing to advance its lib ({}) with producer {} ({})".format(node.getInfo()["last_irreversible_block_num"], producerNum, lib) - for node in apiNodes: - assert node.waitForBlock(headBlockNum, timeout = 10, reportInterval = 1) != None, "API node failed to get block number {}".format(headBlockNum) - node.getBlock(headBlockId) # if it isn't there it will throw an exception - assert node.waitForBlock(lib, blockType=BlockType.lib), \ - "API node is failing to advance its lib ({}) with producer {} ({})".format(node.getInfo()["last_irreversible_block_num"], producerNum, lib) - - Utils.Print("Ensure all nodes are in-sync") - assert node.waitForBlock(lib + 1, blockType=BlockType.lib, reportInterval = 1) != None, "Producer node failed to advance lib ahead one block to: {}".format(lib + 1) - - verifyInSync(producerNum=0) - - featureDict = producers[0].getSupportedProtocolFeatureDict() - Utils.Print("feature dict: {}".format(json.dumps(featureDict, indent=4, sort_keys=True))) - - Utils.Print("act feature dict: {}".format(json.dumps(producers[0].getActivatedProtocolFeatures(), indent=4, sort_keys=True))) - timeout = ( pnodes * 12 / 2 ) * 2 # (number of producers * blocks produced / 0.5 blocks per second) * 2 rounds - producers[0].waitUntilBeginningOfProdTurn(blockProducer, timeout=timeout) - feature = "SECURITY_GROUP" - producers[0].activateFeatures([feature]) - assert producers[0].containsFeatures([feature]), "{} feature was not activated".format(feature) - - if sanityTest: - testSuccessful=True - exit(0) - - def publishContract(account, wasmFile, waitForTransBlock=False): - Print("Publish contract") - return producers[0].publishContract(account, "unittests/test-contracts/security_group_test/", wasmFile, abiFile=None, waitForTransBlock=waitForTransBlock) - - publishContract(testAccounts[0], 'security_group_test.wasm', waitForTransBlock=True) + securityGroup = cluster.getSecurityGroup() + cluster.reportInfo() + + Utils.Print("Add all producers and relay nodes to security group") + prodsAndRelays = copy.copy(producers) + prodsAndRelays.extend(relays) + securityGroup.editSecurityGroup(prodsAndRelays) + securityGroup.verifySecurityGroup() + + allButLastApiNodes = apiNodes[:-1] + lastApiNode = [apiNodes[-1]] + + Utils.Print("Add all but last API node and verify they receive blocks and the last API node does not") + securityGroup.editSecurityGroup(addNodes=allButLastApiNodes) + securityGroup.verifySecurityGroup() + + Utils.Print("Add the last API node and verify it receives blocks") + securityGroup.editSecurityGroup(addNodes=lastApiNode) + securityGroup.verifySecurityGroup() testSuccessful=True finally: From d85c4e961d85ad620c0adbaaaae3d5fdaf88b776 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 23 Apr 2021 08:59:16 -0500 Subject: [PATCH 140/157] Fixed the comment test descriptions. --- tests/privacy_simple_network.py | 9 ++++++--- tests/privacy_startup_network.py | 7 +++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 3d7d198aa35..326a326b489 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -17,10 +17,13 @@ import time ############################################################### -# privacy_startup_network +# privacy_simple_network # -# General test for Privacy to verify TLS connections, and slowly adding participants to the security group and verifying -# how blocks and transactions are sent/not sent. +# Implements Privacy Test Case #2 (and other misc scenarios). It creates a simple network of mesh connected +# producers and non-producer nodes. It adds the producers to the security group and verifies they are in +# sync and the non-producers are not. Then, one by one it adds the non-producing nodes to the security +# group, and verifies that the correct nodes are in sync and the others are not. It also repeatedly changes +# the security group, not letting it finalize, to verify Test Case #2. # ############################################################### diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index 1554e568e4c..9a91df83c2e 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -19,8 +19,11 @@ ############################################################### # privacy_startup_network # -# General test for Privacy to verify TLS connections, and slowly adding participants to the security group and verifying -# how blocks and transactions are sent/not sent. +# Script implements Privacy Test Case #1. It pairs up producers with p2p connections with relay nodes +# and the relay nodes connected to at least 2 or more API nodes. The producers and relay nodes are +# added to the security Group and then it validates they are in sync and the api nodes do not receive +# blocks. Then it adds all but one api nodes and verifies they are in sync with producers, then all +# nodes are added and verifies that all nodes are in sync. # ############################################################### From 236cceb89784e0a60267de2fe90dc80d6659061b Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 23 Apr 2021 12:20:00 -0500 Subject: [PATCH 141/157] Added comment to better explain what relay and API nodes are. --- tests/privacy_startup_network.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/privacy_startup_network.py b/tests/privacy_startup_network.py index 9a91df83c2e..3f52c50d451 100755 --- a/tests/privacy_startup_network.py +++ b/tests/privacy_startup_network.py @@ -25,6 +25,12 @@ # blocks. Then it adds all but one api nodes and verifies they are in sync with producers, then all # nodes are added and verifies that all nodes are in sync. # +# NOTE: A relay node is a node that an entity running a producer uses to prevent outside nodes from +# affecting the producing node. An API Node is a node that is setup for the general community to +# connect to and will have more p2p connections. This script doesn't necessarily setup the API nodes +# the way that they are setup in the real world, but it is referencing them this way to explain what +# the test is intending to verify. +# ############################################################### Print=Utils.Print From b4c69616b003d6e65b63191512723e63a8ff2d6a Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 23 Apr 2021 13:07:17 -0500 Subject: [PATCH 142/157] Fixing handling for unstarted nodes. --- tests/SecurityGroup.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/SecurityGroup.py b/tests/SecurityGroup.py index 3f8d02d33a6..df2dad80487 100644 --- a/tests/SecurityGroup.py +++ b/tests/SecurityGroup.py @@ -16,9 +16,16 @@ def __init__(self, nonParticipants, contractAccount, defaultNode=None, minAddRem self.contractAccount = contractAccount assert len(nonParticipants) > 0 # copy over all the running processes - self.nonParticipants = [x for x in nonParticipants if x.pid] + self.nonParticipants = copy.copy(nonParticipants) if Utils.Debug: Utils.Print("Creating SecurityGroup with the following nonParticipants: []".format(SecurityGroup.createAction(self.nonParticipants))) - self.defaultNode = defaultNode if defaultNode else nonParticipants[0] + def findDefault(nodes): + for node in nodes: + if node.pid: + return node + + Utils.errorExit("SecurityGroup is being constructed with no running nodes, there needs to be at least one running node") + + self.defaultNode = defaultNode if defaultNode else findDefault(self.nonParticipants) self.publishProcessNum = minAddRemEntriesToPublish if activateAndPublish: SecurityGroup.activateFeature(self.defaultNode) @@ -111,9 +118,14 @@ def copyIfNeeded(nodes): def verifyParticipantsTransactionFinalized(self, transId): Utils.Print("Verify participants are in sync") assert transId + atLeastOne = False for part in self.participants: + if part.pid is None: + continue + atLeastOne = True if part.waitForTransFinalization(transId) == None: Utils.errorExit("Transaction: {}, never finalized".format(trans)) + assert atLeastOne, "None of the participants are currently running, no reason to call verifyParticipantsTransactionFinalized" # verify that the block for the transaction ID is never finalized in nonParticipants def verifyNonParticipants(self, transId): @@ -131,11 +143,16 @@ def verifyNonParticipants(self, transId): # verify each nonParticipant in the list has not advanced its lib to the publish block, since the block that would cause it to become finalized would # never have been forwarded to a nonParticipant for nonParticipant in self.nonParticipants: + if nonParticipant.pid is None: + continue nonParticipantPostLIB = nonParticipant.getBlockNum(blockType=BlockType.lib) assert nonParticipantPostLIB < publishBlock, "Participants not in security group should not have advanced LIB to {}, but it has advanced to {}".format(publishBlock, nonParticipantPostLIB) nonParticipantHead = nonParticipant.getBlockNum() assert nonParticipantHead < producerHead, "Participants (that are not producers themselves) should not advance head to {}, but it has advanced to {}".format(producerHead, nonParticipantHead) + def getLatestPublishTransId(self): + publishTransId = Node.getTransId(self.publishTrans) + # verify that the participants' and nonParticipants' nodes are consistent based on the publish transaction def verifySecurityGroup(self, publishTrans = None): if publishTrans is None: From cc4d02f6f09f525ebbe57024cb2a6201daffe96e Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 23 Apr 2021 14:32:14 -0500 Subject: [PATCH 143/157] Fixed returning value. --- tests/SecurityGroup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/SecurityGroup.py b/tests/SecurityGroup.py index df2dad80487..9a5f94a23d1 100644 --- a/tests/SecurityGroup.py +++ b/tests/SecurityGroup.py @@ -151,7 +151,7 @@ def verifyNonParticipants(self, transId): assert nonParticipantHead < producerHead, "Participants (that are not producers themselves) should not advance head to {}, but it has advanced to {}".format(producerHead, nonParticipantHead) def getLatestPublishTransId(self): - publishTransId = Node.getTransId(self.publishTrans) + return Node.getTransId(self.publishTrans) # verify that the participants' and nonParticipants' nodes are consistent based on the publish transaction def verifySecurityGroup(self, publishTrans = None): From aff0b798a1db37e5393dabf0c9d76e2f0c097180 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 26 Apr 2021 13:39:42 -0500 Subject: [PATCH 144/157] Buildkite failure debug statement. --- tests/privacy_simple_network.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 326a326b489..f48167aeb88 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -139,8 +139,11 @@ def is_done(): done = False # keep adding and removing nodes till we are done while not done: + preInfo = securityGroup.defaultNode.getInfo() # waiting for a block change to prevent duplicate publish transactions securityGroup.defaultNode.waitForNextBlock() + postInfo = securityGroup.defaultNode.getInfo() + Utils.Print("Pre-info: \n{}\n\nPost-info: \n{}\n".format(json.dumps(preInfo, indent=4, sort_keys=True), json.dumps(postInfo, indent=4, sort_keys=True))) while not done and len(securityGroup.participants) > pnodes: publishTrans = securityGroup.removeFromSecurityGroup() From a99c8992c1863aa11d5b0205928d40a4fb9063e3 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Mon, 26 Apr 2021 18:56:10 -0500 Subject: [PATCH 145/157] Fixing issue with duplicate transactions getting rejected. --- tests/Node.py | 2 +- tests/privacy_simple_network.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index 3d2bfe715be..a9b740d9923 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -502,7 +502,7 @@ def waitForNextBlock(self, timeout=WaitSpec.default(), blockType=BlockType.head) num=self.getBlockNum(blockType=blockType) if isinstance(timeout, WaitSpec): timeout.convert(num, num+1) - lam = lambda: self.getHeadBlockNum() > num + lam = lambda: self.getBlockNum(blockType=blockType) > num ret=Utils.waitForTruth(lam, timeout) return ret diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index f48167aeb88..1ee7391865c 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -132,6 +132,7 @@ # alternate adding/removing participants to ensure the security group doesn't change initialBlockNum = None blockNum = None + lib = None def is_done(): # want to ensure that we can identify the range of libs the security group was changed in return blockNum - initialBlockNum > 12 @@ -139,11 +140,11 @@ def is_done(): done = False # keep adding and removing nodes till we are done while not done: - preInfo = securityGroup.defaultNode.getInfo() - # waiting for a block change to prevent duplicate publish transactions - securityGroup.defaultNode.waitForNextBlock() - postInfo = securityGroup.defaultNode.getInfo() - Utils.Print("Pre-info: \n{}\n\nPost-info: \n{}\n".format(json.dumps(preInfo, indent=4, sort_keys=True), json.dumps(postInfo, indent=4, sort_keys=True))) + currentLib = securityGroup.defaultNode.getBlockNum(blockType=BlockType.lib) + if lib and currentLib != lib: + # waiting for lib block change to prevent duplicate publish transactions + securityGroup.defaultNode.waitForBlock(currentLib + 1, blockType=BlockType.lib) + lib = currentLib while not done and len(securityGroup.participants) > pnodes: publishTrans = securityGroup.removeFromSecurityGroup() From c7941bea979b40acb1f5d9961e93744007961439 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 27 Apr 2021 12:04:06 -0500 Subject: [PATCH 146/157] Fixed waitForNextBlock to work for BlockType.lib and also reusing waitForBlock which has more diagnostics. --- tests/Node.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index a9b740d9923..7417f8d86d3 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -498,15 +498,11 @@ def waitForTransFinalization(self, transId, timeout=None): ret=Utils.waitForTruth(lam, timeout) return ret - def waitForNextBlock(self, timeout=WaitSpec.default(), blockType=BlockType.head): + def waitForNextBlock(self, timeout=WaitSpec.default(), blockType=BlockType.head, sleepTime=3): num=self.getBlockNum(blockType=blockType) - if isinstance(timeout, WaitSpec): - timeout.convert(num, num+1) - lam = lambda: self.getBlockNum(blockType=blockType) > num - ret=Utils.waitForTruth(lam, timeout) - return ret + return self.waitForBlock(num+1, timeout=timeout, blockType=blockType, sleepTime=sleepTime) - def waitForBlock(self, blockNum, timeout=WaitSpec.default(), blockType=BlockType.head, reportInterval=None, errorContext=None): + def waitForBlock(self, blockNum, timeout=WaitSpec.default(), blockType=BlockType.head, sleepTime=3, reportInterval=None, errorContext=None): currentBlockNum=self.getBlockNum(blockType=blockType) currentTime=time.time() if isinstance(timeout, WaitSpec): From bf048e2f8d6e99fef227b2e201158273dfab8fbe Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 27 Apr 2021 12:05:30 -0500 Subject: [PATCH 147/157] Changed the test to have a minimum of 4 nodes and fixed duplicate publish transaction issue. --- tests/privacy_simple_network.py | 109 +++++++++++++++++--------------- 1 file changed, 57 insertions(+), 52 deletions(-) diff --git a/tests/privacy_simple_network.py b/tests/privacy_simple_network.py index 1ee7391865c..e9f161ff0fd 100755 --- a/tests/privacy_simple_network.py +++ b/tests/privacy_simple_network.py @@ -36,7 +36,7 @@ ,"--wallet-port"}) port=args.port pnodes=args.p -apiNodes=0 # minimum number of apiNodes that will be used in this test +apiNodes=1 # minimum number of apiNodes that will be used in this test # bios is also treated as an API node, but doesn't count against totalnodes count minTotalNodes=pnodes+apiNodes totalNodes=args.n if args.n >= minTotalNodes else minTotalNodes @@ -117,64 +117,69 @@ cluster.reportInfo() - # if we have more than 1 api node, we will add and remove all those nodes in bulk, if not it is just a repeat of the above test - if len(apiNodes) > 1: - Utils.Print("Add all api nodes to security group at the same time") - securityGroup.editSecurityGroup(addNodes=securityGroup.nonParticipants) - securityGroup.verifySecurityGroup() + Utils.Print("Add all api nodes to security group at the same time") + securityGroup.editSecurityGroup(addNodes=securityGroup.nonParticipants) + securityGroup.verifySecurityGroup() - cluster.reportInfo() + cluster.reportInfo() - # waiting for a block to change (2 blocks since transaction indication could be 1 behind) to prevent duplicate remove transactions - removeBlockNum = Node.getTransBlockNum(removeTrans) - securityGroup.defaultNode.waitForBlock(removeBlockNum + 2) - - # alternate adding/removing participants to ensure the security group doesn't change - initialBlockNum = None - blockNum = None - lib = None - def is_done(): - # want to ensure that we can identify the range of libs the security group was changed in - return blockNum - initialBlockNum > 12 - - done = False - # keep adding and removing nodes till we are done - while not done: - currentLib = securityGroup.defaultNode.getBlockNum(blockType=BlockType.lib) - if lib and currentLib != lib: - # waiting for lib block change to prevent duplicate publish transactions - securityGroup.defaultNode.waitForBlock(currentLib + 1, blockType=BlockType.lib) - lib = currentLib - - while not done and len(securityGroup.participants) > pnodes: - publishTrans = securityGroup.removeFromSecurityGroup() - blockNum = Node.getTransBlockNum(publishTrans) - if initialBlockNum is None: - initialBlockNum = blockNum - done = is_done() - - while not done and len(securityGroup.nonParticipants) > 0: - publishTrans = securityGroup.moveToSecurityGroup() - blockNum = Node.getTransBlockNum(publishTrans) - done = is_done() - - Utils.Print("First adjustment to security group was in block num: {}, verifying no changes till block num: {} is finalized".format(initialBlockNum, blockNum)) - securityGroup.verifySecurityGroup() + # waiting for a block to change (2 blocks since transaction indication could be 1 behind) to prevent duplicate remove transactions + removeBlockNum = Node.getTransBlockNum(removeTrans) + securityGroup.defaultNode.waitForBlock(removeBlockNum + 2) - cluster.reportInfo() + # alternate adding/removing participants to ensure the security group doesn't change + initialBlockNum = None + blockNums = [] + lib = None - if len(securityGroup.nonParticipants) > 0: - Utils.Print("Add all remaining non-participants to security group at the same time, so all api nodes can be removed as one group") - securityGroup.editSecurityGroup(addNodes=securityGroup.nonParticipants) - securityGroup.verifySecurityGroup() + # draw out sending each + blocksPerProducer = 12 + # space out each send so that by the time we have removed and added each api node (except one), that we have covered more than a + # full production window + numberOfSends = len(apiNodes) * 2 - 1 + blocksToWait = (blocksPerProducer + numberOfSends - 1) / numberOfSends + 1 - cluster.reportInfo() + def wait(): + securityGroup.defaultNode.waitForBlock(blockNums[-1] + blocksToWait, sleepTime=0.4) - Utils.Print("Remove all api nodes from security group at the same time") - securityGroup.editSecurityGroup(removeNodes=apiNodes) - securityGroup.verifySecurityGroup() + while len(securityGroup.participants) > pnodes: + publishTrans = securityGroup.removeFromSecurityGroup() + blockNums.append(Node.getTransBlockNum(publishTrans)) + if initialBlockNum is None: + initialBlockNum = blockNums[-1] + wait() + + while True: + publishTrans = securityGroup.moveToSecurityGroup() + blockNums.append(Node.getTransBlockNum(publishTrans)) + if len(securityGroup.nonParticipants) > 1: + wait() + else: + break + + Utils.Print("Adjustments to security group were made in block nums: [{}], verifying no changes till block num: {} is finalized".format(", ".join([str(x) for x in blockNums]), blockNums[-1])) + lastBlockNum = blockNums[0] + for blockNum in blockNums[1:]: + # because the transaction's block number is only a possible block number, assume that the second block really was sent in the next block + if blockNum + 1 - lastBlockNum >= blocksPerProducer: + Utils.Print("WARNING: Had a gap of {} blocks between publish actions due to sleep not being exact, so if the security group verification fails " + + "it is likely due to that") + lastBlockNum = blockNum + securityGroup.verifySecurityGroup() - cluster.reportInfo() + cluster.reportInfo() + + Utils.Print("Add all remaining non-participants to security group at the same time, so all api nodes can be removed as one group next") + securityGroup.editSecurityGroup(addNodes=securityGroup.nonParticipants) + securityGroup.verifySecurityGroup() + + cluster.reportInfo() + + Utils.Print("Remove all api nodes from security group at the same time") + securityGroup.editSecurityGroup(removeNodes=apiNodes) + securityGroup.verifySecurityGroup() + + cluster.reportInfo() testSuccessful=True finally: From 1f8bdb5db1d93eb9d5d8db26a391f15964efd079 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 28 Apr 2021 14:34:09 -0500 Subject: [PATCH 148/157] Removing transaction_hook. --- .../eosio/chain/global_property_object.hpp | 22 +++++-------------- libraries/chain/include/eosio/chain/types.hpp | 12 ---------- 2 files changed, 6 insertions(+), 28 deletions(-) diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index e550a35688c..5a97b8474b3 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -62,7 +62,7 @@ namespace eosio { namespace chain { */ class global_property_object : public chainbase::object { - OBJECT_CTOR(global_property_object, (proposed_schedule)(proposed_security_group_participants)(transaction_hooks)) + OBJECT_CTOR(global_property_object, (proposed_schedule)(proposed_security_group_participants)) public: id_type id; @@ -75,7 +75,6 @@ namespace eosio { namespace chain { block_num_type proposed_security_group_block_num = 0; // members that are containers need to be shared_* containers, since this object is stored in a multi-index container shared_set proposed_security_group_participants; - shared_vector transaction_hooks; void initalize_from(const legacy::snapshot_global_property_object_v2& legacy, const chain_id_type& chain_id_val, const kv_database_config& kv_config_val, const wasm_config& wasm_config_val) { @@ -111,11 +110,6 @@ namespace eosio { namespace chain { proposed_security_group_participants.key_comp(), proposed_security_group_participants.get_allocator()}; } - - template - void set_transaction_hooks(Iter begin, Iter end) { - transaction_hooks = {begin, end, transaction_hooks.get_allocator()}; - } }; @@ -141,14 +135,12 @@ namespace eosio { namespace chain { struct extension_v0 { // libstdc++ requires the following two constructors to work. extension_v0(){}; - extension_v0(block_num_type num, flat_set participants, vector trx_hooks) + extension_v0(block_num_type num, flat_set participants) : proposed_security_group_block_num(num) - , proposed_security_group_participants(std::move(participants)) - , transaction_hooks(std::move(trx_hooks)) {} + , proposed_security_group_participants(std::move(participants)) {} block_num_type proposed_security_group_block_num = 0; flat_set proposed_security_group_participants; - vector transaction_hooks; }; // for future extensions, please use the following pattern: @@ -167,8 +159,7 @@ namespace eosio { namespace chain { inline snapshot_global_property_object::extension_t get_gpo_extension(const global_property_object& gpo) { return snapshot_global_property_object::extension_v0{ gpo.proposed_security_group_block_num, - {gpo.proposed_security_group_participants.begin(), gpo.proposed_security_group_participants.end()}, - {gpo.transaction_hooks.begin(), gpo.transaction_hooks.end()}}; + {gpo.proposed_security_group_participants.begin(), gpo.proposed_security_group_participants.end()}}; } inline void set_gpo_extension(global_property_object& gpo, @@ -178,7 +169,6 @@ namespace eosio { namespace chain { gpo.proposed_security_group_block_num = ext.proposed_security_group_block_num; gpo.set_proposed_security_group_participants(ext.proposed_security_group_participants.begin(), ext.proposed_security_group_participants.end()); - gpo.set_transaction_hooks(ext.transaction_hooks.begin(), ext.transaction_hooks.end()); }, extension); } @@ -245,7 +235,7 @@ CHAINBASE_SET_INDEX_TYPE(eosio::chain::dynamic_global_property_object, FC_REFLECT(eosio::chain::global_property_object, (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id)(kv_configuration)(wasm_configuration) - (proposed_security_group_block_num)(proposed_security_group_participants)(transaction_hooks) + (proposed_security_group_block_num)(proposed_security_group_participants) ) FC_REFLECT(eosio::chain::legacy::snapshot_global_property_object_v2, @@ -261,7 +251,7 @@ FC_REFLECT(eosio::chain::legacy::snapshot_global_property_object_v4, ) FC_REFLECT(eosio::chain::snapshot_global_property_object::extension_v0, - (proposed_security_group_block_num)(proposed_security_group_participants)(transaction_hooks) + (proposed_security_group_block_num)(proposed_security_group_participants) ) FC_REFLECT(eosio::chain::snapshot_global_property_object, diff --git a/libraries/chain/include/eosio/chain/types.hpp b/libraries/chain/include/eosio/chain/types.hpp index 39498bfc194..a161853e02c 100644 --- a/libraries/chain/include/eosio/chain/types.hpp +++ b/libraries/chain/include/eosio/chain/types.hpp @@ -330,17 +330,6 @@ namespace eosio { namespace chain { } }; - /** - * Defines a type used for defining transaction hooks - * Transaction hooks are a way for system contracts to inspect / make modifications to the transaction - * at a certain point, for example prior to its execution. - */ - struct transaction_hook{ - uint32_t type; - account_name contract; - action_name action; - }; - /** * Extentions are prefixed with type and are a buffer that can be * interpreted by code that is aware and ignored by unaware code. @@ -489,5 +478,4 @@ namespace chainbase { } } -FC_REFLECT(eosio::chain::transaction_hook, (type)(contract)(action)) FC_REFLECT_EMPTY( eosio::chain::void_t ) From 228a4c99db29a426870a3cf62eec57cfafbfbb64 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 28 Apr 2021 17:38:31 -0500 Subject: [PATCH 149/157] Fixed typo. --- libraries/chain/snapshot.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/chain/snapshot.cpp b/libraries/chain/snapshot.cpp index 302b0cee984..80e3f3f850d 100644 --- a/libraries/chain/snapshot.cpp +++ b/libraries/chain/snapshot.cpp @@ -49,7 +49,7 @@ void variant_snapshot_reader::validate() const { uint64_t got_version = version.as_uint64(); EOS_ASSERT(got_version >= minimum_snapshot_version && got_version<= current_snapshot_version, snapshot_validation_exception, - "Variant snapshot is an unsuppored version. Expected : [${min_version}, ${current}] , Got: ${actual}", + "Variant snapshot is an unsupported version. Expected : [${min_version}, ${current}] , Got: ${actual}", ("min_version", minimum_snapshot_version) ("current", current_snapshot_version)("actual",got_version)); EOS_ASSERT(o.contains("sections"), snapshot_validation_exception, @@ -221,7 +221,7 @@ void istream_snapshot_reader::validate() const { uint32_t version; snapshot.read((char*)&version, sizeof(version)); EOS_ASSERT(version > 0 && version <= current_snapshot_version, snapshot_exception, - "Binary snapshot is an unsuppored version. version is ${actual} while code supports version(s) [1,${max}]", + "Binary snapshot is an unsupported version. version is ${actual} while code supports version(s) [1,${max}]", ("max", current_snapshot_version)("actual", version)); while (validate_section()) {} From 180082da300810c99e61aaa2253b92eda17082a7 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 28 Apr 2021 17:57:46 -0500 Subject: [PATCH 150/157] Added mutex in security_group_manager to protect it instead of using connections_mtx. --- .../include/eosio/net_plugin/security_group_manager.hpp | 2 ++ plugins/net_plugin/net_plugin.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp b/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp index 79e89f1ab3a..16aebe1bddb 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp @@ -2,6 +2,7 @@ #include #include +#include #include @@ -23,6 +24,7 @@ namespace eosio { bool is_in_security_group(chain::account_name participant) const { return cache_.empty() || (cache_.find(participant) != cache_.end()); } + mutable std::shared_mutex mtx_; private: uint32_t version_ {0}; ///! The security group version participant_list_t cache_; ///! Cache of participants in the security group diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 4d56b9247f7..701e524a49a 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -266,7 +266,7 @@ namespace eosio { bool use_socket_read_watermark = false; /** @} */ - mutable std::shared_mutex connections_mtx; // protects both connections and security_group + mutable std::shared_mutex connections_mtx; std::set< connection_ptr > connections; // todo: switch to a thread safe container to avoid big mutex over complete collection security_group_manager security_group; @@ -884,7 +884,7 @@ namespace eosio { if(my_impl->ssl_enabled) { account_name participant = participant_name_ ? *participant_name_ : account_name{}; - std::lock_guard connection_guard(my_impl->connections_mtx); + std::shared_lock sg_guard(my_impl->security_group.mtx_); bool participating = my_impl->security_group.is_in_security_group(participant); fc_dlog( logger, "[${peer}] participant: [${name}] participating: [${enabled}]", @@ -3657,7 +3657,7 @@ namespace eosio { } } - // called from any thread + // called from application thread void net_plugin_impl::update_security_group(const block_state_ptr& bs) { // update cache // @@ -3669,7 +3669,7 @@ namespace eosio { if(security_group.current_version() == update.version) { return; } - std::lock_guard connection_guard(connections_mtx); + std::lock_guard sg_guard(security_group.mtx_); if(!security_group.update_cache(update.version, update.participants)) { return; } From 01d2ba6ac64ae7980c181b58e65f8cea0d9d68e1 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 29 Apr 2021 11:28:13 -0500 Subject: [PATCH 151/157] Reverted changes for adding mutex for security_group_manager and split up reading and writing locks. --- .../net_plugin/security_group_manager.hpp | 2 -- plugins/net_plugin/net_plugin.cpp | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp b/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp index 16aebe1bddb..79e89f1ab3a 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/security_group_manager.hpp @@ -2,7 +2,6 @@ #include #include -#include #include @@ -24,7 +23,6 @@ namespace eosio { bool is_in_security_group(chain::account_name participant) const { return cache_.empty() || (cache_.find(participant) != cache_.end()); } - mutable std::shared_mutex mtx_; private: uint32_t version_ {0}; ///! The security group version participant_list_t cache_; ///! Cache of participants in the security group diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 701e524a49a..e019d4c66b5 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -266,7 +266,7 @@ namespace eosio { bool use_socket_read_watermark = false; /** @} */ - mutable std::shared_mutex connections_mtx; + mutable std::shared_mutex connections_mtx; // protects both connections and security_group std::set< connection_ptr > connections; // todo: switch to a thread safe container to avoid big mutex over complete collection security_group_manager security_group; @@ -883,9 +883,13 @@ namespace eosio { inline void setup_participant() { if(my_impl->ssl_enabled) { account_name participant = participant_name_ ? *participant_name_ : account_name{}; - - std::shared_lock sg_guard(my_impl->security_group.mtx_); - bool participating = my_impl->security_group.is_in_security_group(participant); + + auto is_participating = [](account_name part) { + std::shared_lock connection_guard(my_impl->connections_mtx); + return my_impl->security_group.is_in_security_group(part); + }; + + const bool participating = is_participating(participant); fc_dlog( logger, "[${peer}] participant: [${name}] participating: [${enabled}]", ("peer", peer_name())("name", participant_name())("enabled", participating)); @@ -3669,11 +3673,14 @@ namespace eosio { if(security_group.current_version() == update.version) { return; } - std::lock_guard sg_guard(security_group.mtx_); - if(!security_group.update_cache(update.version, update.participants)) { - return; + else { + std::lock_guard connection_guard(my_impl->connections_mtx); + if(!security_group.update_cache(update.version, update.participants)) { + return; + } } + std::shared_lock connection_guard(my_impl->connections_mtx); // update connections // fc_dlog( logger, "SecurityGroup changed to version: ${v}", ("v", security_group.current_version()) ); From 4e983758cf68239cdf835d2a26ce2cbde175a09a Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 29 Apr 2021 14:40:32 -0500 Subject: [PATCH 152/157] Peer review cleanup. --- plugins/net_plugin/net_plugin.cpp | 48 ++++++++++++------- plugins/net_plugin/security_group_manager.cpp | 5 ++ 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index e019d4c66b5..e7a9e47d26f 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -898,7 +898,7 @@ namespace eosio { } std::string certificate_subject(ssl::verify_context &ctx) { - X509 *cert = X509_STORE_CTX_get_current_cert(ctx.native_handle()); + X509* cert = X509_STORE_CTX_get_current_cert(ctx.native_handle()); if (!cert) return ""; @@ -913,7 +913,7 @@ namespace eosio { return {subject_str}; } - bool verify_certificate(bool preverified, ssl::verify_context &ctx) { + bool verify_certificate(bool preverified, ssl::verify_context& ctx) { peer_dlog(this, "preverified: [${p}] certificate subject: ${s}", ("p", preverified)("s", certificate_subject(ctx))); //certificate depth means number of certificate issuers verified current certificate //openssl provides those one by one starting from root certificate @@ -926,12 +926,32 @@ namespace eosio { } //return pointer is managed by openssl - X509 *cert = X509_STORE_CTX_get_current_cert(ctx.native_handle()); + X509* cert = X509_STORE_CTX_get_current_cert(ctx.native_handle()); if (!cert) { peer_dlog(this, "X509_STORE_CTX_get_current_cert returned null certificate"); return false; } + extract_participant_name(cert); + + //we keep connection if peer has valid certificate but participant name is not authorized + //however that connection doesn't receive updates + return preverified; + } + + void reset_socket() { + socket.reset_socket(); + if (my_impl->ssl_enabled) { + socket.ssl_socket()->set_verify_callback( + [this](auto bverified, auto& ctx){ + return verify_certificate(bverified, ctx); + }); + } + } + + private: + void extract_participant_name(X509* cert) + { //max subject size is 256 bytes char buf[256]; //return pointer is managed by openssl @@ -949,20 +969,6 @@ namespace eosio { peer_dlog(this, "received unauthorized participant: ${s}", ("s", organization)); } } - - //we keep connection if peer has valid certificate but participant name is not authorized - //however that connection doesn't receive updates - return preverified; - } - - void reset_socket() { - socket.reset_socket(); - if (my_impl->ssl_enabled) { - socket.ssl_socket()->set_verify_callback( - [this](auto bverified, auto& ctx){ - return verify_certificate(bverified, ctx); - }); - } } }; @@ -4064,7 +4070,13 @@ namespace eosio { } //if we have certificate option that TLS must be enabled - if ( options.count("p2p-tls-own-certificate-file") ) { + const bool tls_own_certificate_file = options.count("p2p-tls-own-certificate-file"); + EOS_ASSERT(tls_own_certificate_file == options.count("p2p-tls-private-key-file") && + tls_own_certificate_file == options.count("p2p-tls-security-group-ca-file"), + ssl_incomplete_configuration, + "\"p2p-tls-own-certificate-file\", \"p2p-tls-private-key-file\", and \"p2p-tls-security-group-ca-file\" either all " + "need to be provided or none of them provided."); + if ( tls_own_certificate_file ) { auto certificate = options["p2p-tls-own-certificate-file"].as(); auto pkey = options["p2p-tls-private-key-file"].as(); auto ca_cert = options["p2p-tls-security-group-ca-file"].as(); diff --git a/plugins/net_plugin/security_group_manager.cpp b/plugins/net_plugin/security_group_manager.cpp index 729deb26655..6318257f77b 100644 --- a/plugins/net_plugin/security_group_manager.cpp +++ b/plugins/net_plugin/security_group_manager.cpp @@ -1,9 +1,14 @@ #include +#include + namespace eosio { bool security_group_manager::update_cache(const uint32_t version, const participant_list_t& participant_list) { if(version == version_) return false; + EOS_ASSERT(version == version_ +1, eosio::chain::plugin_exception, + "The active security group version should only increase ever increase by one. Current version: " + "${current}, new version: ${new}", ("current", version_)("new", version)); version_ = version; cache_ = participant_list; return true; From 861832debab1901705ba5895265ba405a35e18b1 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 29 Apr 2021 14:44:12 -0500 Subject: [PATCH 153/157] Reverting change for testing. --- .gitignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 08b6880c0e9..db9cde05559 100644 --- a/.gitignore +++ b/.gitignore @@ -87,6 +87,4 @@ var/lib/node_* *.iws .DS_Store -!*.swagger.* - -ZScaler* \ No newline at end of file +!*.swagger.* \ No newline at end of file From 8cd1d19a90a0a02283b1db1af56f494f2b0f42ec Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 29 Apr 2021 20:53:25 -0500 Subject: [PATCH 154/157] Removed transaction_hook from abieos. --- libraries/abieos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/abieos b/libraries/abieos index ccae7a06439..7ebffd3ad55 160000 --- a/libraries/abieos +++ b/libraries/abieos @@ -1 +1 @@ -Subproject commit ccae7a06439a5cba6140cdecab93eddcaaafe211 +Subproject commit 7ebffd3ad55d7a7fa908ad32130079dbb7d9ee2a From 73ea43f64cbe0366b805b1585d3648d4b315394f Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 29 Apr 2021 20:58:11 -0500 Subject: [PATCH 155/157] Made method name clearer and clean-up. --- plugins/net_plugin/net_plugin.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index e7a9e47d26f..2f23b08ca87 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -913,7 +913,7 @@ namespace eosio { return {subject_str}; } - bool verify_certificate(bool preverified, ssl::verify_context& ctx) { + bool process_certificate(bool preverified, ssl::verify_context& ctx) { peer_dlog(this, "preverified: [${p}] certificate subject: ${s}", ("p", preverified)("s", certificate_subject(ctx))); //certificate depth means number of certificate issuers verified current certificate //openssl provides those one by one starting from root certificate @@ -944,7 +944,7 @@ namespace eosio { if (my_impl->ssl_enabled) { socket.ssl_socket()->set_verify_callback( [this](auto bverified, auto& ctx){ - return verify_certificate(bverified, ctx); + return process_certificate(bverified, ctx); }); } } @@ -3847,7 +3847,6 @@ namespace eosio { ssl_context->set_options(ssl::context::default_workarounds | ssl::context::no_sslv2 | ssl::context::no_sslv3 ); - ssl_context->set_password_callback([](auto,auto){ return "test"; }); error_code ec; From 4258bdfc78c9ef5a3ec7742775f394e5395913d3 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 29 Apr 2021 21:18:54 -0500 Subject: [PATCH 156/157] Corrected error handling for TLS configuration parameters. --- plugins/net_plugin/net_plugin.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 2f23b08ca87..78f1e11e68f 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -4070,11 +4070,13 @@ namespace eosio { //if we have certificate option that TLS must be enabled const bool tls_own_certificate_file = options.count("p2p-tls-own-certificate-file"); - EOS_ASSERT(tls_own_certificate_file == options.count("p2p-tls-private-key-file") && - tls_own_certificate_file == options.count("p2p-tls-security-group-ca-file"), + EOS_ASSERT(tls_own_certificate_file == options.count("p2p-tls-private-key-file"), ssl_incomplete_configuration, - "\"p2p-tls-own-certificate-file\", \"p2p-tls-private-key-file\", and \"p2p-tls-security-group-ca-file\" either all " - "need to be provided or none of them provided."); + "\"p2p-tls-own-certificate-file\" and \"p2p-tls-private-key-file\" either both need to be provided " + "or neither of them provided."); + EOS_ASSERT(!options.count("p2p-tls-security-group-ca-file") || tls_own_certificate_file, + ssl_incomplete_configuration, + "\"p2p-tls-security-group-ca-file\" cannot be provided without \"p2p-tls-own-certificate-file\" and \"p2p-tls-private-key-file\"."); if ( tls_own_certificate_file ) { auto certificate = options["p2p-tls-own-certificate-file"].as(); auto pkey = options["p2p-tls-private-key-file"].as(); From dd3a99b0d08a4bacd6ce7e1459f4a1f60592ea06 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 29 Apr 2021 21:23:49 -0500 Subject: [PATCH 157/157] More PR comment fixes. --- .../chain/include/eosio/chain/global_property_object.hpp | 6 +++--- plugins/net_plugin/security_group_manager.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index 5a97b8474b3..524a60ecbf9 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -162,8 +162,8 @@ namespace eosio { namespace chain { {gpo.proposed_security_group_participants.begin(), gpo.proposed_security_group_participants.end()}}; } - inline void set_gpo_extension(global_property_object& gpo, - snapshot_global_property_object::extension_t&& extension) { + inline void set_gpo_extension(global_property_object& gpo, + const snapshot_global_property_object::extension_t& extension) { std::visit( [&gpo](auto& ext) { gpo.proposed_security_group_block_num = ext.proposed_security_group_block_num; @@ -199,7 +199,7 @@ namespace eosio { namespace chain { value.chain_id = row.chain_id; value.kv_configuration = row.kv_configuration; value.wasm_configuration = row.wasm_configuration; - set_gpo_extension(value, std::move(row.extension)); + set_gpo_extension(value, row.extension); } }; } diff --git a/plugins/net_plugin/security_group_manager.cpp b/plugins/net_plugin/security_group_manager.cpp index 6318257f77b..30dbcc28328 100644 --- a/plugins/net_plugin/security_group_manager.cpp +++ b/plugins/net_plugin/security_group_manager.cpp @@ -7,7 +7,7 @@ namespace eosio { if(version == version_) return false; EOS_ASSERT(version == version_ +1, eosio::chain::plugin_exception, - "The active security group version should only increase ever increase by one. Current version: " + "The active security group version should only ever increase by one. Current version: " "${current}, new version: ${new}", ("current", version_)("new", version)); version_ = version; cache_ = participant_list;