From d76e8857841872276a990550a922a697a6ac4f2e Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Fri, 11 Jun 2021 10:58:18 -0400 Subject: [PATCH 1/2] feature privacy snapshot test --- .../tests/postgres_backend_tests.cpp | 1 + tests/CMakeLists.txt | 3 + tests/SecurityGroup.py | 13 +- tests/privacy_network_from_snapshot.py | 119 ++++++++++++++++++ 4 files changed, 133 insertions(+), 3 deletions(-) create mode 100755 tests/privacy_network_from_snapshot.py diff --git a/plugins/blockvault_client_plugin/tests/postgres_backend_tests.cpp b/plugins/blockvault_client_plugin/tests/postgres_backend_tests.cpp index 98829c763c5..a3b55d6d99d 100644 --- a/plugins/blockvault_client_plugin/tests/postgres_backend_tests.cpp +++ b/plugins/blockvault_client_plugin/tests/postgres_backend_tests.cpp @@ -1,5 +1,6 @@ #include "../postgres_backend.hpp" #include +#include #include #include diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7668751ab4f..8b3b8b1d44f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -73,6 +73,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_config_test_activate.py ${CMA configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_config_test_restart.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_config_test_restart.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_config_test_snapshot.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_config_test_snapshot.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_config_test_no_ca.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_config_test_no_ca.py COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_network_from_snapshot.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_network_from_snapshot.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/read_only_query_tests.py ${CMAKE_CURRENT_BINARY_DIR}/read_only_query_tests.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cleos_action_no_params.py ${CMAKE_CURRENT_BINARY_DIR}/cleos_action_no_params.py COPYONLY) @@ -149,6 +150,8 @@ add_test(NAME privacy_config_test_snapshot COMMAND tests/privacy_config_test_sna set_property(TEST privacy_config_test_snapshot PROPERTY LABELS nonparallelizable_tests) add_test(NAME privacy_config_test_no_ca COMMAND tests/privacy_config_test_no_ca.py -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST privacy_config_test_no_ca PROPERTY LABELS nonparallelizable_tests) +add_test(NAME privacy_network_from_snapshot COMMAND tests/privacy_network_from_snapshot.py -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST privacy_network_from_snapshot PROPERTY LABELS nonparallelizable_tests) add_test(NAME read_only_query COMMAND tests/read_only_query_tests.py -p 1 -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST read_only_query PROPERTY LABELS nonparallelizable_tests) add_test(NAME cleos_action_no_params COMMAND tests/cleos_action_no_params.py WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/tests/SecurityGroup.py b/tests/SecurityGroup.py index c78127bc4f0..c79b5ce32be 100644 --- a/tests/SecurityGroup.py +++ b/tests/SecurityGroup.py @@ -78,11 +78,14 @@ def __remParticipants(self, nodes): self.nonParticipants.extend(nodes) + @staticmethod + def toString(nodes): + return "[[{}]]".format(','.join(['"{}"'.format(node.getParticipant()) for node in nodes])) + # create the action payload for an add or remove action @staticmethod def createAction(nodes): - return None if len(nodes) == 0 else \ - "[[{}]]".format(','.join(['"{}"'.format(node.getParticipant()) for node in nodes])) + return None if len(nodes) == 0 else SecurityGroup.toString(nodes) # sends actions to add/remove the provided nodes to/from the network's security group def editSecurityGroup(self, addNodes=[], removeNodes=[], node=None): @@ -134,9 +137,10 @@ def verifyParticipantsTransactionFinalized(self, transId): if part.pid is None: continue if part.waitForTransFinalization(transId) == None: - Utils.errorExit("Transaction: {}, never finalized".format(trans)) + Utils.errorExit("Transaction: {}, never finalized".format(transId)) headAtTransFinalized = part.getBlockNum() assert headAtTransFinalized, "None of the participants are currently running, no reason to call verifyParticipantsTransactionFinalized" + Utils.Print("Head when transaction been finalized: {}".format(headAtTransFinalized)) return headAtTransFinalized # verify that the block for the transaction ID is never finalized in nonParticipants @@ -161,9 +165,12 @@ def verifyNonParticipants(self, transId, headAtTransFinalization): if nonParticipant.pid is None: continue nonParticipantPostLIB = nonParticipant.getBlockNum(blockType=BlockType.lib) + Utils.Print("node {} lib = {}".format(nonParticipant.nodeId, nonParticipantPostLIB)) assert nonParticipantPostLIB < publishBlock, "Participants not in security group should not have advanced LIB to {}, but it has advanced to {}".format(publishBlock, nonParticipantPostLIB) nonParticipantHead = nonParticipant.getBlockNum() + Utils.Print("node {} head = {}".format(nonParticipant.nodeId, nonParticipantHead)) if nonParticipantHead > headAtTransFinalization: + Utils.Print("non-participant head is beyond transaction head, checking that producer is out of participating group") producer = nonParticipant.getBlockProducerByNum(nonParticipantHead) assert producer in expectedProducers, \ "Participants should not advance head to {} unless they are producing their own blocks. It has advanced to {} with producer {}, \ diff --git a/tests/privacy_network_from_snapshot.py b/tests/privacy_network_from_snapshot.py new file mode 100755 index 00000000000..1a2fa8a7333 --- /dev/null +++ b/tests/privacy_network_from_snapshot.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 + +from SecurityGroup import SecurityGroup +from testUtils import Utils +from Cluster import Cluster +from WalletMgr import WalletMgr +from TestHelper import TestHelper + +import time +import signal +import json +import os +import shutil + +############################################################### +# privacy_network_from_snapshot +# +# This test ensures that security group cluster can be restarte from snapshot +# +############################################################### + +def cleanNodeData(nodes): + for node in nodes: + dataDir = Utils.getNodeDataDir(node.nodeId) + state = os.path.join(dataDir, "state") + # removing state dir + shutil.rmtree(state, ignore_errors=True) + + existingBlocksDir = os.path.join(dataDir, "blocks") + # removing blocks dir + shutil.rmtree(existingBlocksDir, ignore_errors=True) + +Print=Utils.Print + +args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v"}) + +pnodes=2 +totalNodes=4 +dumpErrorDetails=args.dump_error_details +keepLogs=args.keep_logs +Utils.Debug=args.v + +testSuccessful=False +cluster=Cluster(host=TestHelper.LOCAL_HOST, port=TestHelper.DEFAULT_PORT, walletd=True, walletMgr=WalletMgr(True)) +try: + TestHelper.printSystemInfo("BEGIN") + cluster.killall(allInstances=True, cleanup=True) + + # we need stale production enabled for producers that are participants in order to produce blocks after restart from snapshot + # node 3 have irreversible read mode in order to save snapshot after being disconnected from security group. + # without that flag node waits till next lib to arrive + specificExtraNodeosArgs = { 0 : "--enable-stale-production", + 1 : "--enable-stale-production", + 3 : "--read-mode irreversible" } + if not cluster.launch(pnodes=pnodes, + totalNodes=totalNodes, + specificExtraNodeosArgs=specificExtraNodeosArgs, + configSecurityGroup=True, + printInfo=True): + Utils.cmdError("launcher") + Utils.errorExit("Failed to stand up eos cluster.") + + Print("Validating system accounts after bootstrap") + cluster.validateAccounts(None) + + cluster.biosNode.kill(signal.SIGTERM) + + # security group consist of 2 producers and 1 api node + secGroupNodes = [cluster.getNode(0), cluster.getNode(1), cluster.getNode(2)] + nonSecGroupNodes = [cluster.getNode(3)] + + securityGroup = cluster.getSecurityGroup() + + trans = securityGroup.editSecurityGroup(addNodes=secGroupNodes) + trans_block_num = secGroupNodes[0].getBlockNumByTransId(trans["transaction_id"], blocksAhead=3) + assert trans_block_num is not None + Print("security group update {} published on block {}".format(SecurityGroup.toString(secGroupNodes), trans_block_num)) + + # this method will wait till transaction become lib on all participants and check that non-participants go out of sync + securityGroup.verifySecurityGroup(publishTrans=trans) + + # snapshot from participant node + snapshot1 = secGroupNodes[0].createSnapshot() + snapshot1_path = snapshot1["snapshot_name"] + Print("snapshot data: {}".format(json.dumps(snapshot1, indent=4, sort_keys=True))) + + #snapshot from non-participant node with lib < security group transaction update + snapshot2 = nonSecGroupNodes[0].createSnapshot() + snapshot2_path = snapshot2["snapshot_name"] + Print("snapshot data: {}".format(json.dumps(snapshot2, indent=4, sort_keys=True))) + + # kill all + cluster.killSomeEosInstances(killCount=len(cluster.getNodes())) + + # we have two snapshots: one with blocks past security group update and another with blocks till this update + chainArg1 = "--snapshot {}".format(snapshot1_path) + chainArg2 = "--snapshot {}".format(snapshot2_path) + # clear data for all + cleanNodeData(cluster.getNodes()) + # relaunch all nodes from corresponding snapshots + for node in secGroupNodes: + node.relaunch(chainArg=chainArg1, cachePopen=True) + for node in nonSecGroupNodes: + node.relaunch(chainArg=chainArg2, cachePopen=True) + + # wait for lib to move + secGroupNodes[0].waitForLibToAdvance() + + # verify security group nodes are in sync after lib moved + cluster.verifyInSync(specificNodes=secGroupNodes) + + # verifying non-participants to stay disconnected from security group + for node in nonSecGroupNodes: + Print("node {} head block is {}".format(node.nodeId, node.getHeadBlockNum())) + assert node.getHeadBlockNum() < trans_block_num, "node {} must not advance past {} block as it is not in security group".format(node.nodeId, trans_block_num) + + testSuccessful=True +finally: + TestHelper.shutdown(cluster, cluster.walletMgr, testSuccessful, True, True, keepLogs, True, dumpErrorDetails) \ No newline at end of file From 92ba6750dee19cbae60ddfaabe7840f5ee9bc9ab Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Fri, 11 Jun 2021 20:00:36 -0400 Subject: [PATCH 2/2] feature privacy snapshot test enhancements --- tests/SecurityGroup.py | 1 + tests/privacy_network_from_snapshot.py | 50 ++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/tests/SecurityGroup.py b/tests/SecurityGroup.py index c79b5ce32be..5ae9a381312 100644 --- a/tests/SecurityGroup.py +++ b/tests/SecurityGroup.py @@ -136,6 +136,7 @@ def verifyParticipantsTransactionFinalized(self, transId): for part in self.participants: if part.pid is None: continue + Utils.Print("Checking node {}".format(part.nodeId)) if part.waitForTransFinalization(transId) == None: Utils.errorExit("Transaction: {}, never finalized".format(transId)) headAtTransFinalized = part.getBlockNum() diff --git a/tests/privacy_network_from_snapshot.py b/tests/privacy_network_from_snapshot.py index 1a2fa8a7333..b8cc5a2240c 100755 --- a/tests/privacy_network_from_snapshot.py +++ b/tests/privacy_network_from_snapshot.py @@ -2,11 +2,13 @@ from SecurityGroup import SecurityGroup from testUtils import Utils +from testUtils import WaitSpec from Cluster import Cluster from WalletMgr import WalletMgr from TestHelper import TestHelper +from Node import Node -import time +import re import signal import json import os @@ -51,7 +53,7 @@ def cleanNodeData(nodes): # without that flag node waits till next lib to arrive specificExtraNodeosArgs = { 0 : "--enable-stale-production", 1 : "--enable-stale-production", - 3 : "--read-mode irreversible" } + 3 : "--read-mode irreversible --plugin eosio::net_api_plugin" } if not cluster.launch(pnodes=pnodes, totalNodes=totalNodes, specificExtraNodeosArgs=specificExtraNodeosArgs, @@ -109,11 +111,55 @@ def cleanNodeData(nodes): # verify security group nodes are in sync after lib moved cluster.verifyInSync(specificNodes=secGroupNodes) + securityGroupEndpoints = [] + for node in secGroupNodes: + curEndpoint = node.getListenEndpoint() + curEndpoint.replace("localhost", "0.0.0.0") + Print("node {} p2p-listen-endpoint is {}".format(node.nodeId, curEndpoint)) + securityGroupEndpoints.append(curEndpoint) + # verifying non-participants to stay disconnected from security group for node in nonSecGroupNodes: Print("node {} head block is {}".format(node.nodeId, node.getHeadBlockNum())) assert node.getHeadBlockNum() < trans_block_num, "node {} must not advance past {} block as it is not in security group".format(node.nodeId, trans_block_num) + + # just to be sure check that non-participant does have connection to any node from security group + foundConnection = False + for connection in node.getConnections(): + p2p_address = connection["last_handshake"]["p2p_address"] + match = re.search("([a-z\.0-9]+:[0-9]{2,5}) ", p2p_address) + if match and len(match.groups()): + cur_address = match.groups()[0].replace("localhost", "0.0.0.0") + Print("node {} has connection from {}".format(node.nodeId, cur_address)) + if cur_address in securityGroupEndpoints: + Print("found connection to one of security group nodes, breaking") + foundConnection = True + break + assert foundConnection + + # now let's make a snapshot with adding to group and then provide it to non-participant + trans = securityGroup.editSecurityGroup(addNodes=nonSecGroupNodes) + trans_block_num = secGroupNodes[0].getBlockNumByTransId(trans["transaction_id"], blocksAhead=3) + assert trans_block_num is not None + Print("security group update {} published on block {}".format(SecurityGroup.toString(nonSecGroupNodes), trans_block_num)) + assert secGroupNodes[0].waitForTransFinalization(Node.getTransId(trans), timeout=WaitSpec.default()) + cluster.verifyInSync(specificNodes=secGroupNodes) + + # snapshot from participant node + snapshot3 = secGroupNodes[0].createSnapshot() + snapshot3_path = snapshot1["snapshot_name"] + Print("snapshot data: {}".format(json.dumps(snapshot3, indent=4, sort_keys=True))) + + for node in nonSecGroupNodes: + node.kill(signal.SIGTERM) + cleanNodeData(nonSecGroupNodes) + for node in nonSecGroupNodes: + node.relaunch(cachePopen=True, addSwapFlags={"--snapshot" : snapshot3_path }) + + secGroupNodes.extend(nonSecGroupNodes) + cluster.verifyInSync(specificNodes=secGroupNodes) + testSuccessful=True finally: TestHelper.shutdown(cluster, cluster.walletMgr, testSuccessful, True, True, keepLogs, True, dumpErrorDetails) \ No newline at end of file