feature privacy snapshot test

plugins/blockvault_client_plugin/tests/postgres_backend_tests.cpp

@@ -1,5 +1,6 @@
 #include "../postgres_backend.hpp"
 #include <boost/test/unit_test.hpp>
+#include <boost/mpl/list.hpp>
 #include <fc/filesystem.hpp>
 #include <fc/log/appender.hpp>
 

tests/CMakeLists.txt

@@ -73,6 +73,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_config_test_activate.py ${CMA
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_config_test_restart.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_config_test_restart.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_config_test_snapshot.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_config_test_snapshot.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_config_test_no_ca.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_config_test_no_ca.py COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/privacy_network_from_snapshot.py ${CMAKE_CURRENT_BINARY_DIR}/privacy_network_from_snapshot.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/read_only_query_tests.py ${CMAKE_CURRENT_BINARY_DIR}/read_only_query_tests.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cleos_action_no_params.py ${CMAKE_CURRENT_BINARY_DIR}/cleos_action_no_params.py COPYONLY)
 
@@ -149,6 +150,8 @@ add_test(NAME privacy_config_test_snapshot COMMAND tests/privacy_config_test_sna
 set_property(TEST privacy_config_test_snapshot PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME privacy_config_test_no_ca COMMAND tests/privacy_config_test_no_ca.py -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST privacy_config_test_no_ca PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME privacy_network_from_snapshot COMMAND tests/privacy_network_from_snapshot.py -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST privacy_network_from_snapshot PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME read_only_query COMMAND tests/read_only_query_tests.py -p 1 -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST read_only_query PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME cleos_action_no_params COMMAND tests/cleos_action_no_params.py WORKING_DIRECTORY ${CMAKE_BINARY_DIR})

tests/SecurityGroup.py

@@ -78,11 +78,14 @@ def __remParticipants(self, nodes):
 
         self.nonParticipants.extend(nodes)
 
+    @staticmethod
+    def toString(nodes):
+        return "[[{}]]".format(','.join(['"{}"'.format(node.getParticipant()) for node in nodes]))
+
     # create the action payload for an add or remove action
     @staticmethod
     def createAction(nodes):
-        return None if len(nodes) == 0 else \
-            "[[{}]]".format(','.join(['"{}"'.format(node.getParticipant()) for node in nodes]))
+        return None if len(nodes) == 0 else SecurityGroup.toString(nodes)
 
     # sends actions to add/remove the provided nodes to/from the network's security group
     def editSecurityGroup(self, addNodes=[], removeNodes=[], node=None):
@@ -133,10 +136,12 @@ def verifyParticipantsTransactionFinalized(self, transId):
         for part in self.participants:
             if part.pid is None:
                 continue
+            Utils.Print("Checking node {}".format(part.nodeId))
             if part.waitForTransFinalization(transId) == None:
-                Utils.errorExit("Transaction: {}, never finalized".format(trans))
+                Utils.errorExit("Transaction: {}, never finalized".format(transId))
             headAtTransFinalized = part.getBlockNum()
         assert headAtTransFinalized, "None of the participants are currently running, no reason to call verifyParticipantsTransactionFinalized"
+        Utils.Print("Head when transaction been finalized: {}".format(headAtTransFinalized))
         return headAtTransFinalized
 
     # verify that the block for the transaction ID is never finalized in nonParticipants
@@ -161,9 +166,12 @@ def verifyNonParticipants(self, transId, headAtTransFinalization):
             if nonParticipant.pid is None:
                 continue
             nonParticipantPostLIB = nonParticipant.getBlockNum(blockType=BlockType.lib)
+            Utils.Print("node {} lib = {}".format(nonParticipant.nodeId, nonParticipantPostLIB))
             assert nonParticipantPostLIB < publishBlock, "Participants not in security group should not have advanced LIB to {}, but it has advanced to {}".format(publishBlock, nonParticipantPostLIB)
             nonParticipantHead = nonParticipant.getBlockNum()
+            Utils.Print("node {} head = {}".format(nonParticipant.nodeId, nonParticipantHead))
             if nonParticipantHead > headAtTransFinalization:
+                Utils.Print("non-participant head is beyond transaction head, checking that producer is out of participating group")
                 producer = nonParticipant.getBlockProducerByNum(nonParticipantHead)
                 assert producer in expectedProducers, \
                        "Participants should not advance head to {} unless they are producing their own blocks. It has advanced to {} with producer {}, \

tests/privacy_network_from_snapshot.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+
+from SecurityGroup import SecurityGroup
+from testUtils import Utils
+from testUtils import WaitSpec
+from Cluster import Cluster
+from WalletMgr import WalletMgr
+from TestHelper import TestHelper
+from Node import Node
+
+import re
+import signal
+import json
+import os
+import shutil
+
+###############################################################
+# privacy_network_from_snapshot
+#
+# This test ensures that security group cluster can be restarte from snapshot
+#
+###############################################################
+
+def cleanNodeData(nodes):
+    for node in nodes:
+        dataDir = Utils.getNodeDataDir(node.nodeId)
+        state = os.path.join(dataDir, "state")
+        # removing state dir
+        shutil.rmtree(state, ignore_errors=True)
+
+        existingBlocksDir = os.path.join(dataDir, "blocks")
+        # removing blocks dir
+        shutil.rmtree(existingBlocksDir, ignore_errors=True)
+
+Print=Utils.Print
+
+args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v"})
+
+pnodes=2
+totalNodes=4
+dumpErrorDetails=args.dump_error_details
+keepLogs=args.keep_logs
+Utils.Debug=args.v
+
+testSuccessful=False
+cluster=Cluster(host=TestHelper.LOCAL_HOST, port=TestHelper.DEFAULT_PORT, walletd=True, walletMgr=WalletMgr(True))
+try:
+    TestHelper.printSystemInfo("BEGIN")
+    cluster.killall(allInstances=True, cleanup=True)
+
+    # we need stale production enabled for producers that are participants in order to produce blocks after restart from snapshot
+    # node 3 have irreversible read mode in order to save snapshot after being disconnected from security group.
+    # without that flag node waits till next lib to arrive
+    specificExtraNodeosArgs = { 0 : "--enable-stale-production",
+                                1 : "--enable-stale-production",
+                                3 : "--read-mode irreversible --plugin eosio::net_api_plugin" }
+    if not cluster.launch(pnodes=pnodes, 
+                          totalNodes=totalNodes,
+                          specificExtraNodeosArgs=specificExtraNodeosArgs,
+                          configSecurityGroup=True,
+                          printInfo=True):
+        Utils.cmdError("launcher")
+        Utils.errorExit("Failed to stand up eos cluster.")
+
+    Print("Validating system accounts after bootstrap")
+    cluster.validateAccounts(None)
+
+    cluster.biosNode.kill(signal.SIGTERM)
+
+    # security group consist of 2 producers and 1 api node
+    secGroupNodes = [cluster.getNode(0), cluster.getNode(1), cluster.getNode(2)]
+    nonSecGroupNodes = [cluster.getNode(3)]
+
+    securityGroup = cluster.getSecurityGroup()
+
+    trans = securityGroup.editSecurityGroup(addNodes=secGroupNodes)
+    trans_block_num = secGroupNodes[0].getBlockNumByTransId(trans["transaction_id"], blocksAhead=3)
+    assert trans_block_num is not None
+    Print("security group update {} published on block {}".format(SecurityGroup.toString(secGroupNodes), trans_block_num))
+
+    # this method will wait till transaction become lib on all participants and check that non-participants go out of sync
+    securityGroup.verifySecurityGroup(publishTrans=trans)
+
+    # snapshot from participant node
+    snapshot1 = secGroupNodes[0].createSnapshot()
+    snapshot1_path = snapshot1["snapshot_name"]
+    Print("snapshot data: {}".format(json.dumps(snapshot1, indent=4, sort_keys=True)))
+
+    #snapshot from non-participant node with lib < security group transaction update
+    snapshot2 = nonSecGroupNodes[0].createSnapshot()
+    snapshot2_path = snapshot2["snapshot_name"]
+    Print("snapshot data: {}".format(json.dumps(snapshot2, indent=4, sort_keys=True)))
+
+    # kill all
+    cluster.killSomeEosInstances(killCount=len(cluster.getNodes()))
+
+    # we have two snapshots: one with blocks past security group update and another with blocks till this update
+    chainArg1 = "--snapshot {}".format(snapshot1_path)
+    chainArg2 = "--snapshot {}".format(snapshot2_path)
+    # clear data for all
+    cleanNodeData(cluster.getNodes())
+    # relaunch all nodes from corresponding snapshots
+    for node in secGroupNodes:
+        node.relaunch(chainArg=chainArg1, cachePopen=True)
+    for node in nonSecGroupNodes:
+        node.relaunch(chainArg=chainArg2, cachePopen=True)
+
+    # wait for lib to move
+    secGroupNodes[0].waitForLibToAdvance()
+
+    # verify security group nodes are in sync after lib moved
+    cluster.verifyInSync(specificNodes=secGroupNodes)
+
+    securityGroupEndpoints = []
+    for node in secGroupNodes:
+        curEndpoint = node.getListenEndpoint()
+        curEndpoint.replace("localhost", "0.0.0.0")
+        Print("node {} p2p-listen-endpoint is {}".format(node.nodeId, curEndpoint))
+        securityGroupEndpoints.append(curEndpoint)
+
+    # verifying non-participants to stay disconnected from security group
+    for node in nonSecGroupNodes:
+        Print("node {} head block is {}".format(node.nodeId, node.getHeadBlockNum()))
+        assert node.getHeadBlockNum() < trans_block_num, "node {} must not advance past {} block as it is not in security group".format(node.nodeId, trans_block_num)
+
+        # just to be sure check that non-participant does have connection to any node from security group
+        foundConnection = False
+        for connection in node.getConnections():
+            p2p_address = connection["last_handshake"]["p2p_address"]
+            match = re.search("([a-z\.0-9]+:[0-9]{2,5}) ", p2p_address)
+            if match and len(match.groups()):
+                cur_address = match.groups()[0].replace("localhost", "0.0.0.0")
+                Print("node {} has connection from {}".format(node.nodeId, cur_address))
+                if cur_address in securityGroupEndpoints:
+                    Print("found connection to one of security group nodes, breaking")
+                    foundConnection = True
+                    break
+        assert foundConnection
+
+    # now let's make a snapshot with adding to group and then provide it to non-participant
+    trans = securityGroup.editSecurityGroup(addNodes=nonSecGroupNodes)
+    trans_block_num = secGroupNodes[0].getBlockNumByTransId(trans["transaction_id"], blocksAhead=3)
+    assert trans_block_num is not None
+    Print("security group update {} published on block {}".format(SecurityGroup.toString(nonSecGroupNodes), trans_block_num))
+
+    assert secGroupNodes[0].waitForTransFinalization(Node.getTransId(trans), timeout=WaitSpec.default())
+    cluster.verifyInSync(specificNodes=secGroupNodes)
+
+    # snapshot from participant node
+    snapshot3 = secGroupNodes[0].createSnapshot()
+    snapshot3_path = snapshot1["snapshot_name"]
+    Print("snapshot data: {}".format(json.dumps(snapshot3, indent=4, sort_keys=True)))
+
+    for node in nonSecGroupNodes:
+        node.kill(signal.SIGTERM)
+    cleanNodeData(nonSecGroupNodes)
+    for node in nonSecGroupNodes:
+        node.relaunch(cachePopen=True, addSwapFlags={"--snapshot" : snapshot3_path })
+
+    secGroupNodes.extend(nonSecGroupNodes)
+    cluster.verifyInSync(specificNodes=secGroupNodes)
+
+    testSuccessful=True
+finally:
+    TestHelper.shutdown(cluster, cluster.walletMgr, testSuccessful, True, True, keepLogs, True, dumpErrorDetails)