Merge pull request #986 from AntelopeIO/GH-980-restore-read-mode-spec…

…ulative-4.0

[4.0] Restore read-mode=speculative

docs/01_nodeos/03_plugins/chain_plugin/index.md

@@ -120,7 +120,7 @@ Config Options for eosio::chain_plugin:
                                         applied to them (may specify multiple
                                         times)
   --read-mode arg (=head)               Database read mode ("head",
-                                        "irreversible").
+                                        "irreversible", "speculative").
                                         In "head" mode: database contains state
                                         changes up to the head block;
                                         transactions received by the node are
@@ -131,7 +131,14 @@ Config Options for eosio::chain_plugin:
                                         received via the P2P network are not
                                         relayed and transactions cannot be
                                         pushed via the chain API.
-
+                                        In "speculative" mode: (DEPRECATED:
+                                        head mode recommended) database
+                                        contains state changes by transactions
+                                        in the blockchain up to the head block
+                                        as well as some transactions not yet
+                                        included in the blockchain;
+                                        transactions received by the node are
+                                        relayed if valid.                                        
   --api-accept-transactions arg (=1)    Allow API transactions to be evaluated
                                         and relayed if valid.
   --validation-mode arg (=full)         Chain validation mode ("full" or

docs/01_nodeos/07_concepts/05_storage-and-read-modes.md

@@ -29,6 +29,7 @@ The `nodeos` service can be run in different "read" modes. These modes control h
 
 - `head`: this only includes the side effects of confirmed transactions, this mode processes unconfirmed transactions but does not include them.
 - `irreversible`: this mode also includes confirmed transactions only up to those included in the last irreversible block.
+- `speculative`: this includes the side effects of confirmed and unconfirmed transactions.
 
 A transaction is considered confirmed when a `nodeos` instance has received, processed, and written it to a block on the blockchain, i.e. it is in the head block or an earlier block.
 
@@ -44,6 +45,16 @@ When `nodeos` is configured to be in irreversible read mode, it will still track
 
 Clients such as `cleos` and the RPC API will see database state as of the current head block of the chain. It **will not** include changes made by transactions known to this node but not included in the chain, such as unconfirmed transactions.
 
+### Speculative Mode ( Deprecated )
+
+Clients such as `cleos` and the RPC API, will see database state as of the current head block plus changes made by all transactions known to this node but potentially not included in the chain, unconfirmed transactions for example.
+
+Speculative mode is low latency but fragile, there is no guarantee that the transactions reflected in the state will be included in the chain OR that they will reflected in the same order the state implies.
+
+This mode features the lowest latency, but is the least consistent.
+
+In speculative mode `nodeos` is able to execute transactions which have TaPoS (Transaction as Proof of Stake) pointing to any valid block in a fork considered to be the best fork by this node.
+
 ## How To Specify the Read Mode
 
 The mode in which `nodeos` is run can be specified using the `--read-mode` option from the `eosio::chain_plugin`.

libraries/chain/controller.cpp

@@ -270,9 +270,7 @@ struct controller_impl {
          prev = fork_db.root();
       }
 
-      if ( read_mode == db_read_mode::HEAD ) {
-         EOS_ASSERT( head->block, block_validate_exception, "attempting to pop a block that was sparsely loaded from a snapshot");
-      }
+      EOS_ASSERT( head->block, block_validate_exception, "attempting to pop a block that was sparsely loaded from a snapshot");
 
       head = prev;
 
@@ -1635,7 +1633,7 @@ struct controller_impl {
             if ( trx->is_transient() ) {
                // remove trx from pending block by not canceling 'restore'
                trx_context.undo(); // this will happen automatically in destructor, but make it more explicit
-            } else if ( pending->_block_status == controller::block_status::ephemeral ) {
+            } else if ( read_mode != db_read_mode::SPECULATIVE && pending->_block_status == controller::block_status::ephemeral ) {
                // An ephemeral block will never become a full block, but on a producer node the trxs should be saved
                // in the un-applied transaction queue for execution during block production. For a non-producer node
                // save the trxs in the un-applied transaction queue for use during block validation to skip signature

libraries/chain/include/eosio/chain/controller.hpp

@@ -49,7 +49,8 @@ namespace eosio { namespace chain {
 
    enum class db_read_mode {
       HEAD,
-      IRREVERSIBLE
+      IRREVERSIBLE,
+      SPECULATIVE
    };
 
    enum class validation_mode {

plugins/chain_plugin/chain_plugin.cpp

@@ -51,6 +51,8 @@ std::ostream& operator<<(std::ostream& osm, eosio::chain::db_read_mode m) {
       osm << "head";
    } else if ( m == eosio::chain::db_read_mode::IRREVERSIBLE ) {
       osm << "irreversible";
+   } else if ( m == eosio::chain::db_read_mode::SPECULATIVE ) {
+      osm << "speculative";
    }
 
    return osm;
@@ -70,10 +72,12 @@ void validate(boost::any& v,
   // one string, it's an error, and exception will be thrown.
   std::string const& s = validators::get_single_string(values);
 
- if ( s == "head" ) {
+  if ( s == "head" ) {
      v = boost::any(eosio::chain::db_read_mode::HEAD);
   } else if ( s == "irreversible" ) {
      v = boost::any(eosio::chain::db_read_mode::IRREVERSIBLE);
+  } else if ( s == "speculative" ) {
+     v = boost::any(eosio::chain::db_read_mode::SPECULATIVE);
   } else {
      throw validation_error(validation_error::invalid_option_value);
   }
@@ -286,10 +290,12 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
          ("sender-bypass-whiteblacklist", boost::program_options::value<vector<string>>()->composing()->multitoken(),
           "Deferred transactions sent by accounts in this list do not have any of the subjective whitelist/blacklist checks applied to them (may specify multiple times)")
          ("read-mode", boost::program_options::value<eosio::chain::db_read_mode>()->default_value(eosio::chain::db_read_mode::HEAD),
-          "Database read mode (\"head\", \"irreversible\").\n"
+          "Database read mode (\"head\", \"irreversible\", \"speculative\").\n"
           "In \"head\" mode: database contains state changes up to the head block; transactions received by the node are relayed if valid.\n"
           "In \"irreversible\" mode: database contains state changes up to the last irreversible block; "
           "transactions received via the P2P network are not relayed and transactions cannot be pushed via the chain API.\n"
+          "In \"speculative\" mode: (DEPRECATED: head mode recommended) database contains state changes by transactions in the blockchain "
+          "up to the head block as well as some transactions not yet included in the blockchain; transactions received by the node are relayed if valid.\n"
           )
          ( "api-accept-transactions", bpo::value<bool>()->default_value(true), "Allow API transactions to be evaluated and relayed if valid.")
          ("validation-mode", boost::program_options::value<eosio::chain::validation_mode>()->default_value(eosio::chain::validation_mode::FULL),

tests/CMakeLists.txt

@@ -138,6 +138,8 @@ set_property(TEST get_account_test PROPERTY LABELS nonparallelizable_tests)
 
 add_test(NAME distributed-transactions-test COMMAND tests/distributed-transactions-test.py -d 2 -p 4 -n 6 -v --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST distributed-transactions-test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME distributed-transactions-speculative-test COMMAND tests/distributed-transactions-test.py -d 2 -p 4 -n 6 --speculative -v --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST distributed-transactions-speculative-test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME restart-scenarios-test-resync COMMAND tests/restart-scenarios-test.py -c resync -p4 -v --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST restart-scenarios-test-resync PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME restart-scenarios-test-hard_replay COMMAND tests/restart-scenarios-test.py -c hardReplay -p4 -v --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})

tests/distributed-transactions-test.py

@@ -3,6 +3,7 @@
 import random
 
 from TestHarness import Cluster, TestHelper, Utils, WalletMgr
+from TestHarness.TestHelper import AppArgs
 
 ###############################################################
 # distributed-transactions-test
@@ -19,8 +20,10 @@
 Print=Utils.Print
 errorExit=Utils.errorExit
 
-args=TestHelper.parse_args({"-p","-n","-d","-s","--nodes-file","--seed"
-                           ,"--dump-error-details","-v","--leave-running","--clean-run","--keep-logs","--unshared"})
+appArgs = AppArgs()
+extraArgs = appArgs.add_bool(flag="--speculative", help="Run nodes in read-mode=speculative")
+args=TestHelper.parse_args({"-p","-n","-d","-s","--nodes-file","--seed", "--speculative"
+                           ,"--dump-error-details","-v","--leave-running","--clean-run","--keep-logs","--unshared"}, applicationSpecificArgs=appArgs)
 
 pnodes=args.p
 topo=args.s
@@ -34,6 +37,7 @@
 dumpErrorDetails=args.dump_error_details
 killAll=args.clean_run
 keepLogs=args.keep_logs
+speculative=args.speculative
 
 killWallet=not dontKill
 killEosInstances=not dontKill
@@ -71,7 +75,11 @@
                (pnodes, total_nodes-pnodes, topo, delay))
 
         Print("Stand up cluster")
-        if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay) is False:
+        extraNodeosArgs = ""
+        if speculative:
+           extraNodeosArgs = " --read-mode speculative "
+
+        if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay, extraNodeosArgs=extraNodeosArgs) is False:
             errorExit("Failed to stand up eos cluster.")
 
         Print ("Wait for Cluster stabilization")

tests/nodeos_irreversible_mode_test.py

@@ -21,7 +21,7 @@
 cmdError = Utils.cmdError
 relaunchTimeout = 30
 numOfProducers = 4
-totalNodes = 10
+totalNodes = 15
 
 # Parse command line arguments
 args = TestHelper.parse_args({"-v","--clean-run","--dump-error-details","--leave-running","--keep-logs","--unshared"})
@@ -32,6 +32,7 @@
 killEosInstances=not dontKill
 killWallet=not dontKill
 keepLogs=args.keep_logs
+speculativeReadMode="head"
 
 # Setup cluster and it's wallet manager
 walletMgr=WalletMgr(True)
@@ -174,7 +175,12 @@ def relaunchNode(node: Node, chainArg="", addSwapFlags=None, relaunchAssertMessa
          0:"--enable-stale-production",
          4:"--read-mode irreversible",
          6:"--read-mode irreversible",
-         9:"--plugin eosio::producer_api_plugin"})
+         9:"--plugin eosio::producer_api_plugin",
+        10:"--read-mode speculative",
+        11:"--read-mode irreversible",
+        12:"--read-mode speculative",
+        13:"--read-mode irreversible",
+        14:"--read-mode speculative --plugin eosio::producer_api_plugin"})
 
    producingNodeId = 0
    producingNode = cluster.getNode(producingNodeId)
@@ -254,7 +260,7 @@ def switchSpecToIrrMode(nodeIdOfNodeToTest, nodeToTest):
 
       # Kill and relaunch in irreversible mode
       nodeToTest.kill(signal.SIGTERM)
-      relaunchNode(nodeToTest, chainArg=" --read-mode irreversible")
+      relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "irreversible"})
 
       # Ensure the node condition is as expected after relaunch
       confirmHeadLibAndForkDbHeadOfIrrMode(nodeToTest, headLibAndForkDbHeadBeforeSwitchMode)
@@ -267,7 +273,7 @@ def switchIrrToSpecMode(nodeIdOfNodeToTest, nodeToTest):
 
       # Kill and relaunch in speculative mode
       nodeToTest.kill(signal.SIGTERM)
-      relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "head"})
+      relaunchNode(nodeToTest, addSwapFlags={"--read-mode": speculativeReadMode})
 
       # Ensure the node condition is as expected after relaunch
       confirmHeadLibAndForkDbHeadOfSpecMode(nodeToTest, headLibAndForkDbHeadBeforeSwitchMode)
@@ -283,7 +289,7 @@ def switchSpecToIrrModeWithConnectedToProdNode(nodeIdOfNodeToTest, nodeToTest):
          # Kill and relaunch in irreversible mode
          nodeToTest.kill(signal.SIGTERM)
          waitForBlksProducedAndLibAdvanced() # Wait for some blks to be produced and lib advance
-         relaunchNode(nodeToTest, chainArg=" --read-mode irreversible")
+         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "irreversible"})
 
          # Ensure the node condition is as expected after relaunch
          ensureHeadLibAndForkDbHeadIsAdvancing(nodeToTest)
@@ -302,7 +308,7 @@ def switchIrrToSpecModeWithConnectedToProdNode(nodeIdOfNodeToTest, nodeToTest):
          # Kill and relaunch in irreversible mode
          nodeToTest.kill(signal.SIGTERM)
          waitForBlksProducedAndLibAdvanced() # Wait for some blks to be produced and lib advance)
-         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "head"})
+         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": speculativeReadMode})
 
          # Ensure the node condition is as expected after relaunch
          ensureHeadLibAndForkDbHeadIsAdvancing(nodeToTest)
@@ -360,15 +366,15 @@ def switchToSpecModeWithIrrModeSnapshot(nodeIdOfNodeToTest, nodeToTest):
          backupBlksDir(nodeIdOfNodeToTest)
 
          # Relaunch in irreversible mode and create the snapshot
-         relaunchNode(nodeToTest, chainArg=" --read-mode irreversible")
+         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "irreversible"})
          confirmHeadLibAndForkDbHeadOfIrrMode(nodeToTest)
          nodeToTest.createSnapshot()
          nodeToTest.kill(signal.SIGTERM)
 
          # Start from clean data dir, recover back up blocks, and then relaunch with irreversible snapshot
          removeState(nodeIdOfNodeToTest)
          recoverBackedupBlksDir(nodeIdOfNodeToTest) # this function will delete the existing blocks dir first
-         relaunchNode(nodeToTest, chainArg=" --snapshot {}".format(getLatestSnapshot(nodeIdOfNodeToTest)), addSwapFlags={"--read-mode": "head"})
+         relaunchNode(nodeToTest, chainArg=" --snapshot {}".format(getLatestSnapshot(nodeIdOfNodeToTest)), addSwapFlags={"--read-mode": speculativeReadMode})
          confirmHeadLibAndForkDbHeadOfSpecMode(nodeToTest)
          # Ensure it automatically replays "reversible blocks", i.e. head lib and fork db should be the same
          headLibAndForkDbHeadAfterRelaunch = getHeadLibAndForkDbHead(nodeToTest)
@@ -405,6 +411,14 @@ def switchToSpecModeWithIrrModeSnapshot(nodeIdOfNodeToTest, nodeToTest):
    testSuccessful = testSuccessful and executeTest(8, replayInIrrModeWithoutRevBlksAndConnectedToProdNode)
    testSuccessful = testSuccessful and executeTest(9, switchToSpecModeWithIrrModeSnapshot)
 
+   # retest with read-mode speculative instead of head
+   speculativeReadMode="speculative"
+   testSuccessful = testSuccessful and executeTest(10, switchSpecToIrrMode)
+   testSuccessful = testSuccessful and executeTest(11, switchIrrToSpecMode)
+   testSuccessful = testSuccessful and executeTest(12, switchSpecToIrrModeWithConnectedToProdNode)
+   testSuccessful = testSuccessful and executeTest(13, switchIrrToSpecModeWithConnectedToProdNode)
+   testSuccessful = testSuccessful and executeTest(14, switchToSpecModeWithIrrModeSnapshot)
+
 finally:
    TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails)
    # Print test result

tests/nodeos_read_terminate_at_block_test.py

@@ -191,7 +191,7 @@ def checkHeadOrSpeculative(head, lib):
         0 : "--enable-stale-production",
         1 : "--read-mode irreversible --terminate-at-block 75",
         2 : "--read-mode head --terminate-at-block 100",
-        3 : "--read-mode head --terminate-at-block 125"
+        3 : "--read-mode speculative --terminate-at-block 125"
     }
 
     # Kill any existing instances and launch cluster