Merge pull request #295 from lazyledger/ismail/cherrypick-e2e-changes

cherrypick e2e changes regarding maverick from tm master

.github/workflows/coverage.yml

@@ -121,7 +121,7 @@ jobs:
       - run: |
           cat ./*profile.out | grep -v "mode: atomic" >> coverage.txt
         if: env.GIT_DIFF
-      - uses: codecov/codecov-action@v1.4.0
+      - uses: codecov/codecov-action@v1.4.1
         with:
           file: ./coverage.txt
         if: env.GIT_DIFF

test/e2e/README.md

@@ -7,7 +7,27 @@ make
 ./build/runner -f networks/ci.toml
 ```
 
-This creates and runs a testnet named `ci` under `networks/ci/` (determined by the manifest filename).
+This creates and runs a testnet named `ci` under `networks/ci/`.
+
+## Conceptual Overview
+
+End-to-end testnets are used to test Tendermint functionality as a user would use it, by spinning up a set of nodes with various configurations and making sure the nodes and network behave correctly. The background for the E2E test suite is outlined in [RFC-001](https://github.com/tendermint/tendermint/blob/master/docs/rfc/rfc-001-end-to-end-testing.md).
+
+The end-to-end tests can be thought of in this manner:
+
+1. Does a certain (valid!) testnet configuration result in a block-producing network where all nodes eventually reach the latest height?
+
+2. If so, does each node in that network satisfy all invariants specified by the Go E2E tests?
+
+The above should hold for any arbitrary, valid network configuration, and that configuration space  should be searched and tested by randomly generating testnets.
+
+A testnet configuration is specified as a TOML testnet manifest (see below). The testnet runner uses the manifest to configure a set of Docker containers and start them in some order. The manifests can be written manually (to test specific configurations) or generated randomly by the testnet generator (to test a wide range of configuration permutations).
+
+When running a testnet, the runner will first start the Docker nodes in some sequence, submit random transactions, and wait for the nodes to come online and the first blocks to be produced. This may involve e.g. waiting for nodes to fast sync and/or state sync. If specified, it will then run any misbehaviors (e.g. double-signing) and perturbations (e.g. killing or disconnecting nodes). It then waits for the testnet to stabilize, with all nodes online and having reached the latest height.
+
+Once the testnet stabilizes, a set of Go end-to-end tests are run against the live testnet to verify network invariants (for example that blocks are identical across nodes). These use the RPC client to interact with the network, and should consider the entire network as a black box (i.e. it should not test any network or node internals, only externally visible behavior via RPC). The tests may use the `testNode()` helper to run parallel tests against each individual testnet node, and/or inspect the full blockchain history via `fetchBlockChain()`.
+
+The tests must take into account the network and/or node configuration, and tolerate that the network is still live and producing blocks. For example, validator tests should only run against nodes that are actually validators, and take into account the node's block retention and/or state sync configuration to not query blocks that don't exist.
 
 ## Testnet Manifests
 

test/e2e/app/main.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strconv"
 	"time"
 
 	"github.com/spf13/viper"
@@ -19,8 +18,6 @@ import (
 	"github.com/lazyledger/lazyledger-core/p2p"
 	"github.com/lazyledger/lazyledger-core/privval"
 	"github.com/lazyledger/lazyledger-core/proxy"
-	mcs "github.com/lazyledger/lazyledger-core/test/maverick/consensus"
-	maverick "github.com/lazyledger/lazyledger-core/test/maverick/node"
 )
 
 var logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
@@ -62,11 +59,12 @@ func run(configFile string) error {
 	// Start app server.
 	switch cfg.Protocol {
 	case "builtin":
-		if len(cfg.Misbehaviors) == 0 {
-			err = startNode(cfg)
-		} else {
-			err = startMaverick(cfg)
-		}
+		// FIXME: Temporarily remove maverick until it is redesigned
+		// if len(cfg.Misbehaviors) == 0 {
+		err = startNode(cfg)
+		// } else {
+		// 	err = startMaverick(cfg)
+		// }
 	default:
 		err = fmt.Errorf("invalid protocol %q", cfg.Protocol)
 	}
@@ -114,41 +112,42 @@ func startNode(cfg *Config) error {
 	return n.Start()
 }
 
+// FIXME: Temporarily disconnected maverick until it is redesigned
 // startMaverick starts a Maverick node that runs the application directly. It assumes the Tendermint
 // configuration is in $TMHOME/config/tendermint.toml.
-func startMaverick(cfg *Config) error {
-	app, err := NewApplication(cfg)
-	if err != nil {
-		return err
-	}
-
-	tmcfg, logger, nodeKey, err := setupNode()
-	if err != nil {
-		return fmt.Errorf("failed to setup config: %w", err)
-	}
-
-	misbehaviors := make(map[int64]mcs.Misbehavior, len(cfg.Misbehaviors))
-	for heightString, misbehaviorString := range cfg.Misbehaviors {
-		height, _ := strconv.ParseInt(heightString, 10, 64)
-		misbehaviors[height] = mcs.MisbehaviorList[misbehaviorString]
-	}
-
-	n, err := maverick.NewNode(tmcfg,
-		maverick.LoadOrGenFilePV(tmcfg.PrivValidatorKeyFile(), tmcfg.PrivValidatorStateFile()),
-		*nodeKey,
-		proxy.NewLocalClientCreator(app),
-		maverick.DefaultGenesisDocProviderFunc(tmcfg),
-		maverick.DefaultDBProvider,
-		maverick.DefaultMetricsProvider(tmcfg.Instrumentation),
-		logger,
-		misbehaviors,
-	)
-	if err != nil {
-		return err
-	}
-
-	return n.Start()
-}
+// func startMaverick(cfg *Config) error {
+// 	app, err := NewApplication(cfg)
+// 	if err != nil {
+// 		return err
+// 	}
+
+// 	tmcfg, logger, nodeKey, err := setupNode()
+// 	if err != nil {
+// 		return fmt.Errorf("failed to setup config: %w", err)
+// 	}
+
+// 	misbehaviors := make(map[int64]mcs.Misbehavior, len(cfg.Misbehaviors))
+// 	for heightString, misbehaviorString := range cfg.Misbehaviors {
+// 		height, _ := strconv.ParseInt(heightString, 10, 64)
+// 		misbehaviors[height] = mcs.MisbehaviorList[misbehaviorString]
+// 	}
+
+// 	n, err := maverick.NewNode(tmcfg,
+// 		maverick.LoadOrGenFilePV(tmcfg.PrivValidatorKeyFile(), tmcfg.PrivValidatorStateFile()),
+// 		*nodeKey,
+// 		proxy.NewLocalClientCreator(app),
+// 		maverick.DefaultGenesisDocProviderFunc(tmcfg),
+// 		maverick.DefaultDBProvider,
+// 		maverick.DefaultMetricsProvider(tmcfg.Instrumentation),
+// 		logger,
+// 		misbehaviors,
+// 	)
+// 	if err != nil {
+// 		return err
+// 	}
+
+// 	return n.Start()
+// }
 
 // startSigner starts a signer server connecting to the given endpoint.
 func startSigner(cfg *Config) error {

test/e2e/docker/Dockerfile

@@ -20,7 +20,8 @@ RUN go mod download
 COPY . .
 RUN make build && cp build/tendermint /usr/bin/tendermint
 COPY test/e2e/docker/entrypoint* /usr/bin/
-RUN cd test/e2e && make maverick && cp build/maverick /usr/bin/maverick
+# FIXME: Temporarily disconnect maverick node until it is redesigned
+# RUN cd test/e2e && make maverick && cp build/maverick /usr/bin/maverick
 RUN cd test/e2e && make app && cp build/app /usr/bin/app
 
 # Set up runtime directory. We don't use a separate runtime image since we need

test/e2e/networks/ci.toml

@@ -36,7 +36,8 @@ seeds = ["seed01"]
 seeds = ["seed01"]
 snapshot_interval = 5
 perturb = ["disconnect"]
-misbehaviors = { 1018 = "double-prevote" }
+# FIXME: maverick has been disabled until it is redesigned (https://github.com/tendermint/tendermint/issues/5575)
+# misbehaviors = { 1018 = "double-prevote" }
 
 [node.validator02]
 seeds = ["seed02"]

test/e2e/pkg/testnet.go

@@ -16,7 +16,6 @@ import (
 	"github.com/lazyledger/lazyledger-core/crypto/ed25519"
 	"github.com/lazyledger/lazyledger-core/crypto/secp256k1"
 	rpchttp "github.com/lazyledger/lazyledger-core/rpc/client/http"
-	mcs "github.com/lazyledger/lazyledger-core/test/maverick/consensus"
 	"github.com/lazyledger/lazyledger-core/types"
 )
 
@@ -356,11 +355,12 @@ func (n Node) Validate(testnet Testnet) error {
 				height, testnet.InitialHeight)
 		}
 		exists := false
-		for possibleBehaviors := range mcs.MisbehaviorList {
-			if possibleBehaviors == misbehavior {
-				exists = true
-			}
-		}
+		// FIXME: Maverick has been disabled until it is redesigned
+		// for possibleBehaviors := range mcs.MisbehaviorList {
+		// 	if possibleBehaviors == misbehavior {
+		// 		exists = true
+		// 	}
+		// }
 		if !exists {
 			return fmt.Errorf("misbehavior %s does not exist", misbehavior)
 		}

test/e2e/runner/setup.go

@@ -128,17 +128,24 @@ func Setup(testnet *e2e.Testnet) error {
 func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) {
 	// Must use version 2 Docker Compose format, to support IPv6.
 	tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{
-		"misbehaviorsToString": func(misbehaviors map[int64]string) string {
-			str := ""
-			for height, misbehavior := range misbehaviors {
-				// after the first behavior set, a comma must be prepended
-				if str != "" {
-					str += ","
-				}
-				heightString := strconv.Itoa(int(height))
-				str += misbehavior + "," + heightString
-			}
-			return str
+		"startCommands": func(misbehaviors map[int64]string, logLevel string) string {
+			command := "start"
+
+			// FIXME: Temporarily disable behaviors until maverick is redesigned
+			// misbehaviorString := ""
+			// for height, misbehavior := range misbehaviors {
+			// 	// after the first behavior set, a comma must be prepended
+			// 	if misbehaviorString != "" {
+			// 		misbehaviorString += ","
+			// 	}
+			// 	heightString := strconv.Itoa(int(height))
+			// 	misbehaviorString += misbehavior + "," + heightString
+			// }
+
+			// if misbehaviorString != "" {
+			// 	command += " --misbehaviors " + misbehaviorString
+			// }
+			return command
 		},
 	}).Parse(`version: '2.4'
 
@@ -164,9 +171,9 @@ services:
     image: tendermint/e2e-node
 {{- if eq .ABCIProtocol "builtin" }}
     entrypoint: /usr/bin/entrypoint-builtin
-{{- else if .Misbehaviors }}
-    entrypoint: /usr/bin/entrypoint-maverick
-    command: ["start", "--misbehaviors", "{{ misbehaviorsToString .Misbehaviors }}"]
+{{- end }}
+{{- if ne .ABCIProtocol "builtin"}}
+    command: {{ startCommands .Misbehaviors .LogLevel }}
 {{- end }}
     init: true
     ports:

test/e2e/tests/validator_test.go

@@ -77,8 +77,9 @@ func TestValidator_Propose(t *testing.T) {
 
 		require.False(t, proposeCount == 0 && expectCount > 0,
 			"node did not propose any blocks (expected %v)", expectCount)
-		require.Less(t, expectCount-proposeCount, 5,
-			"validator missed proposing too many blocks (proposed %v out of %v)", proposeCount, expectCount)
+		if expectCount > 5 {
+			require.GreaterOrEqual(t, proposeCount, 3, "validator didn't propose even 3 blocks")
+		}
 	})
 }
 
@@ -115,9 +116,10 @@ func TestValidator_Sign(t *testing.T) {
 		}
 
 		require.False(t, signCount == 0 && expectCount > 0,
-			"node did not sign any blocks (expected %v)", expectCount)
-		require.Less(t, float64(expectCount-signCount)/float64(expectCount), 0.5,
-			"validator missed signing too many blocks (signed %v out of %v)", signCount, expectCount)
+			"validator did not sign any blocks (expected %v)", expectCount)
+		if expectCount > 7 {
+			require.GreaterOrEqual(t, signCount, 3, "validator didn't sign even 3 blocks (expected %v)", expectCount)
+		}
 	})
 }