Skip to content

Commit

Permalink
integration tests - split steps; run in parallel (#371)
Browse files Browse the repository at this point in the history 
* separate out happy path steps

* split run steps into functions; run in parallel

* refactor after reviews; rm genesis override in actions

* add logging on parallel runs

* appease golangci-lint

* add verbose flag; rename genesis changes field

* fix race condition when using local-sdk-path

* remove logging; simplify code

* remove .gitignore changes

Co-authored-by: Jehan <jehan.tremback@gmail.com>
Co-authored-by: Daniel T <30197399+danwt@users.noreply.github.com>
Co-authored-by: Shawn Marshall-Spitzbart <44221603+smarshall-spitzbart@users.noreply.github.com>
  • Loading branch information
@MSalopek @jtremback
@danwt @shaspitz
4 people authored Oct 7, 2022
1 parent ba1fa6d commit 3f2091e
Show file tree
Hide file tree
Showing 9 changed files with 894 additions and 1,011 deletions.
2 changes: 2 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,8 @@ go run ./tests/integration/...
go run ./tests/integration/... --local-sdk-path "/Users/bob/Documents/cosmos-sdk/"
# run golang native fuzz tests (https://go.dev/doc/tutorial/fuzz)
go test -fuzz=<regex-to-match-test-name>
# run verbose integration tests
go run ./tests/integration/... --local-sdk-path "/Users/bob/Documents/cosmos-sdk/" --verbose
```

### Linters and Static Analysis
Expand Down
20 changes: 10 additions & 10 deletions tests/integration/actions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,10 +361,9 @@ func (tr TestRun) voteGovProposal(
}

type startConsumerChainAction struct {
consumerChain chainID
providerChain chainID
genesisChanges string
validators []StartChainValidator
consumerChain chainID
providerChain chainID
validators []StartChainValidator
}

func (tr TestRun) startConsumerChain(
Expand All @@ -391,15 +390,16 @@ func (tr TestRun) startConsumerChain(
log.Fatal(err, "\n", string(bz))
}

genesisChanges := ".app_state.ccvconsumer = " + string(bz)
if action.genesisChanges != "" {
genesisChanges = genesisChanges + " | " + action.genesisChanges
consumerGenesis := ".app_state.ccvconsumer = " + string(bz)
consumerGenesisChanges := tr.chainConfigs[action.consumerChain].genesisChanges
if consumerGenesisChanges != "" {
consumerGenesis = consumerGenesis + " | " + consumerGenesisChanges
}

tr.startChain(StartChainAction{
chain: action.consumerChain,
validators: action.validators,
genesisChanges: genesisChanges,
genesisChanges: consumerGenesis,
skipGentx: true,
}, verbose)
}
Expand Down Expand Up @@ -640,7 +640,7 @@ func (tr TestRun) transferChannelComplete(
}

func executeCommand(cmd *exec.Cmd, cmdName string) {
if verbose {
if verbose != nil && *verbose {
fmt.Println(cmdName+" cmd:", cmd.String())
}

Expand All @@ -658,7 +658,7 @@ func executeCommand(cmd *exec.Cmd, cmdName string) {

for scanner.Scan() {
out := scanner.Text()
if verbose {
if verbose != nil && *verbose {
fmt.Println(cmdName + ": " + out)
}
}
Expand Down
108 changes: 71 additions & 37 deletions tests/integration/config.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package main

import (
"flag"
"fmt"
"strconv"
"time"
Expand Down Expand Up @@ -52,45 +51,52 @@ type TestRun struct {
validatorConfigs map[validatorID]ValidatorConfig
chainConfigs map[chainID]ChainConfig
localSdkPath string

name string
}

func getDefaultValidators() map[validatorID]ValidatorConfig {
return map[validatorID]ValidatorConfig{
validatorID("alice"): {
mnemonic: "pave immune ethics wrap gain ceiling always holiday employ earth tumble real ice engage false unable carbon equal fresh sick tattoo nature pupil nuclear",
delAddress: "cosmos19pe9pg5dv9k5fzgzmsrgnw9rl9asf7ddwhu7lm",
valoperAddress: "cosmosvaloper19pe9pg5dv9k5fzgzmsrgnw9rl9asf7ddtrgtng",
valconsAddress: "cosmosvalcons1qmq08eruchr5sf5s3rwz7djpr5a25f7xw4mceq",
privValidatorKey: `{"address":"06C0F3E47CC5C748269088DC2F36411D3AAA27C6","pub_key":{"type":"tendermint/PubKeyEd25519","value":"RrclQz9bIhkIy/gfL485g3PYMeiIku4qeo495787X10="},"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"uX+ZpDMg89a6gtqs/+MQpCTSqlkZ0nJQJOhLlCJvwvdGtyVDP1siGQjL+B8vjzmDc9gx6IiS7ip6jj3nvztfXQ=="}}`,
nodeKey: `{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"fjw4/DAhyRPnwKgXns5SV7QfswRSXMWJpHS7TyULDmJ8ofUc5poQP8dgr8bZRbCV5RV8cPqDq3FPdqwpmUbmdA=="}}`,
ipSuffix: "4",
},
validatorID("bob"): {
mnemonic: "glass trip produce surprise diamond spin excess gaze wash drum human solve dress minor artefact canoe hard ivory orange dinner hybrid moral potato jewel",
delAddress: "cosmos1dkas8mu4kyhl5jrh4nzvm65qz588hy9qcz08la",
valoperAddress: "cosmosvaloper1dkas8mu4kyhl5jrh4nzvm65qz588hy9qakmjnw",
valconsAddress: "cosmosvalcons1nx7n5uh0ztxsynn4sje6eyq2ud6rc6klc96w39",
privValidatorKey: `{"address":"99BD3A72EF12CD024E7584B3AC900AE3743C6ADF","pub_key":{"type":"tendermint/PubKeyEd25519","value":"mAN6RXYxSM4MNGSIriYiS7pHuwAcOHDQAy9/wnlSzOI="},"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"QePcwfWtOavNK7pBJrtoLMzarHKn6iBWfWPFeyV+IdmYA3pFdjFIzgw0ZIiuJiJLuke7ABw4cNADL3/CeVLM4g=="}}`,
nodeKey: `{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"TQ4vHcO/vKdzGtWpelkX53WdMQd4kTsWGFrdcatdXFvWyO215Rewn5IRP0FszPLWr2DqPzmuH8WvxYGk5aeOXw=="}}`,
ipSuffix: "5",
},
validatorID("carol"): {
mnemonic: "sight similar better jar bitter laptop solve fashion father jelly scissors chest uniform play unhappy convince silly clump another conduct behave reunion marble animal",
delAddress: "cosmos19hz4m226ztankqramvt4a7t0shejv4dc79gp9u",
valoperAddress: "cosmosvaloper19hz4m226ztankqramvt4a7t0shejv4dcm3u5f0",
valconsAddress: "cosmosvalcons1ezyrq65s3gshhx5585w6mpusq3xsj3ayzf4uv6",
privValidatorKey: `{"address":"C888306A908A217B9A943D1DAD8790044D0947A4","pub_key":{"type":"tendermint/PubKeyEd25519","value":"IHo4QEikWZfIKmM0X+N+BjKttz8HOzGs2npyjiba3Xk="},"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"z08bmSB91uFVpVmR3t2ewd/bDjZ/AzwQpe5rKjWiPG0gejhASKRZl8gqYzRf434GMq23Pwc7MazaenKOJtrdeQ=="}}`,
nodeKey: `{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"WLTcHEjbwB24Wp3z5oBSYTvtGQonz/7IQabOFw85BN0UkkyY5HDf38o8oHlFxVI26f+DFVeICuLbe9aXKGnUeg=="}}`,
ipSuffix: "6",
},
}
}

func DefaultTestRun() TestRun {
return TestRun{
name: "default",
containerConfig: ContainerConfig{
containerName: "interchain-security-container",
instanceName: "interchain-security-instance",
ccvVersion: "1",
now: time.Now(),
},
validatorConfigs: map[validatorID]ValidatorConfig{
validatorID("alice"): {
mnemonic: "pave immune ethics wrap gain ceiling always holiday employ earth tumble real ice engage false unable carbon equal fresh sick tattoo nature pupil nuclear",
delAddress: "cosmos19pe9pg5dv9k5fzgzmsrgnw9rl9asf7ddwhu7lm",
valoperAddress: "cosmosvaloper19pe9pg5dv9k5fzgzmsrgnw9rl9asf7ddtrgtng",
valconsAddress: "cosmosvalcons1qmq08eruchr5sf5s3rwz7djpr5a25f7xw4mceq",
privValidatorKey: `{"address":"06C0F3E47CC5C748269088DC2F36411D3AAA27C6","pub_key":{"type":"tendermint/PubKeyEd25519","value":"RrclQz9bIhkIy/gfL485g3PYMeiIku4qeo495787X10="},"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"uX+ZpDMg89a6gtqs/+MQpCTSqlkZ0nJQJOhLlCJvwvdGtyVDP1siGQjL+B8vjzmDc9gx6IiS7ip6jj3nvztfXQ=="}}`,
nodeKey: `{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"fjw4/DAhyRPnwKgXns5SV7QfswRSXMWJpHS7TyULDmJ8ofUc5poQP8dgr8bZRbCV5RV8cPqDq3FPdqwpmUbmdA=="}}`,
ipSuffix: "4",
},
validatorID("bob"): {
mnemonic: "glass trip produce surprise diamond spin excess gaze wash drum human solve dress minor artefact canoe hard ivory orange dinner hybrid moral potato jewel",
delAddress: "cosmos1dkas8mu4kyhl5jrh4nzvm65qz588hy9qcz08la",
valoperAddress: "cosmosvaloper1dkas8mu4kyhl5jrh4nzvm65qz588hy9qakmjnw",
valconsAddress: "cosmosvalcons1nx7n5uh0ztxsynn4sje6eyq2ud6rc6klc96w39",
privValidatorKey: `{"address":"99BD3A72EF12CD024E7584B3AC900AE3743C6ADF","pub_key":{"type":"tendermint/PubKeyEd25519","value":"mAN6RXYxSM4MNGSIriYiS7pHuwAcOHDQAy9/wnlSzOI="},"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"QePcwfWtOavNK7pBJrtoLMzarHKn6iBWfWPFeyV+IdmYA3pFdjFIzgw0ZIiuJiJLuke7ABw4cNADL3/CeVLM4g=="}}`,
nodeKey: `{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"TQ4vHcO/vKdzGtWpelkX53WdMQd4kTsWGFrdcatdXFvWyO215Rewn5IRP0FszPLWr2DqPzmuH8WvxYGk5aeOXw=="}}`,
ipSuffix: "5",
},
validatorID("carol"): {
mnemonic: "sight similar better jar bitter laptop solve fashion father jelly scissors chest uniform play unhappy convince silly clump another conduct behave reunion marble animal",
delAddress: "cosmos19hz4m226ztankqramvt4a7t0shejv4dc79gp9u",
valoperAddress: "cosmosvaloper19hz4m226ztankqramvt4a7t0shejv4dcm3u5f0",
valconsAddress: "cosmosvalcons1ezyrq65s3gshhx5585w6mpusq3xsj3ayzf4uv6",
privValidatorKey: `{"address":"C888306A908A217B9A943D1DAD8790044D0947A4","pub_key":{"type":"tendermint/PubKeyEd25519","value":"IHo4QEikWZfIKmM0X+N+BjKttz8HOzGs2npyjiba3Xk="},"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"z08bmSB91uFVpVmR3t2ewd/bDjZ/AzwQpe5rKjWiPG0gejhASKRZl8gqYzRf434GMq23Pwc7MazaenKOJtrdeQ=="}}`,
nodeKey: `{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"WLTcHEjbwB24Wp3z5oBSYTvtGQonz/7IQabOFw85BN0UkkyY5HDf38o8oHlFxVI26f+DFVeICuLbe9aXKGnUeg=="}}`,
ipSuffix: "6",
},
},
validatorConfigs: getDefaultValidators(),
chainConfigs: map[chainID]ChainConfig{
chainID("provi"): {
chainId: chainID("provi"),
Expand All @@ -116,12 +122,41 @@ func DefaultTestRun() TestRun {
".app_state.slashing.params.downtime_jail_duration = \"2s\" | " +
".app_state.slashing.params.slash_fraction_downtime = \"0.010000000000000000\"",
},
},
}
}

func DemocracyTestRun() TestRun {
return TestRun{
name: "democracy",
containerConfig: ContainerConfig{
containerName: "interchain-security-democ-container",
instanceName: "interchain-security-democ-instance",
ccvVersion: "1",
now: time.Now(),
},
validatorConfigs: getDefaultValidators(),
chainConfigs: map[chainID]ChainConfig{
chainID("provi"): {
chainId: chainID("provi"),
binaryName: "interchain-security-pd",
ipPrefix: "7.7.7",
votingWaitTime: 5,
genesisChanges: ".app_state.gov.voting_params.voting_period = \"5s\" | " +
// Custom slashing parameters for testing validator downtime functionality
// See https://docs.cosmos.network/main/modules/slashing/04_begin_block.html#uptime-tracking
".app_state.slashing.params.signed_blocks_window = \"2\" | " +
".app_state.slashing.params.min_signed_per_window = \"0.500000000000000000\" | " +
".app_state.slashing.params.downtime_jail_duration = \"2s\" | " +
".app_state.slashing.params.slash_fraction_downtime = \"0.010000000000000000\"",
},
chainID("democ"): {
chainId: chainID("democ"),
binaryName: "interchain-security-cdd",
ipPrefix: "7.7.9",
votingWaitTime: 10,
genesisChanges: ".app_state.gov.voting_params.voting_period = \"10s\" | " +
genesisChanges: ".app_state.ccvconsumer.params.blocks_per_distribution_transmission = \"10\" | " +
".app_state.gov.voting_params.voting_period = \"10s\" | " +
".app_state.slashing.params.signed_blocks_window = \"2\" | " +
".app_state.slashing.params.min_signed_per_window = \"0.500000000000000000\" | " +
".app_state.slashing.params.downtime_jail_duration = \"2s\" | " +
Expand All @@ -131,12 +166,11 @@ func DefaultTestRun() TestRun {
}
}

func (s *TestRun) ParseCLIFlags() {
localSdkPath := flag.String("local-sdk-path", "",
"path of a local sdk version to build and reference in integration tests")
flag.Parse()
s.localSdkPath = *localSdkPath
fmt.Println(s.localSdkPath)
func (s *TestRun) SetLocalSDKPath(path string) {
if path != "" {
fmt.Println("USING LOCAL SDK", path)
}
s.localSdkPath = path
}

// ValidateStringLiterals enforces that configs follow the constraints
Expand Down
67 changes: 45 additions & 22 deletions tests/integration/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,44 +2,52 @@ package main

import (
"bufio"
"flag"
"fmt"
"log"
"os/exec"
"reflect"
"sync"
"time"

"github.com/kylelemons/godebug/pretty"
)

var verbose = false
var verbose = flag.Bool("verbose", false, "turn verbose logging on/off")
var localSdkPath = flag.String("local-sdk-path", "",
"path of a local sdk version to build and reference in integration tests")

// runs integration tests
// all docker containers are built sequentially to avoid race conditions when using local cosmos-sdk
// after building docker containers, all tests are run in parallel using their respective docker containers
func main() {
fmt.Println("============================================ start happy path tests ============================================")
flag.Parse()

// wg waits for all runners to complete
var wg sync.WaitGroup

start := time.Now()
tr := DefaultTestRun()
tr.ParseCLIFlags()
tr.SetLocalSDKPath(*localSdkPath)
tr.ValidateStringLiterals()
tr.startDocker()

for _, step := range happyPathSteps {
tr.runStep(step, verbose)
}
dmc := DemocracyTestRun()
dmc.SetLocalSDKPath(*localSdkPath)
dmc.ValidateStringLiterals()
dmc.startDocker()

fmt.Printf("happy path tests successful - time elapsed %v\n", time.Since(start))
wg.Add(1)
go tr.ExecuteSteps(&wg, happyPathSteps)

fmt.Println("============================================ start democracy tests ============================================")
start = time.Now()
tr.startDocker()
wg.Add(1)
go dmc.ExecuteSteps(&wg, democracySteps)

for _, step := range democracySteps {
tr.runStep(step, verbose)
}

fmt.Printf("democracy tests successful - time elapsed %v\n", time.Since(start))
wg.Wait()
fmt.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start))
}

func (tr TestRun) runStep(step Step, verbose bool) {
fmt.Printf("%#v\n", step.action)
func (tr *TestRun) runStep(step Step, verbose bool) {
switch action := step.action.(type) {
case StartChainAction:
tr.startChain(action, verbose)
Expand Down Expand Up @@ -80,7 +88,7 @@ func (tr TestRun) runStep(step Step, verbose bool) {
case registerRepresentativeAction:
tr.registerRepresentative(action, verbose)
default:
log.Fatalf(fmt.Sprintf(`unknown action: %#v`, action))
log.Fatalf(fmt.Sprintf(`unknown action in testRun %s: %#v`, tr.name, action))
}

modelState := step.state
Expand All @@ -92,16 +100,31 @@ func (tr TestRun) runStep(step Step, verbose bool) {
pretty.Print("model state", modelState)
log.Fatal(`actual state (-) not equal to model state (+): ` + pretty.Compare(actualState, modelState))
}
}

pretty.Print(actualState)
// Starts docker container and sequentially runs steps
func (tr *TestRun) ExecuteSteps(wg *sync.WaitGroup, steps []Step) {
defer wg.Done()
fmt.Printf("=============== started %s tests ===============\n", tr.name)

start := time.Now()
for i, step := range steps {
// print something the show the test is alive
if i%10 == 0 {
fmt.Printf("running %s: step %d\n", tr.name, i+1)
}
tr.runStep(step, *verbose)
}

fmt.Printf("=============== finished %s tests in %v ===============\n", tr.name, time.Since(start))
}

func (tr TestRun) startDocker() {
func (tr *TestRun) startDocker() {
fmt.Printf("=============== building %s testRun ===============\n", tr.name)
scriptStr := "tests/integration/testnet-scripts/start-docker.sh " +
tr.containerConfig.containerName + " " +
tr.containerConfig.instanceName + " " +
tr.localSdkPath

//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
cmd := exec.Command("/bin/bash", "-c", scriptStr)

Expand All @@ -119,7 +142,7 @@ func (tr TestRun) startDocker() {

for scanner.Scan() {
out := scanner.Text()
if verbose {
if verbose != nil && *verbose {
fmt.Println("startDocker: " + out)
}
if out == "beacon!!!!!!!!!!" {
Expand Down
Loading

0 comments on commit 3f2091e

Please sign in to comment.