From 1a678a0400700c89f3b7544ede1901061ff290ee Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Tue, 26 Nov 2024 21:49:41 +0100 Subject: [PATCH] chore: prepare systemtests release --- systemtests/CHANGELOG.md | 2 + systemtests/go.mod | 39 +++-- tests/systemtests/getting_started.md | 215 --------------------------- tests/systemtests/go.mod | 49 +++--- tests/systemtests/go.sum | 2 + 5 files changed, 44 insertions(+), 263 deletions(-) delete mode 100644 tests/systemtests/getting_started.md diff --git a/systemtests/CHANGELOG.md b/systemtests/CHANGELOG.md index 52aaba381361..7934d5f60259 100644 --- a/systemtests/CHANGELOG.md +++ b/systemtests/CHANGELOG.md @@ -36,6 +36,8 @@ Ref: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.j ## [Unreleased] +## [v1.0.0-rc.1] - 2024-11-26 + ### Features * [#22578](https://github.com/cosmos/cosmos-sdk/pull/22578) Extract system test framework diff --git a/systemtests/go.mod b/systemtests/go.mod index 6a507c647041..b2036b6ceb71 100644 --- a/systemtests/go.mod +++ b/systemtests/go.mod @@ -2,34 +2,16 @@ module cosmossdk.io/systemtests go 1.23 -require ( - github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect - github.com/cosmos/cosmos-sdk v0.50.6 - github.com/cosmos/gogogateway v1.2.0 // indirect - github.com/cosmos/gogoproto v1.7.0 // indirect - github.com/cosmos/iavl v1.1.4 // indirect - github.com/dvsekhvalnov/jose2go v1.6.0 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/gorilla/mux v1.8.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.20.5 // indirect - github.com/spf13/cast v1.7.0 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.9.0 - github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect - google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/grpc v1.68.0 -) - require ( cosmossdk.io/math v1.4.0 github.com/cometbft/cometbft v0.38.15 github.com/cometbft/cometbft/api v1.0.0-rc.1 + github.com/cosmos/cosmos-sdk v0.50.6 github.com/creachadair/tomledit v0.0.26 + github.com/stretchr/testify v1.9.0 github.com/tidwall/gjson v1.14.2 github.com/tidwall/sjson v1.2.5 + google.golang.org/grpc v1.68.0 ) require ( @@ -63,7 +45,11 @@ require ( github.com/cometbft/cometbft-db v0.14.1 // indirect github.com/cosmos/btcutil v1.0.5 // indirect github.com/cosmos/cosmos-db v1.0.3-0.20240829004618-717cba019b33 // indirect + github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect + github.com/cosmos/gogogateway v1.2.0 // indirect + github.com/cosmos/gogoproto v1.7.0 // indirect + github.com/cosmos/iavl v1.1.4 // indirect github.com/cosmos/ics23/go v0.11.0 // indirect github.com/cosmos/ledger-cosmos-go v0.13.3 // indirect github.com/danieljoos/wincred v1.1.2 // indirect @@ -73,6 +59,7 @@ require ( github.com/dgraph-io/badger/v4 v4.2.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/emicklei/dot v1.6.2 // indirect github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -86,14 +73,17 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/flatbuffers v1.12.1 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/orderedcode v0.0.1 // indirect github.com/gorilla/handlers v1.5.1 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect @@ -126,7 +116,9 @@ require ( github.com/oklog/run v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect @@ -139,8 +131,12 @@ require ( github.com/sasha-s/go-deadlock v0.3.5 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.7.0 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.19.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/tendermint/go-amino v0.16.0 // indirect github.com/tidwall/btree v1.7.0 // indirect github.com/tidwall/match v1.1.1 // indirect @@ -159,6 +155,7 @@ require ( golang.org/x/sys v0.27.0 // indirect golang.org/x/term v0.26.0 // indirect golang.org/x/text v0.20.0 // indirect + google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/tests/systemtests/getting_started.md b/tests/systemtests/getting_started.md deleted file mode 100644 index 7adc98f1b3e9..000000000000 --- a/tests/systemtests/getting_started.md +++ /dev/null @@ -1,215 +0,0 @@ -# Getting started with a new system test - -## Preparation - -Build a new binary from current branch and copy it to the `tests/systemtests/binaries` folder by running system tests. -In project root: - -```shell -make test-system -``` - -Or via manual steps - -```shell -make build -mkdir -p ./tests/systemtests/binaries -cp ./build/simd ./tests/systemtests/binaries/ -``` - -## Part 1: Writing the first system test - -Switch to the `tests/systemtests` folder to work from here. - -If there is no test file matching your use case, start a new test file here. -for example `bank_test.go` to begin with: - -```go -//go:build system_test - -package systemtests - -import ( - "testing" -) - -func TestQueryTotalSupply(t *testing.T) { - sut.ResetChain(t) - sut.StartChain(t) - - cli := NewCLIWrapper(t, sut, verbose) - raw := cli.CustomQuery("q", "bank", "total-supply") - t.Log("### got: " + raw) -} -``` - -The file begins with a Go build tag to exclude it from regular go test runs. -All tests in the `systemtests` folder build upon the *test runner* initialized in `main_test.go`. -This gives you a multi node chain started on your box. -It is a good practice to reset state in the beginning so that you have a stable base. - -The system tests framework comes with a CLI wrapper that makes it easier to interact or parse results. -In this example we want to execute `simd q bank total-supply --output json --node tcp://localhost:26657` which queries -the bank module. -Then print the result to for the next steps - -### Run the test - -```shell -go test -mod=readonly -tags='system_test' -v ./... --run TestQueryTotalSupply --verbose -``` - -This give very verbose output. You would see all simd CLI commands used for starting the server or by the client to interact. -In the example code, we just log the output. Watch out for - -```shell - bank_test.go:15: ### got: { - "supply": [ - { - "denom": "stake", - "amount": "2000000190" - }, - { - "denom": "testtoken", - "amount": "4000000000" - } - ], - "pagination": { - "total": "2" - } - } -``` - -At the end is a tail from the server log printed. This can sometimes be handy when debugging issues. - - -### Tips - -* Passing `--nodes-count=1` overwrites the default node count and can speed up your test for local runs - -## Part 2: Working with json - -When we have a json response, the [gjson](https://github.com/tidwall/gjson) lib can shine. It comes with jquery like -syntax that makes it easy to navigation within the document. - -For example `gjson.Get(raw, "supply").Array()` gives us all the childs to `supply` as an array. -Or `gjson.Get("supply.#(denom==stake).amount").Int()` for the amount of the stake token as int64 type. - -In order to test our assumptions in the system test, we modify the code to use `gjson` to fetch the data: - -```go - raw := cli.CustomQuery("q", "bank", "total-supply") - - exp := map[string]int64{ - "stake": int64(500000000 * sut.nodesCount), - "testtoken": int64(1000000000 * sut.nodesCount), - } - require.Len(t, gjson.Get(raw, "supply").Array(), len(exp), raw) - - for k, v := range exp { - got := gjson.Get(raw, fmt.Sprintf("supply.#(denom==%q).amount", k)).Int() - assert.Equal(t, v, got, raw) - } -``` - -The assumption on the staking token usually fails due to inflation minted on the staking token. Let's fix this in the next step - -### Run the test - -```shell -go test -mod=readonly -tags='system_test' -v ./... --run TestQueryTotalSupply --verbose -``` - -### Tips - -* Putting the `raw` json response to the assert/require statements helps with debugging on failures. You are usually lacking - context when you look at the values only. - - -## Part 3: Setting state via genesis - -First step is to disable inflation. This can be done via the `ModifyGenesisJSON` helper. But to add some complexity, -we also introduce a new token and update the balance of the account for key `node0`. -The setup code looks quite big and unreadable now. Usually a good time to think about extracting helper functions for -common operations. The `genesis_io.go` file contains some examples already. I would skip this and take this to showcase the mix -of `gjson`, `sjson` and stdlib json operations. - -```go - sut.ResetChain(t) - cli := NewCLIWrapper(t, sut, verbose) - - sut.ModifyGenesisJSON(t, func(genesis []byte) []byte { - // disable inflation - genesis, err := sjson.SetRawBytes(genesis, "app_state.mint.minter.inflation", []byte(`"0.000000000000000000"`)) - require.NoError(t, err) - - // add new token to supply - var supply []json.RawMessage - rawSupply := gjson.Get(string(genesis), "app_state.bank.supply").String() - require.NoError(t, json.Unmarshal([]byte(rawSupply), &supply)) - supply = append(supply, json.RawMessage(`{"denom": "mytoken","amount": "1000000"}`)) - newSupply, err := json.Marshal(supply) - require.NoError(t, err) - genesis, err = sjson.SetRawBytes(genesis, "app_state.bank.supply", newSupply) - require.NoError(t, err) - - // add amount to any balance - anyAddr := cli.GetKeyAddr("node0") - newBalances := GetGenesisBalance(genesis, anyAddr).Add(sdk.NewInt64Coin("mytoken", 1000000)) - newBalancesBz, err := newBalances.MarshalJSON() - require.NoError(t, err) - newState, err := sjson.SetRawBytes(genesis, fmt.Sprintf("app_state.bank.balances.#[address==%q]#.coins", anyAddr), newBalancesBz) - require.NoError(t, err) - return newState - }) - sut.StartChain(t) -``` - -Next step is to add the new token to the assert map. But we can also make it more resilient to different node counts. - -```go - exp := map[string]int64{ - "stake": int64(500000000 * sut.nodesCount), - "testtoken": int64(1000000000 * sut.nodesCount), - "mytoken": 1000000, - } -``` - -```shell -go test -mod=readonly -tags='system_test' -v ./... --run TestQueryTotalSupply --verbose --nodes-count=1 -``` - -## Part 4: Set state via TX - -Complexer workflows and tests require modifying state on a running chain. This works only with builtin logic and operations. -If we want to burn some of our new tokens, we need to submit a bank burn message to do this. -The CLI wrapper works similar to the query. Just pass the parameters. It uses the `node0` key as *default*: - -```go - // and when - txHash := cli.Run("tx", "bank", "burn", "node0", "400000mytoken") - RequireTxSuccess(t, txHash) -``` - -`RequireTxSuccess` or `RequireTxFailure` can be used to ensure the expected result of the operation. -Next, check that the changes are applied. - -```go - exp["mytoken"] = 600_000 // update expected state - raw = cli.CustomQuery("q", "bank", "total-supply") - for k, v := range exp { - got := gjson.Get(raw, fmt.Sprintf("supply.#(denom==%q).amount", k)).Int() - assert.Equal(t, v, got, raw) - } - assert.Equal(t, int64(600_000), cli.QueryBalance(cli.GetKeyAddr("node0"), "mytoken")) -``` - -While tests are still more or less readable, it can gets harder the longer they are. I found it helpful to add -some comments at the beginning to describe what the intention is. For example: - -```go - // scenario: - // given a chain with a custom token on genesis - // when an amount is burned - // then this is reflected in the total supply -``` diff --git a/tests/systemtests/go.mod b/tests/systemtests/go.mod index 6071136afd4c..f40d32b9bf70 100644 --- a/tests/systemtests/go.mod +++ b/tests/systemtests/go.mod @@ -1,33 +1,11 @@ -module cosmossdk.io/tests/systemtests +module github.com/cosmos/cosmos-sdk/tests/systemtests go 1.23 -require ( - github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect - github.com/cosmos/cosmos-sdk v0.50.6 - github.com/cosmos/gogogateway v1.2.0 // indirect - github.com/cosmos/gogoproto v1.7.0 // indirect - github.com/cosmos/iavl v1.1.4 // indirect - github.com/dvsekhvalnov/jose2go v1.6.0 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/gorilla/mux v1.8.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.20.5 // indirect - github.com/spf13/cast v1.7.0 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.10.0 - github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect - google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/grpc v1.68.0 // indirect -) - require ( cosmossdk.io/math v1.4.0 - cosmossdk.io/systemtests v0.0.0-00010101000000-000000000000 - github.com/tidwall/gjson v1.14.2 - github.com/tidwall/sjson v1.2.5 + cosmossdk.io/systemtests v0.0.0-20241126144654-14d98d277124 + github.com/cosmos/cosmos-sdk v0.50.6 ) require ( @@ -63,7 +41,11 @@ require ( github.com/cometbft/cometbft/api v1.0.0-rc.1 // indirect github.com/cosmos/btcutil v1.0.5 // indirect github.com/cosmos/cosmos-db v1.0.3-0.20240829004618-717cba019b33 // indirect + github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect + github.com/cosmos/gogogateway v1.2.0 // indirect + github.com/cosmos/gogoproto v1.7.0 // indirect + github.com/cosmos/iavl v1.1.4 // indirect github.com/cosmos/ics23/go v0.11.0 // indirect github.com/cosmos/ledger-cosmos-go v0.13.3 // indirect github.com/creachadair/tomledit v0.0.26 // indirect @@ -74,6 +56,7 @@ require ( github.com/dgraph-io/badger/v4 v4.2.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/emicklei/dot v1.6.2 // indirect github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -87,14 +70,17 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/flatbuffers v1.12.1 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/orderedcode v0.0.1 // indirect github.com/gorilla/handlers v1.5.1 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect @@ -127,7 +113,9 @@ require ( github.com/oklog/run v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect @@ -140,12 +128,19 @@ require ( github.com/sasha-s/go-deadlock v0.3.5 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.7.0 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.19.0 // indirect + github.com/stretchr/testify v1.10.0 github.com/subosito/gotenv v1.6.0 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/tendermint/go-amino v0.16.0 // indirect github.com/tidwall/btree v1.7.0 // indirect + github.com/tidwall/gjson v1.14.2 github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/sjson v1.2.5 github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v0.14.3 // indirect @@ -160,8 +155,10 @@ require ( golang.org/x/sys v0.27.0 // indirect golang.org/x/term v0.26.0 // indirect golang.org/x/text v0.20.0 // indirect + google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect + google.golang.org/grpc v1.68.0 // indirect google.golang.org/protobuf v1.35.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -170,5 +167,3 @@ require ( pgregory.net/rapid v1.1.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) - -replace cosmossdk.io/systemtests => ../../systemtests diff --git a/tests/systemtests/go.sum b/tests/systemtests/go.sum index f518ea1b9fb4..fa4b568d5a0c 100644 --- a/tests/systemtests/go.sum +++ b/tests/systemtests/go.sum @@ -16,6 +16,8 @@ cosmossdk.io/math v1.4.0 h1:XbgExXFnXmF/CccPPEto40gOO7FpWu9yWNAZPN3nkNQ= cosmossdk.io/math v1.4.0/go.mod h1:O5PkD4apz2jZs4zqFdTr16e1dcaQCc5z6lkEnrrppuk= cosmossdk.io/store v1.1.0 h1:LnKwgYMc9BInn9PhpTFEQVbL9UK475G2H911CGGnWHk= cosmossdk.io/store v1.1.0/go.mod h1:oZfW/4Fc/zYqu3JmQcQdUJ3fqu5vnYTn3LZFFy8P8ng= +cosmossdk.io/systemtests v0.0.0-20241126144654-14d98d277124 h1:pzhysmpO6+HA/vnJKVTfmtZAtCJJzLqsBZAwzOCFa7Q= +cosmossdk.io/systemtests v0.0.0-20241126144654-14d98d277124/go.mod h1:IQrj3oVhIBn1X8183/IL1z3l7HgjePVSHLeZtUUjGjI= cosmossdk.io/x/tx v0.13.3-0.20240419091757-db5906b1e894 h1:kHEvzVqpNv/9pnaEPBsgE/FMc+cVmWjSsInRufkZkpQ= cosmossdk.io/x/tx v0.13.3-0.20240419091757-db5906b1e894/go.mod h1:Tb6/tpONmtL5qFdOMdv1pdvrtJNxcazZBoz04HB71ss= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=