Skip to content

Commit

Permalink
Fix cosmos tests workflow
Browse files Browse the repository at this point in the history
- add parallel execution
- add simulation tests
  • Loading branch information
ivivanov committed Feb 2, 2023
1 parent 9badcef commit b3f6858
Show file tree
Hide file tree
Showing 2 changed files with 188 additions and 43 deletions.
158 changes: 149 additions & 9 deletions .github/workflows/test-cosmos.yaml
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
name: Test Cosmos
name: Test cosmos-sdk
on:
schedule:
- cron: '0 3 * * *'
- cron: "0 3 * * *"

permissions:
contents: read

concurrency:
group: ci-${{ github.ref }}-tests
cancel-in-progress: true

jobs:
test-unit-cosmos:
name: Run cosmos-sdk tests
split-test-files:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v3
Expand All @@ -14,10 +20,144 @@ jobs:

- uses: actions/checkout@v3

- name: Pull project dependencies
run: go mod tidy
- name: Create a file with all core Cosmos SDK pkgs
run: |
go mod download
COSMOSSDK_DIR=$(go list -m -f '{{.Dir}}' github.com/cosmos/cosmos-sdk)
echo "Cosmos SDK Path: $COSMOSSDK_DIR"
go list $COSMOSSDK_DIR/... > pkgs.txt
- name: Split pkgs into 4 files
run: split -d -n l/4 pkgs.txt pkgs.txt.part.

- uses: actions/upload-artifact@v3
with:
name: "${{ github.sha }}-00"
path: ./pkgs.txt.part.00

- uses: actions/upload-artifact@v3
with:
name: "${{ github.sha }}-01"
path: ./pkgs.txt.part.01

- uses: actions/upload-artifact@v3
with:
name: "${{ github.sha }}-02"
path: ./pkgs.txt.part.02

- uses: actions/upload-artifact@v3
with:
name: "${{ github.sha }}-03"
path: ./pkgs.txt.part.03

tests:
runs-on: ubuntu-latest
needs: split-test-files

- name: Run cosmos-sdk tests
shell: bash
strategy:
fail-fast: false
matrix:
part: ["00", "01", "02", "03"]

steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: 1.18

- uses: technote-space/get-diff-action@v6.1.2
id: git_diff
with:
PATTERNS: |
**/*.go
go.mod
go.sum
**/go.mod
**/go.sum
- uses: actions/download-artifact@v3
with:
name: "${{ github.sha }}-${{ matrix.part }}"

- name: test
if: env.GIT_DIFF
run: |
./scripts/test/run-test-unit-cosmos.sh >&2
./scripts/test/run-test-unit-cosmos.sh pkgs.txt.part.${{ matrix.part }}
test-sim-nondeterminism:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: 1.18

- uses: technote-space/get-diff-action@v6.1.2
id: git_diff
with:
PATTERNS: |
**/*.go
go.mod
go.sum
**/go.mod
**/go.sum
**/Makefile
Makefile
- name: test-sim-nondeterminism
if: env.GIT_DIFF
run: |
go test github.com/cosmos/cosmos-sdk/simapp -run TestAppStateDeterminism -Enabled=true -NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -v -timeout 24h
# TODO: run integration tests when we upgrade cosmos-sdk to 0.46+
# test-integration:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# - uses: actions/setup-go@v3
# with:
# go-version: 1.18

# - uses: technote-space/get-diff-action@v6.1.2
# id: git_diff
# with:
# PATTERNS: |
# **/*.go
# go.mod
# go.sum
# **/go.mod
# **/go.sum
# **/Makefile
# Makefile

# - name: integration tests
# if: env.GIT_DIFF
# run: |
# go test -mod=readonly github.com/cosmos/cosmos-sdk/integration/... -timeout 30m

# TODO: run e2e tests when we upgrade cosmos-sdk to 0.46+
# test-e2e:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# - uses: actions/setup-go@v3
# with:
# go-version: 1.18

# - uses: technote-space/get-diff-action@v6.1.2
# id: git_diff
# with:
# PATTERNS: |
# **/*.go
# go.mod
# go.sum
# **/go.mod
# **/go.sum
# **/Makefile
# Makefile

# - name: e2e tests
# if: env.GIT_DIFF
# run: |
# go test github.com/cosmos/cosmos-sdk/e2e/... -mod=readonly -timeout 30m -race -tags='e2e'
73 changes: 39 additions & 34 deletions scripts/test/run-test-unit-cosmos.sh
Original file line number Diff line number Diff line change
@@ -1,50 +1,55 @@
#!/bin/bash
set -euo pipefail

COSMOSSDK_DIR=$(go list -m -f '{{.Dir}}' github.com/cosmos/cosmos-sdk)
echo "Cosmos SDK Path: $COSMOSSDK_DIR"

# Usage: skip_test "$testFoo" "$packageBar"
# Returns TEST_LIST containing all the tests to be executed
# To skip it, we need to:
# 1. Retrieve all the tests in the same package
# 2. Remove the test we want to skip
# 3. Format the list of remaining tests as a regex in the form 'TestName|TestName|...'
# 4. Pass the list of tests to the go test command with the -run flag
#
#
# There is no easier way to skip specific tests with go test currently.
# See: https://github.com/golang/go/issues/41583
skip_test() {
TEST_LIST="$(go test -list '.*' $2 |
grep -v $1 |
sed '$d' |
sed ':a; /$/N; s/\n/|/; ta')"
grep -v $1 |
sed '$d' |
sed ':a; /$/N; s/\n/|/; ta')"

if [ -z "$TEST_LIST" ]; then
TEST_LIST="skip all"
fi
}

if [ -z "$COSMOSSDK_DIR" ]; then
echo "There is no Cosmos SDK"
else
COSMOSSDK_PACKAGES=$(go list $COSMOSSDK_DIR/... | uniq)
# COSMOSSDK_PACKAGES="github.com/cosmos/cosmos-sdk/server github.com/cosmos/cosmos-sdk/x/upgrade/types"
for PACKAGE in $COSMOSSDK_PACKAGES; do
TEST_LIST=""

# Skip the TestInterceptConfigsWithBadPermissions test, as it fails when running the test as the root user
if [ "$PACKAGE" == "github.com/cosmos/cosmos-sdk/server" ]; then
skip_test "TestInterceptConfigsWithBadPermissions" $PACKAGE
fi

# Skip because TestIntegrationTestSuite/TestBroadcastTx_GRPCGateway/valid_request fails
if [ "$PACKAGE" == "github.com/cosmos/cosmos-sdk/x/auth/tx" ]; then
skip_test "TestIntegrationTestSuite" $PACKAGE
fi

# skip entire package due to build error
if [ "$PACKAGE" == "github.com/cosmos/cosmos-sdk/server/grpc" ]; then
continue
fi

echo "go test -mod=readonly -tags='cgo ledger test_ledger_mock norace' -run=$TEST_LIST $PACKAGE"
go test -mod=readonly -tags='cgo ledger test_ledger_mock norace' -run=$TEST_LIST $PACKAGE
done
fi
PKG_LIST=$(cat $1)

for PACKAGE in $PKG_LIST; do
TEST_LIST=""

# skip entire package due to build error. Also there are no tests in it
if [ "$PACKAGE" == "github.com/cosmos/cosmos-sdk/server/grpc" ]; then
continue
fi

# failed to initialize database: open /tmp/Test_runMigrateCmd3518174278/001/keys/keys.db/LOCK: permission denied
if [ "$PACKAGE" == "github.com/cosmos/cosmos-sdk/client/keys" ]; then
skip_test "Test_runMigrateCmd" $PACKAGE
fi

# failed to initialize database: open /tmp/TestLegacyKeybase2255353028/001/keys/keys.db/LOCK: permission denied
if [ "$PACKAGE" == "github.com/cosmos/cosmos-sdk/crypto/keyring" ]; then
skip_test "TestLegacyKeybase" $PACKAGE
fi

# Expected nil, but got: &fs.PathError{Op:"open", Path:".touch", Err:0xd}
if [ "$PACKAGE" == "github.com/cosmos/cosmos-sdk/store/streaming" ]; then
skip_test "TestStreamingServiceConstructor" $PACKAGE
fi

if [ "$TEST_LIST" == "skip all" ]; then
echo "No tests in package: $PACKAGE"
else
go test -mod=readonly -tags='ledger test_ledger_mock' -race -timeout 30m -run=$TEST_LIST $PACKAGE
fi
done

0 comments on commit b3f6858

Please sign in to comment.