From 8412d3bda5119691d8186d46cb539512b80d55aa Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Thu, 14 Jul 2022 16:15:53 -0400 Subject: [PATCH] feat: mutation testing setup (#1853) Use make test-mutation Co-authored-by: Dev Ojha --- Makefile | 5 +++++ go.mod | 4 ++++ go.sum | 12 ++++++++++++ scripts/mutation-test.sh | 32 ++++++++++++++++++++++++++++++++ tools/tools.go | 1 + 5 files changed, 54 insertions(+) create mode 100644 scripts/mutation-test.sh diff --git a/Makefile b/Makefile index 766e6f39f58..6eb060d1786 100644 --- a/Makefile +++ b/Makefile @@ -237,6 +237,9 @@ test-e2e: test-e2e-skip-upgrade: @VERSION=$(VERSION) OSMOSIS_E2E_SKIP_UPGRADE=True go test -mod=readonly -timeout=25m -v $(PACKAGES_E2E) +test-mutation: + @sh scripts/mutation-test.sh + benchmark: @go test -mod=readonly -bench=. $(PACKAGES_UNIT) @@ -253,6 +256,8 @@ docker-build-e2e-init-chain: docker-build-e2e-init-node: @docker build -t osmosis-e2e-init-node:debug --build-arg E2E_SCRIPT_NAME=node -f tests/e2e/initialization/init.Dockerfile . +.PHONY: test-mutation + ############################################################################### ### Linting ### ############################################################################### diff --git a/go.mod b/go.mod index 2f7557701a7..067730b5f42 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/stretchr/testify v1.8.0 github.com/tendermint/tendermint v0.34.19 github.com/tendermint/tm-db v0.6.8-0.20220506192307-f628bb5dc95b + github.com/zimmski/go-mutesting v0.0.0-20210610104036-6d9217011a00 golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd google.golang.org/grpc v1.46.2 @@ -151,6 +152,7 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/improbable-eng/grpc-web v0.14.1 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jessevdk/go-flags v1.4.0 // indirect github.com/jgautheron/goconst v1.5.1 // indirect github.com/jingyugao/rowserrcheck v1.1.1 // indirect github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect @@ -246,6 +248,8 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/yagipy/maintidx v1.0.0 // indirect github.com/yeya24/promlinter v0.2.0 // indirect + github.com/zimmski/go-tool v0.0.0-20150119110811-2dfdc9ac8439 // indirect + github.com/zimmski/osutil v0.0.0-20190128123334-0d0b3ca231ac // indirect github.com/zondax/hid v0.9.0 // indirect gitlab.com/bosi/decorder v0.2.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect diff --git a/go.sum b/go.sum index 2b6732e6289..976dfb6b888 100644 --- a/go.sum +++ b/go.sum @@ -757,6 +757,7 @@ github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab/go.mod github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jgautheron/goconst v1.5.1 h1:HxVbL1MhydKs8R8n/HE5NPvzfaYmQJA3o879lE4+WcM= github.com/jgautheron/goconst v1.5.1/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4= @@ -796,6 +797,7 @@ github.com/julz/importas v0.1.0/go.mod h1:oSFU2R4XK/P7kNBrnL/FEQlDGN1/6WoxXEjSSX github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/errcheck v1.6.0 h1:YTDO4pNy7AUN/021p+JGHycQyYNIyMoenM1YDVK6RlY= github.com/kisielk/errcheck v1.6.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -890,6 +892,7 @@ github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRR github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/goveralls v0.0.3/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -994,6 +997,7 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= @@ -1384,6 +1388,12 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zimmski/go-mutesting v0.0.0-20210610104036-6d9217011a00 h1:KNiPkpQpqXvq40f8hh/1T7QasLJT/1MuBoOYA2vlxJk= +github.com/zimmski/go-mutesting v0.0.0-20210610104036-6d9217011a00/go.mod h1:RJt5SMnyha63GbdwCKJiX9djvvEC4KsfXJSZ5oTmSPw= +github.com/zimmski/go-tool v0.0.0-20150119110811-2dfdc9ac8439 h1:yHqsjUkj0HWbKPw/6ZqC0/eMklaRpqubA199vaRLzzE= +github.com/zimmski/go-tool v0.0.0-20150119110811-2dfdc9ac8439/go.mod h1:G4FVqCRvfz74AEB1crDNdQuvMfOoKtk7DlePsnV2yGs= +github.com/zimmski/osutil v0.0.0-20190128123334-0d0b3ca231ac h1:uiFRlKzyIzHeLOthe0ethUkSGW7POlqxU3Tc21R8QpQ= +github.com/zimmski/osutil v0.0.0-20190128123334-0d0b3ca231ac/go.mod h1:wJ9WGevuM/rw8aB2pQPFMUgXZWeaouI0ueFamR0DUPE= github.com/zondax/hid v0.9.0 h1:eiT3P6vNxAEVxXMw66eZUAAnU2zD33JBkfG/EnfAKl8= github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= gitlab.com/bosi/decorder v0.2.1 h1:ehqZe8hI4w7O4b1vgsDZw1YU1PE7iJXrQWFMsocbQ1w= @@ -1759,6 +1769,7 @@ golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190307163923-6a08e3108db3/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4= @@ -1783,6 +1794,7 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190916130336-e45ffcd953cc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191018212557-ed542cd5b28a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/scripts/mutation-test.sh b/scripts/mutation-test.sh new file mode 100644 index 00000000000..350aa0d9c7c --- /dev/null +++ b/scripts/mutation-test.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -eo pipefail + +DISABLED_MUTATORS='branch/*' + +# Only consider the following: +# * go files in types or keeper packages +# * ignore test and Protobuf files +MUTATION_SOURCES=$(find ./x -type f \( -path '*/keeper/*' -or -path '*/types/*' \) \( -name '*.go' -and -not -name '*_test.go' -and -not -name '*pb*' \)) + +# XXX: Filter on a module-by-module basis and expand when we think other modules +# are ready. Once all modules are considered stable enough to be tested, remove +# this filter entirely. +MUTATION_SOURCES=$(echo "$MUTATION_SOURCES" | grep './x/tokenfactory') + +# Collect multiple lines into a single line to be fed into go-mutesting +MUTATION_SOURCES=$(echo $MUTATION_SOURCES | tr '\n' ' ') + +OUTPUT=$(go run github.com/zimmski/go-mutesting/cmd/go-mutesting --disable=$DISABLED_MUTATORS $MUTATION_SOURCES) + +# Fetch the final result output and the overall mutation testing score +RESULT=$(echo "$OUTPUT" | grep 'The mutation score') +SCORE=$(echo "$RESULT" | grep -Eo '[[:digit:]]\.[[:digit:]]+') + +echo $RESULT + +# Return a non-zero exit code if the score is below 75% +if (( $(echo "$SCORE < 0.75" |bc -l) )); then + echo "Mutation testing score below desired level ($SCORE < 0.75)" + exit 1 +fi diff --git a/tools/tools.go b/tools/tools.go index a16035f6194..61580ec665d 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -9,5 +9,6 @@ package tools import ( _ "github.com/golangci/golangci-lint/cmd/golangci-lint" + _ "github.com/zimmski/go-mutesting/cmd/go-mutesting" _ "mvdan.cc/gofumpt" )