diff --git a/.github/workflows/linearizability.yaml b/.github/workflows/linearizability.yaml index 1a1a2feafc9f..87a92c22dfa4 100644 --- a/.github/workflows/linearizability.yaml +++ b/.github/workflows/linearizability.yaml @@ -9,7 +9,10 @@ jobs: with: go-version: "1.19.1" - run: | + pushd tools/mod; go install go.etcd.io/gofail; popd mkdir -p /tmp/linearizability + FAILPOINTS=true make build + cat server/etcdserver/raft.fail.go EXPECT_DEBUG=true GO_TEST_FLAGS=-v RESULTS_DIR=/tmp/linearizability make test-linearizability - uses: actions/upload-artifact@v2 if: always() diff --git a/Makefile b/Makefile index 4c3bcfab52b8..6f8a27f8c170 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,8 @@ test-e2e-release: build PASSES="release e2e" ./scripts/test.sh $(GO_TEST_FLAGS) .PHONY: test-linearizability -test-linearizability: build - PASSES="linearizability" ./scripts/test.sh $(GO_TEST_FLAGS) +test-linearizability: + FAILPOINTS=true PASSES="linearizability" ./scripts/test.sh $(GO_TEST_FLAGS) # Static analysis diff --git a/go.mod b/go.mod index 74956a2eb692..4686dc582081 100644 --- a/go.mod +++ b/go.mod @@ -41,13 +41,20 @@ require ( require ( cloud.google.com/go v0.81.0 // indirect github.com/VividCortex/ewma v1.1.1 // indirect + github.com/anishathalye/porcupine v0.1.2 // indirect github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect + github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 // indirect + github.com/cockroachdb/errors v1.2.4 // indirect + github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/creack/pty v1.1.11 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.13.0 // indirect + github.com/getsentry/raven-go v0.2.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -68,6 +75,7 @@ require ( github.com/mattn/go-runewidth v0.0.12 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.12.2 // indirect github.com/prometheus/client_model v0.2.0 // indirect diff --git a/go.sum b/go.sum index 70338cde9b5f..c53270f72f89 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/anishathalye/porcupine v0.1.2 h1:eqWNeLcnTzXt6usipDJ4RFn6XOWqY5wEqBYVG3yFLSE= +github.com/anishathalye/porcupine v0.1.2/go.mod h1:/X9OQYnVb7DzfKCQVO4tI1Aq+o56UJW+RvN/5U4EuZA= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -60,7 +62,9 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= @@ -81,13 +85,18 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -106,6 +115,7 @@ github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -124,6 +134,7 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= @@ -227,6 +238,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -545,6 +557,7 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= diff --git a/scripts/build_lib.sh b/scripts/build_lib.sh index ad5c732a5491..2bbbce8d3562 100755 --- a/scripts/build_lib.sh +++ b/scripts/build_lib.sh @@ -19,6 +19,22 @@ toggle_failpoints() { mode="$1" if command -v gofail >/dev/null 2>&1; then run gofail "$mode" server/etcdserver/ server/storage/backend/ + ( + cd ./server + run go get go.etcd.io/gofail/runtime + ) || exit 2 + ( + cd ./etcdutl + run go get go.etcd.io/gofail/runtime + ) || exit 2 + ( + cd ./etcdctl + run go get go.etcd.io/gofail/runtime + ) || exit 2 + ( + cd ./tests + run go get go.etcd.io/gofail/runtime + ) || exit 2 elif [[ "$mode" != "disable" ]]; then log_error "FAILPOINTS set but gofail not found" exit 1 diff --git a/tests/framework/e2e/cluster.go b/tests/framework/e2e/cluster.go index 5096df714f51..ff57f35ebee4 100644 --- a/tests/framework/e2e/cluster.go +++ b/tests/framework/e2e/cluster.go @@ -182,6 +182,7 @@ type EtcdProcessClusterConfig struct { CorruptCheckTime time.Duration CompactHashCheckEnabled bool CompactHashCheckTime time.Duration + GoFailEnabled bool } // NewEtcdProcessCluster launches a new cluster from etcd processes, returning @@ -388,12 +389,20 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in if cfg.CompactHashCheckTime != 0 { args = append(args, "--experimental-compact-hash-check-time", cfg.CompactHashCheckTime.String()) } + envVars := map[string]string{} + for key, value := range cfg.EnvVars { + envVars[key] = value + } + if cfg.GoFailEnabled { + port = (i+1)*10000 + 2381 + envVars["GOFAIL_HTTP"] = fmt.Sprintf("127.0.0.1:%d", port) + } return &EtcdServerProcessConfig{ lg: cfg.Logger, ExecPath: cfg.ExecPath, Args: args, - EnvVars: cfg.EnvVars, + EnvVars: envVars, TlsArgs: cfg.TlsArgs(), DataDirPath: dataDirPath, KeepDataDir: cfg.KeepDataDir, diff --git a/tests/linearizability/failpoints.go b/tests/linearizability/failpoints.go index eddbd53c8f3a..744eb1a8d451 100644 --- a/tests/linearizability/failpoints.go +++ b/tests/linearizability/failpoints.go @@ -15,14 +15,19 @@ package linearizability import ( + "bytes" "context" + "fmt" "math/rand" + "net/http" + "time" "go.etcd.io/etcd/tests/v3/framework/e2e" ) var ( - KillFailpoint Failpoint = killFailpoint{} + KillFailpoint Failpoint = killFailpoint{} + RaftBeforeSavePanic Failpoint = goFailpoint{"etcdserver/raftBeforeSave", "panic"} ) type Failpoint interface { @@ -47,3 +52,45 @@ func (f killFailpoint) Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster } return nil } + +type goFailpoint struct { + failpoint string + payload string +} + +func (f goFailpoint) Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster) error { + err := triggerGoFailpoint(f.failpoint, f.payload) + if err != nil { + return fmt.Errorf("failed to trigger failpoint %q, err: %v", f.failpoint, err) + } + err = clus.Procs[0].Wait() + if err != nil { + return err + } + err = clus.Procs[0].Start(ctx) + if err != nil { + return err + } + return nil +} + +func triggerGoFailpoint(failpoint, payload string) error { + // TODO: Send failpoints to different members + r, err := http.NewRequest("PUT", "http://127.0.0.1:12381/"+failpoint, bytes.NewBuffer([]byte(payload))) + if err != nil { + return err + } + resp, err := httpClient.Do(r) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("bad status code: %d", resp.StatusCode) + } + return nil +} + +var httpClient = http.Client{ + Timeout: 10 * time.Millisecond, +} diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index fc4d0629db3d..1f94b1754152 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -60,6 +60,14 @@ func TestLinearizability(t *testing.T) { ClusterSize: 3, }, }, + { + name: "Issue14370", + failpoint: RaftBeforeSavePanic, + config: e2e.EtcdProcessClusterConfig{ + ClusterSize: 1, + GoFailEnabled: true, + }, + }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { diff --git a/tools/mod/go.mod b/tools/mod/go.mod index a1453f0d15f5..2cca3bdd4c7a 100644 --- a/tools/mod/go.mod +++ b/tools/mod/go.mod @@ -15,6 +15,7 @@ require ( github.com/mdempsky/unconvert v0.0.0-20200228143138-95ecdbfc0b5f github.com/mgechev/revive v1.2.1 github.com/mikefarah/yq/v4 v4.24.2 + go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0 go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116 gotest.tools/gotestsum v1.7.0 gotest.tools/v3 v3.1.0 diff --git a/tools/mod/go.sum b/tools/mod/go.sum index e17785453461..f18f73c929ba 100644 --- a/tools/mod/go.sum +++ b/tools/mod/go.sum @@ -238,6 +238,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0 h1:TcXBU/YdVROXQ7FUowVK1ih9gu2yi3YMLE+tQb9q964= +go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0/go.mod h1:bOzzUWJ5bNHifkNkoIN6Ydf/z/UPT0bYuPghFYVC8+4= go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116 h1:QQiUXlqz+d96jyNG71NE+IGTgOK6Xlhdx+PzvfbLHlQ= go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116/go.mod h1:F9kog+iVAuvPJucb1dkYcDcbV0g4uyGEHllTP5NrXiw= go.mongodb.org/mongo-driver v1.7.3 h1:G4l/eYY9VrQAK/AUgkV0koQKzQnyddnWxrd/Etf0jIs= diff --git a/tools/mod/tools.go b/tools/mod/tools.go index ccc3338eb85d..c2d1ce5819ec 100644 --- a/tools/mod/tools.go +++ b/tools/mod/tools.go @@ -34,6 +34,7 @@ import ( _ "github.com/mdempsky/unconvert" _ "github.com/mgechev/revive" _ "github.com/mikefarah/yq/v4" + _ "go.etcd.io/gofail" _ "go.etcd.io/protodoc" _ "gotest.tools/gotestsum" _ "gotest.tools/v3"