-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(compute/metadata): retry error when talking to metadata service (#…
…4648) This was reported internally to be causing issues. This package is used by some of our auth flows so it should be good to make this package more resilient to transient failures. Implementation inspired by what we do for some of our http based services. Because this package is currently not context aware I needed to add attempts so retrying does not happen forever. Five attempts was arbitrarily chosen. Fixes: #4642 Release-As: 0.94.0
- Loading branch information
Showing
5 changed files
with
278 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,77 @@ | ||
// Copyright 2021 Google LLC | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package metadata | ||
|
||
import ( | ||
"io" | ||
"time" | ||
|
||
"github.com/googleapis/gax-go/v2" | ||
) | ||
|
||
const ( | ||
maxRetryAttempts = 5 | ||
) | ||
|
||
var ( | ||
syscallRetryable = func(err error) bool { return false } | ||
) | ||
|
||
func newRetryer() *metadataRetryer { | ||
return &metadataRetryer{bo: &gax.Backoff{Initial: 100 * time.Millisecond}} | ||
} | ||
|
||
type backoff interface { | ||
Pause() time.Duration | ||
} | ||
|
||
type metadataRetryer struct { | ||
bo backoff | ||
attempts int | ||
} | ||
|
||
func (r *metadataRetryer) Retry(status int, err error) (time.Duration, bool) { | ||
retryOk := shouldRetry(status, err) | ||
if !retryOk { | ||
return 0, false | ||
} | ||
if r.attempts == maxRetryAttempts { | ||
return 0, false | ||
} | ||
r.attempts++ | ||
return r.bo.Pause(), true | ||
} | ||
|
||
func shouldRetry(status int, err error) bool { | ||
if 500 <= status && status <= 599 { | ||
return true | ||
} | ||
if err == io.ErrUnexpectedEOF { | ||
return true | ||
} | ||
// Transient network errors should be retried. | ||
if syscallRetryable(err) { | ||
return true | ||
} | ||
if err, ok := err.(interface{ Temporary() bool }); ok { | ||
if err.Temporary() { | ||
return true | ||
} | ||
} | ||
if err, ok := err.(interface{ Unwrap() error }); ok { | ||
return shouldRetry(status, err.Unwrap()) | ||
} | ||
return false | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
// Copyright 2021 Google LLC | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
// +build linux | ||
|
||
package metadata | ||
|
||
import "syscall" | ||
|
||
func init() { | ||
// Initialize syscallRetryable to return true on transient socket-level | ||
// errors. These errors are specific to Linux. | ||
syscallRetryable = func(err error) bool { return err == syscall.ECONNRESET || err == syscall.ECONNREFUSED } | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
// Copyright 2021 Google LLC | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
// +build linux | ||
|
||
package metadata | ||
|
||
import ( | ||
"syscall" | ||
"testing" | ||
) | ||
|
||
func TestMetadataRetryerLinux(t *testing.T) { | ||
retryer := metadataRetryer{bo: constantBackoff{}} | ||
|
||
t.Run("retry on syscall.ECONNRESET", func(t *testing.T) { | ||
_, shouldRetry := retryer.Retry(400, syscall.ECONNRESET) | ||
if !shouldRetry { | ||
t.Fatal("retryer.Retry(400, syscall.ECONNRESET) = false, want true") | ||
} | ||
}) | ||
t.Run("retry on syscall.ECONNREFUSED", func(t *testing.T) { | ||
_, shouldRetry := retryer.Retry(400, syscall.ECONNREFUSED) | ||
if !shouldRetry { | ||
t.Fatal("retryer.Retry(400, syscall.ECONNREFUSED) = false, want true") | ||
} | ||
}) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,120 @@ | ||
// Copyright 2021 Google LLC | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package metadata | ||
|
||
import ( | ||
"io" | ||
"testing" | ||
"time" | ||
) | ||
|
||
type constantBackoff struct{} | ||
|
||
func (b constantBackoff) Pause() time.Duration { return 100 } | ||
|
||
type errTemp struct{} | ||
|
||
func (e errTemp) Error() string { return "temporary error" } | ||
|
||
func (e errTemp) Temporary() bool { return true } | ||
|
||
type errWrapped struct { | ||
e error | ||
} | ||
|
||
func (e errWrapped) Error() string { return "unwrap me to get more context" } | ||
|
||
func (e errWrapped) Unwrap() error { return e.e } | ||
|
||
func TestMetadataRetryer(t *testing.T) { | ||
tests := []struct { | ||
name string | ||
code int | ||
err error | ||
wantDelay time.Duration | ||
wantShouldRetry bool | ||
}{ | ||
{ | ||
name: "retry on 500", | ||
code: 500, | ||
wantDelay: 100, | ||
wantShouldRetry: true, | ||
}, | ||
{ | ||
name: "don't retry on 400", | ||
code: 400, | ||
err: io.EOF, | ||
wantDelay: 0, | ||
wantShouldRetry: false, | ||
}, | ||
{ | ||
name: "retry on io.ErrUnexpectedEOF", | ||
code: 400, | ||
err: io.ErrUnexpectedEOF, | ||
wantDelay: 100, | ||
wantShouldRetry: true, | ||
}, | ||
{ | ||
name: "retry on temporary error", | ||
code: 400, | ||
err: errTemp{}, | ||
wantDelay: 100, | ||
wantShouldRetry: true, | ||
}, | ||
{ | ||
name: "retry on wrapped temporary error", | ||
code: 400, | ||
err: errWrapped{errTemp{}}, | ||
wantDelay: 100, | ||
wantShouldRetry: true, | ||
}, | ||
{ | ||
name: "don't retry on wrapped io.EOF", | ||
code: 400, | ||
err: errWrapped{io.EOF}, | ||
wantDelay: 0, | ||
wantShouldRetry: false, | ||
}, | ||
} | ||
|
||
for _, tc := range tests { | ||
t.Run(tc.name, func(t *testing.T) { | ||
retryer := metadataRetryer{bo: constantBackoff{}} | ||
delay, shouldRetry := retryer.Retry(tc.code, tc.err) | ||
if delay != tc.wantDelay { | ||
t.Fatalf("retryer.Retry(%v, %v) = %v, want %v", tc.code, tc.err, delay, tc.wantDelay) | ||
} | ||
if shouldRetry != tc.wantShouldRetry { | ||
t.Fatalf("retryer.Retry(%v, %v) = %v, want %v", tc.code, tc.err, shouldRetry, tc.wantShouldRetry) | ||
} | ||
}) | ||
} | ||
} | ||
|
||
func TestMetadataRetryerAttempts(t *testing.T) { | ||
retryer := metadataRetryer{bo: constantBackoff{}} | ||
for i := 1; i <= 6; i++ { | ||
_, shouldRetry := retryer.Retry(500, nil) | ||
if i == 6 { | ||
if shouldRetry { | ||
t.Fatal("an error should only be retried 5 times") | ||
} | ||
break | ||
} | ||
if !shouldRetry { | ||
t.Fatalf("retryer.Retry(500, nil) = false, want true") | ||
} | ||
} | ||
} |