From 276f921dd4c01c0cf6e641694ade05504475f6ca Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 3 Jul 2023 14:43:51 -0400 Subject: [PATCH 1/2] add exponential backoff for client retry --- x/sync/client.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/x/sync/client.go b/x/sync/client.go index 02c9a24492f7..02d6f510168f 100644 --- a/x/sync/client.go +++ b/x/sync/client.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "math" "sync/atomic" "time" @@ -23,7 +24,9 @@ import ( ) const ( - failedRequestSleepInterval = 10 * time.Millisecond + initialRetryWait = 10 * time.Millisecond + maxRetryWait = time.Second + retryWaitFactor = 1.5 epsilon = 1e-6 // small amount to add to time to avoid division by 0 ) @@ -211,6 +214,11 @@ func getAndParse[T any]( lastErr = err } + retryWait := initialRetryWait * time.Duration(math.Pow(retryWaitFactor, float64(attempt))) + if retryWait > maxRetryWait || retryWait < 0 { // Handle overflows with negative check. + retryWait = maxRetryWait + } + select { case <-ctx.Done(): if lastErr != nil { @@ -221,7 +229,7 @@ func getAndParse[T any]( ) } return nil, ctx.Err() - case <-time.After(failedRequestSleepInterval): + case <-time.After(retryWait): } } } From 4ec2972d41d5e656d03215584a7622a7588076b6 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 3 Jul 2023 14:48:58 -0400 Subject: [PATCH 2/2] add comment --- x/sync/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x/sync/client.go b/x/sync/client.go index 02d6f510168f..5e4ef5c3fc58 100644 --- a/x/sync/client.go +++ b/x/sync/client.go @@ -26,7 +26,7 @@ import ( const ( initialRetryWait = 10 * time.Millisecond maxRetryWait = time.Second - retryWaitFactor = 1.5 + retryWaitFactor = 1.5 // Larger --> timeout grows more quickly epsilon = 1e-6 // small amount to add to time to avoid division by 0 )