BW-1320 Finite retry `User project specified in the request is invali…

…d` (#6807)
broadinstitute · Jul 19, 2022 · fd5a6b5 · fd5a6b5
1 parent 293f233
commit fd5a6b5
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,9 @@
 
  * Restored missing example configuration file
  * Upgraded to latest version of the Google Cloud Storage NIO library (0.124.8)
+ * Cromwell will now finitely retry the following Google Cloud Storage I/O error.
+   * Response code `400` bad request, message `User project specified in the request is invalid`
+   * The default retry count is `5` and may be customized with `system.io.number-of-attempts`.
 
 ## 81 Release Notes
 

diff --git a/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala b/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala
@@ -11,8 +11,8 @@ import javax.net.ssl.SSLException
 object RetryableRequestSupport {
 
   /**
-    * Failures that are considered retryable.
-    * Retrying them should increase the "retry counter"
+    * Failures that are considered retryable. Retrying them increases the retry counter.
+    * The default count is `5` and may be customized with `system.io.number-of-attempts`.
     */
   def isRetryable(failure: Throwable): Boolean = failure match {
     case gcs: StorageException => gcs.isRetryable ||
@@ -47,7 +47,9 @@ object RetryableRequestSupport {
 
   // Error messages not included in the list of built-in GCS retryable errors (com.google.cloud.storage.StorageException) but that we still want to retry
   private val AdditionalRetryableErrorMessages = List(
-    "Connection closed prematurely"
+    "Connection closed prematurely",
+    // This is a 400 "non-retryable" error that is nevertheless sporadic and succeeds on subsequent re-runs with identical parameters [BW-1320]
+    "User project specified in the request is invalid"
   ).map(_.toLowerCase)
 
   /**

diff --git a/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala b/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala
@@ -253,6 +253,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I
       new StorageException(504, "message"),
       new StorageException(408, "message"),
       new StorageException(429, "message"),
+      new StorageException(400, "User project specified in the request is invalid"),
       BatchFailedException(new Exception),
       ChecksumFailedException("message"),
       new SocketException(),