From 55735791122f97b7f67cb962b489fd1f12210af5 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 21 Nov 2023 11:51:45 -0600 Subject: [PATCH] fix: ensure query job retry has longer deadline than API request deadline (#1734) In cases where we can't disambiguate API failure from job failure, this ensures we can still retry the job at least once. --- google/cloud/bigquery/retry.py | 9 +++++++-- tests/unit/test_retry.py | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/google/cloud/bigquery/retry.py b/google/cloud/bigquery/retry.py index d0830ed13..b01c0662c 100644 --- a/google/cloud/bigquery/retry.py +++ b/google/cloud/bigquery/retry.py @@ -34,7 +34,12 @@ auth_exceptions.TransportError, ) -_DEFAULT_JOB_DEADLINE = 60.0 * 10.0 # seconds +_DEFAULT_RETRY_DEADLINE = 10.0 * 60.0 # 10 minutes + +# Allow for a few retries after the API request times out. This relevant for +# rateLimitExceeded errors, which can be raised either by the Google load +# balancer or the BigQuery job server. +_DEFAULT_JOB_DEADLINE = 3.0 * _DEFAULT_RETRY_DEADLINE def _should_retry(exc): @@ -51,7 +56,7 @@ def _should_retry(exc): return reason in _RETRYABLE_REASONS -DEFAULT_RETRY = retry.Retry(predicate=_should_retry, deadline=600.0) +DEFAULT_RETRY = retry.Retry(predicate=_should_retry, deadline=_DEFAULT_RETRY_DEADLINE) """The default retry object. Any method with a ``retry`` parameter will be retried automatically, diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 60d04de89..1109b7ff2 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -125,6 +125,7 @@ def test_DEFAULT_JOB_RETRY_predicate(): def test_DEFAULT_JOB_RETRY_deadline(): - from google.cloud.bigquery.retry import DEFAULT_JOB_RETRY + from google.cloud.bigquery.retry import DEFAULT_JOB_RETRY, DEFAULT_RETRY - assert DEFAULT_JOB_RETRY._deadline == 600 + # Make sure we can retry the job at least once. + assert DEFAULT_JOB_RETRY._deadline > DEFAULT_RETRY._deadline