🐛 Source Google Sheets: Retry on server errors (#13446)

* Retry on server errors * retry on any 5XX * bump version * rename function for clarity * reset * delete unused code * auto-bump connector version Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Jun 3, 2022 · 19a20f0 · 19a20f0
1 parent 88390f2
commit 19a20f0
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 9 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -343,7 +343,7 @@
 - name: Google Sheets
   sourceDefinitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7
   dockerRepository: airbyte/source-google-sheets
-  dockerImageTag: 0.2.14
+  dockerImageTag: 0.2.15
   documentationUrl: https://docs.airbyte.io/integrations/sources/google-sheets
   icon: google-sheets.svg
   sourceType: file

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -3290,7 +3290,7 @@
         oauthFlowOutputParameters:
         - - "access_token"
         - - "refresh_token"
-- dockerImage: "airbyte/source-google-sheets:0.2.14"
+- dockerImage: "airbyte/source-google-sheets:0.2.15"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/google-sheets"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-google-sheets/Dockerfile b/airbyte-integrations/connectors/source-google-sheets/Dockerfile
@@ -34,5 +34,5 @@ COPY google_sheets_source ./google_sheets_source
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.14
+LABEL io.airbyte.version=0.2.15
 LABEL io.airbyte.name=airbyte/source-google-sheets
diff --git a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/client.py b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/client.py
@@ -12,26 +12,28 @@
 from .helpers import SCOPES, Helpers
 
 
-def error_handler(error):
-    return error.resp.status != status_codes.TOO_MANY_REQUESTS
+def give_up(error):
+    code = error.resp.status
+    # Stop retrying if it's not a problem with the rate limit or on the server end
+    return not (code == status_codes.TOO_MANY_REQUESTS or 500 <= code < 600)
 
 
 class GoogleSheetsClient:
     def __init__(self, credentials: Dict[str, str], scopes: List[str] = SCOPES):
         self.client = Helpers.get_authenticated_sheets_client(credentials, scopes)
 
-    @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=error_handler)
+    @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=give_up)
     def get(self, **kwargs):
         return self.client.get(**kwargs).execute()
 
-    @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=error_handler)
+    @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=give_up)
     def create(self, **kwargs):
         return self.client.create(**kwargs).execute()
 
-    @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=error_handler)
+    @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=give_up)
     def get_values(self, **kwargs):
         return self.client.values().batchGet(**kwargs).execute()
 
-    @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=error_handler)
+    @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=give_up)
     def update_values(self, **kwargs):
         return self.client.values().batchUpdate(**kwargs).execute()
diff --git a/docs/integrations/sources/google-sheets.md b/docs/integrations/sources/google-sheets.md
@@ -103,6 +103,7 @@ The Airbyte UI will ask for two things:
 
 | Version | Date       | Pull Request                                               | Subject                                                                       |
 |---------|------------|------------------------------------------------------------|-------------------------------------------------------------------------------|
+| 0.2.15  | 2022-06-02 | [13446](https://github.com/airbytehq/airbyte/pull/13446)   | Retry requests resulting in a server error                                    |
 | 0.2.13  | 2022-05-06 | [12685](https://github.com/airbytehq/airbyte/pull/12685)   | Update CDK to v0.1.56 to emit an `AirbyeTraceMessage` on uncaught exceptions  |
 | 0.2.12  | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230)   | Update connector to use a `spec.yaml`                                         |
 | 0.2.11  | 2022-04-13 | [11977](https://github.com/airbytehq/airbyte/pull/11977)   | Replace leftover print statement with airbyte logger                          |