From 19a20f0efc7558cee8505470c0143def04a34ae3 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 2 Jun 2022 18:28:45 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Source=20Google=20Sheets:=20Retr?= =?UTF-8?q?y=20on=20server=20errors=20(#13446)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Retry on server errors * retry on any 5XX * bump version * rename function for clarity * reset * delete unused code * auto-bump connector version Co-authored-by: Octavia Squidington III --- .../main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-google-sheets/Dockerfile | 2 +- .../google_sheets_source/client.py | 14 ++++++++------ docs/integrations/sources/google-sheets.md | 1 + 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c2342832703b..52c435f13d80 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -343,7 +343,7 @@ - name: Google Sheets sourceDefinitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 dockerRepository: airbyte/source-google-sheets - dockerImageTag: 0.2.14 + dockerImageTag: 0.2.15 documentationUrl: https://docs.airbyte.io/integrations/sources/google-sheets icon: google-sheets.svg sourceType: file diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 6575c3232bc2..343b1e9c6113 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3290,7 +3290,7 @@ oauthFlowOutputParameters: - - "access_token" - - "refresh_token" -- dockerImage: "airbyte/source-google-sheets:0.2.14" +- dockerImage: "airbyte/source-google-sheets:0.2.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/google-sheets" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-google-sheets/Dockerfile b/airbyte-integrations/connectors/source-google-sheets/Dockerfile index 11bf0493ba19..0400c3e9914c 100644 --- a/airbyte-integrations/connectors/source-google-sheets/Dockerfile +++ b/airbyte-integrations/connectors/source-google-sheets/Dockerfile @@ -34,5 +34,5 @@ COPY google_sheets_source ./google_sheets_source ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.14 +LABEL io.airbyte.version=0.2.15 LABEL io.airbyte.name=airbyte/source-google-sheets diff --git a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/client.py b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/client.py index 63aa1893fb5e..de3562f17460 100644 --- a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/client.py +++ b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/client.py @@ -12,26 +12,28 @@ from .helpers import SCOPES, Helpers -def error_handler(error): - return error.resp.status != status_codes.TOO_MANY_REQUESTS +def give_up(error): + code = error.resp.status + # Stop retrying if it's not a problem with the rate limit or on the server end + return not (code == status_codes.TOO_MANY_REQUESTS or 500 <= code < 600) class GoogleSheetsClient: def __init__(self, credentials: Dict[str, str], scopes: List[str] = SCOPES): self.client = Helpers.get_authenticated_sheets_client(credentials, scopes) - @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=error_handler) + @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=give_up) def get(self, **kwargs): return self.client.get(**kwargs).execute() - @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=error_handler) + @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=give_up) def create(self, **kwargs): return self.client.create(**kwargs).execute() - @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=error_handler) + @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=give_up) def get_values(self, **kwargs): return self.client.values().batchGet(**kwargs).execute() - @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=error_handler) + @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=give_up) def update_values(self, **kwargs): return self.client.values().batchUpdate(**kwargs).execute() diff --git a/docs/integrations/sources/google-sheets.md b/docs/integrations/sources/google-sheets.md index e7b827869ace..6d53a3b94eda 100644 --- a/docs/integrations/sources/google-sheets.md +++ b/docs/integrations/sources/google-sheets.md @@ -103,6 +103,7 @@ The Airbyte UI will ask for two things: | Version | Date | Pull Request | Subject | |---------|------------|------------------------------------------------------------|-------------------------------------------------------------------------------| +| 0.2.15 | 2022-06-02 | [13446](https://github.com/airbytehq/airbyte/pull/13446) | Retry requests resulting in a server error | | 0.2.13 | 2022-05-06 | [12685](https://github.com/airbytehq/airbyte/pull/12685) | Update CDK to v0.1.56 to emit an `AirbyeTraceMessage` on uncaught exceptions | | 0.2.12 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml` | | 0.2.11 | 2022-04-13 | [11977](https://github.com/airbytehq/airbyte/pull/11977) | Replace leftover print statement with airbyte logger |