From f5b793d37f4f735136e8e0db8f0bba8a3557f32a Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Wed, 4 Jan 2023 13:53:46 +0100 Subject: [PATCH] Source Snapchat Marketing: permission error (#20865) * Source Snapchat Marketing: permission error * Source Snapchat Marketing: add test; docs update * auto-bump connector version Co-authored-by: Octavia Squidington III --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-snapchat-marketing/Dockerfile | 2 +- .../source_snapchat_marketing/source.py | 8 ++++++++ .../source-snapchat-marketing/unit_tests/unit_test.py | 9 +++++++++ docs/integrations/sources/snapchat-marketing.md | 1 + 6 files changed, 21 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 394a4d6ec3bb..701ebdf4a1a1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1530,7 +1530,7 @@ - name: Snapchat Marketing sourceDefinitionId: 200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b dockerRepository: airbyte/source-snapchat-marketing - dockerImageTag: 0.1.10 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.com/integrations/sources/snapchat-marketing icon: snapchat.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 03506b4e1429..7296f4db22a7 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -13617,7 +13617,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-snapchat-marketing:0.1.10" +- dockerImage: "airbyte/source-snapchat-marketing:0.1.11" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/snapchat-marketing" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile b/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile index 33c8bb7fa56f..bd07a631cad4 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile @@ -25,5 +25,5 @@ COPY source_snapchat_marketing ./source_snapchat_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-snapchat-marketing diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py index 118807c4dd37..0da2d403ae13 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py @@ -161,6 +161,7 @@ def get_parent_ids(parent) -> List: class SnapchatMarketingStream(HttpStream, ABC): url_base = "https://adsapi.snapchat.com/v1/" primary_key = "id" + raise_on_http_errors = True def __init__(self, start_date, end_date, **kwargs): super().__init__(**kwargs) @@ -213,6 +214,13 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp raise SnapchatMarketingException(error_text) yield resp.get(self.response_item_name) + def should_retry(self, response: requests.Response) -> bool: + if response.status_code == 403: + setattr(self, "raise_on_http_errors", False) + self.logger.warning(f"Got permission error when accessing URL {response.request.url}. " f"Skipping {self.name} stream.") + return False + return super().should_retry(response) + class IncrementalSnapchatMarketingStream(SnapchatMarketingStream, ABC): cursor_field = "updated_at" diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/unit_test.py index 104f6cc180e7..8671c7ce0883 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/unit_test.py @@ -387,3 +387,12 @@ def test_retry_get_access_token(requests_mock): token = auth.get_access_token() assert len(requests_mock.request_history) == 3 assert token == "token" + + +def test_should_retry_403_error(requests_mock): + requests_mock.register_uri("GET", "https://adsapi.snapchat.com/v1/me/organizations", + [{"status_code": 403, "json": {"organizations": []}}]) + stream = Organizations(**config_mock) + records = list(stream.read_records(sync_mode=SyncMode.full_refresh)) + + assert not records diff --git a/docs/integrations/sources/snapchat-marketing.md b/docs/integrations/sources/snapchat-marketing.md index 602babf6a0e5..8eb452c230e8 100644 --- a/docs/integrations/sources/snapchat-marketing.md +++ b/docs/integrations/sources/snapchat-marketing.md @@ -113,6 +113,7 @@ Snapchat Marketing API has limitations to 1000 items per page. | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------| +| 0.1.11 | 2022-12-23 | [20865](https://github.com/airbytehq/airbyte/pull/20865) | Handle 403 permission error | | 0.1.10 | 2022-12-15 | [20537](https://github.com/airbytehq/airbyte/pull/20537) | Run on CDK 0.15.0 | | 0.1.9 | 2022-12-14 | [20498](https://github.com/airbytehq/airbyte/pull/20498) | Fix output state when no records are read | | 0.1.8 | 2022-10-05 | [17596](https://github.com/airbytehq/airbyte/pull/17596) | Retry 429 and 5xx errors when refreshing access token |