🐛 Source Facebook Marketing: reduce request limit after specific error (

airbytehq#18734) * 🐛 Source Facebook Marketing: reduce request limit after specific error * 🐛 Source Facebook Marketing: bump version; update docs * 🐛 Source Facebook Marketing: add test * 🐛 Source Facebook Marketing: increase timeout
drewrasm · Nov 2, 2022 · b1ec6d5 · b1ec6d5
1 parent e494879
commit b1ec6d5
Show file tree

Hide file tree

Showing 8 changed files with 42 additions and 6 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -336,7 +336,7 @@
 - name: Facebook Marketing
   sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
   dockerRepository: airbyte/source-facebook-marketing
-  dockerImageTag: 0.2.70
+  dockerImageTag: 0.2.71
   documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing
   icon: facebook.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -2873,7 +2873,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-facebook-marketing:0.2.70"
+- dockerImage: "airbyte/source-facebook-marketing:0.2.71"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/facebook-marketing"
     changelogUrl: "https://docs.airbyte.com/integrations/sources/facebook-marketing"

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile
@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=0.2.70
+LABEL io.airbyte.version=0.2.71
 LABEL io.airbyte.name=airbyte/source-facebook-marketing
diff --git a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml
@@ -14,6 +14,7 @@ tests:
   basic_read:
     - config_path: "secrets/config.json"
       empty_streams: ["videos"]
+      timeout_seconds: 2400
   incremental:
     - config_path: "secrets/config.json"
       timeout_seconds: 2400

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/setup.py b/airbyte-integrations/connectors/source-facebook-marketing/setup.py
@@ -6,7 +6,7 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
-    "airbyte-cdk~=0.1",
+    "airbyte-cdk~=0.2",
     "cached_property==1.5.2",
     "facebook_business==15.0.0",
     "pendulum>=2,<3",

diff --git a/...grations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py b/...grations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py
@@ -33,6 +33,15 @@ def log_retry_attempt(details):
         logger.info(str(exc))
         logger.info(f"Caught retryable error after {details['tries']} tries. Waiting {details['wait']} more seconds then retrying...")
 
+    def reduce_request_record_limit(details):
+        _, exc, _ = sys.exc_info()
+        if (
+            details.get("kwargs", {}).get("params", {}).get("limit")
+            and exc.http_status() == http.client.INTERNAL_SERVER_ERROR
+            and exc.api_error_message() == "Please reduce the amount of data you're asking for, then retry your request"
+        ):
+            details["kwargs"]["params"]["limit"] = int(int(details["kwargs"]["params"]["limit"]) / 2)
+
     def should_retry_api_error(exc):
         if isinstance(exc, FacebookRequestError):
             call_rate_limit_error = exc.api_error_code() in FACEBOOK_RATE_LIMIT_ERROR_CODES
@@ -58,7 +67,7 @@ def should_retry_api_error(exc):
         backoff_type,
         exception,
         jitter=None,
-        on_backoff=log_retry_attempt,
+        on_backoff=[log_retry_attempt, reduce_request_record_limit],
         giveup=lambda exc: not should_retry_api_error(exc),
         **wait_gen_kwargs,
     )

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py
@@ -38,6 +38,18 @@ def fb_call_rate_response_fixture():
     }
 
 
+@pytest.fixture(name="fb_call_amount_data_response")
+def fb_call_amount_data_response_fixture():
+    error = {"message": "Please reduce the amount of data you're asking for, then retry your request", "code": 1}
+
+    return {
+        "json": {
+            "error": error,
+        },
+        "status_code": 500,
+    }
+
+
 class TestBackoff:
     def test_limit_reached(self, mocker, requests_mock, api, fb_call_rate_response, account_id):
         """Error once, check that we retry and not fail"""
@@ -132,3 +144,16 @@ def test_common_error_retry(self, error_response, requests_mock, api, account_id
         accounts = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}))
 
         assert accounts == [account_data]
+
+    def test_limit_error_retry(self, fb_call_amount_data_response, requests_mock, api, account_id):
+        """Error every time, check limit parameter decreases by 2 times every new call"""
+
+        res = requests_mock.register_uri(
+            "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/campaigns", [fb_call_amount_data_response]
+        )
+
+        stream = Campaigns(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100)
+        try:
+            list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}))
+        except FacebookRequestError:
+            assert [x.qs.get("limit")[0] for x in res.request_history] == ["100", "50", "25", "12", "6"]
diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md
@@ -127,7 +127,8 @@ Please be informed that the connector uses the `lookback_window` parameter to pe
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                                                                                                                                           |
 |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| 0.2.70  | 2022-10-26 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Upgrade FB SDK to v15.0 |
+| 0.2.71  | 2022-10-31 | [18734](https://github.com/airbytehq/airbyte/pull/18734) | Reduce request record limit on retry                                                                                                                                                                                                                                                              |
+| 0.2.70  | 2022-10-26 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Upgrade FB SDK to v15.0                                                                                                                                                                                                                                                                           |
 | 0.2.69  | 2022-10-17 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Remove "pixel" field from the Custom Conversions stream schema                                                                                                                                                                                                                                    |
 | 0.2.68  | 2022-10-12 | [17869](https://github.com/airbytehq/airbyte/pull/17869) | Remove "format" from optional datetime `end_date` field                                                                                                                                                                                                                                           |
 | 0.2.67  | 2022-10-04 | [17551](https://github.com/airbytehq/airbyte/pull/17551) | Add `cursor_field` for custom_insights stream schema                                                                                                                                                                                                                                              |