From 9ca3c6910d9ba35cfcf44fa8c01b3023fdedc92e Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Mon, 18 Jul 2022 10:57:53 +0300 Subject: [PATCH 1/9] check_task_permission added Signed-off-by: Sergey Chvalyuk --- .../source-facebook-marketing/Dockerfile | 2 +- .../streams/streams.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index 3b4f866c6eaf..112a1e94b21f 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.54 +LABEL io.airbyte.version=0.2.55 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py index 27e636b66a93..a1dd43460d5b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py @@ -12,6 +12,7 @@ from cached_property import cached_property from facebook_business.adobjects.abstractobject import AbstractObject from facebook_business.adobjects.adaccount import AdAccount as FBAdAccount +from facebook_business.adobjects.user import User from .base_insight_streams import AdsInsights from .base_streams import FBMarketingIncrementalStream, FBMarketingReversedIncrementalStream, FBMarketingStream @@ -149,6 +150,24 @@ class AdAccount(FBMarketingStream): use_batch = False enable_deleted = False + def check_task_permission(self, name: str) -> bool: + me = User(fbid="me", api=self._api.api) + for business in me.get_businesses(): + assigned_users = self._api.account.get_assigned_users(params={"business": business.get_id()}) + for assigned_user in assigned_users: + if name in assigned_user["tasks"]: + return True + return False + + @cached_property + def fields(self) -> List[str]: + """List of fields that we want to query, for now just all properties from stream's schema""" + + properties = list(self.get_json_schema().get("properties", {}).keys()) + if "funding_source_details" in properties and not self.check_task_permission("MANAGE"): + properties.remove("funding_source_details") + return properties + def list_objects(self, params: Mapping[str, Any]) -> Iterable: """noop in case of AdAccount""" return [FBAdAccount(self._api.account.get_id())] From 9c22dba7e20f1e72c4b05d6da5448892c78a08ba Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Mon, 18 Jul 2022 11:03:47 +0300 Subject: [PATCH 2/9] facebook-marketing.md updated Signed-off-by: Sergey Chvalyuk --- docs/integrations/sources/facebook-marketing.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 3c55348b68fd..4945f09e681a 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -120,7 +120,8 @@ Please be informed that the connector uses the `lookback_window` parameter to pe | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.2.54 | 2022-06-29 | [14267](https://github.com/airbytehq/airbyte/pull/14267) | Make MAX_BATCH_SIZE available in config | +| 0.2.55 | 2022-07-18 | [14786](https://github.com/airbytehq/airbyte/pull/14786) | Check "MANAGE" task permission | +| 0.2.54 | 2022-06-29 | [14267](https://github.com/airbytehq/airbyte/pull/14267) | Make MAX_BATCH_SIZE available in config | | 0.2.53 | 2022-06-16 | [13623](https://github.com/airbytehq/airbyte/pull/13623) | Add fields `bid_amount` `bid_strategy` `bid_constraints` to `ads_set` stream | | 0.2.52 | 2022-06-14 | [13749](https://github.com/airbytehq/airbyte/pull/13749) | Fix the `not syncing any data` issue | | 0.2.51 | 2022-05-30 | [13317](https://github.com/airbytehq/airbyte/pull/13317) | Change tax_id to string (Canadian has letter in tax_id) | From afb45dc9b5595bfded6e8c66fcbfebbc9d5a8fb7 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 19 Jul 2022 09:00:08 +0300 Subject: [PATCH 3/9] fix check_task_permission -> get_task_permissions Signed-off-by: Sergey Chvalyuk --- .../streams/streams.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py index a1dd43460d5b..c4af85040e1d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py @@ -4,7 +4,7 @@ import base64 import logging -from typing import Any, Iterable, List, Mapping, Optional +from typing import Any, Iterable, List, Mapping, Optional, Set import pendulum import requests @@ -150,21 +150,22 @@ class AdAccount(FBMarketingStream): use_batch = False enable_deleted = False - def check_task_permission(self, name: str) -> bool: + def get_task_permissions(self) -> Set[str]: + res = set() me = User(fbid="me", api=self._api.api) for business in me.get_businesses(): assigned_users = self._api.account.get_assigned_users(params={"business": business.get_id()}) for assigned_user in assigned_users: - if name in assigned_user["tasks"]: - return True - return False + res.update(set(assigned_user["tasks"])) + return res @cached_property def fields(self) -> List[str]: - """List of fields that we want to query, for now just all properties from stream's schema""" - - properties = list(self.get_json_schema().get("properties", {}).keys()) - if "funding_source_details" in properties and not self.check_task_permission("MANAGE"): + properties = super().fields + # https://developers.facebook.com/docs/marketing-apis/guides/javascript-ads-dialog-for-payments/ + # To access "funding_source_details", the user making the API call must have a MANAGE task permission for + # that specific ad account. + if "funding_source_details" in properties and "MANAGE" not in self.get_task_permissions(): properties.remove("funding_source_details") return properties From 4c0b7a05ff5ac78f93c2778995681d4b0ffc9144 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 19 Jul 2022 09:10:12 +0300 Subject: [PATCH 4/9] test_common_error_retry fixed Signed-off-by: Sergey Chvalyuk --- .../source-facebook-marketing/unit_tests/test_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py index 91a9077c28f9..30dd365373a6 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py @@ -124,6 +124,7 @@ def test_common_error_retry(self, error_response, requests_mock, api, account_id }, ] + requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/me/businesses", json={"data": []}) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", responses) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{account_data['id']}/", responses) From 5e15d5ed5267b7ee404a27390ad10144dafe6c9e Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 19 Jul 2022 10:46:02 +0300 Subject: [PATCH 5/9] improve get_task_permissions Signed-off-by: Sergey Chvalyuk --- .../source_facebook_marketing/streams/streams.py | 9 ++++++--- .../source-facebook-marketing/unit_tests/test_client.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py index c4af85040e1d..35984b82c2ee 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py @@ -151,12 +151,15 @@ class AdAccount(FBMarketingStream): enable_deleted = False def get_task_permissions(self) -> Set[str]: + """https://developers.facebook.com/docs/marketing-api/reference/ad-account/assigned_users/""" res = set() me = User(fbid="me", api=self._api.api) - for business in me.get_businesses(): - assigned_users = self._api.account.get_assigned_users(params={"business": business.get_id()}) + business_users = me.get_business_users() + for business_user in business_users: + assigned_users = self._api.account.get_assigned_users(params={"business": business_user["business"].get_id()}) for assigned_user in assigned_users: - res.update(set(assigned_user["tasks"])) + if business_user.get_id() == assigned_user.get_id(): + res.update(set(assigned_user["tasks"])) return res @cached_property diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py index 30dd365373a6..600b27bca652 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py @@ -124,7 +124,7 @@ def test_common_error_retry(self, error_response, requests_mock, api, account_id }, ] - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/me/businesses", json={"data": []}) + requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/me/business_users", json={"data": []}) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", responses) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{account_data['id']}/", responses) From 1a1237963b0656a41b925f211e7888797e3e89f6 Mon Sep 17 00:00:00 2001 From: Serhii Chvaliuk Date: Tue, 19 Jul 2022 10:46:36 +0300 Subject: [PATCH 6/9] Update docs/integrations/sources/facebook-marketing.md Co-authored-by: Sherif A. Nada --- docs/integrations/sources/facebook-marketing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 4945f09e681a..ed1a66b5038a 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -120,7 +120,7 @@ Please be informed that the connector uses the `lookback_window` parameter to pe | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.2.55 | 2022-07-18 | [14786](https://github.com/airbytehq/airbyte/pull/14786) | Check "MANAGE" task permission | +| 0.2.55 | 2022-07-18 | [14786](https://github.com/airbytehq/airbyte/pull/14786) | Check if the authorized user has the "MANAGE" task permission when getting the `funding_source_details` field in the ad_account stream | | 0.2.54 | 2022-06-29 | [14267](https://github.com/airbytehq/airbyte/pull/14267) | Make MAX_BATCH_SIZE available in config | | 0.2.53 | 2022-06-16 | [13623](https://github.com/airbytehq/airbyte/pull/13623) | Add fields `bid_amount` `bid_strategy` `bid_constraints` to `ads_set` stream | | 0.2.52 | 2022-06-14 | [13749](https://github.com/airbytehq/airbyte/pull/13749) | Fix the `not syncing any data` issue | From d01f64efd2e502aa4921883001f6b4a0d7a115c1 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 19 Jul 2022 10:49:13 +0300 Subject: [PATCH 7/9] lint doc Signed-off-by: Sergey Chvalyuk --- docs/integrations/sources/facebook-marketing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index ed1a66b5038a..835eebe0115e 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -120,7 +120,7 @@ Please be informed that the connector uses the `lookback_window` parameter to pe | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.2.55 | 2022-07-18 | [14786](https://github.com/airbytehq/airbyte/pull/14786) | Check if the authorized user has the "MANAGE" task permission when getting the `funding_source_details` field in the ad_account stream | +| 0.2.55 | 2022-07-18 | [14786](https://github.com/airbytehq/airbyte/pull/14786) | Check if the authorized user has the "MANAGE" task permission when getting the `funding_source_details` field in the ad\_account stream | | 0.2.54 | 2022-06-29 | [14267](https://github.com/airbytehq/airbyte/pull/14267) | Make MAX_BATCH_SIZE available in config | | 0.2.53 | 2022-06-16 | [13623](https://github.com/airbytehq/airbyte/pull/13623) | Add fields `bid_amount` `bid_strategy` `bid_constraints` to `ads_set` stream | | 0.2.52 | 2022-06-14 | [13749](https://github.com/airbytehq/airbyte/pull/13749) | Fix the `not syncing any data` issue | From 71c37fab3545df07e8ac16ad0136599892581776 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 19 Jul 2022 11:51:29 +0300 Subject: [PATCH 8/9] simplify code ;) Signed-off-by: Sergey Chvalyuk --- .../source_facebook_marketing/streams/streams.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py index 35984b82c2ee..6bf55c29dc72 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py @@ -154,8 +154,7 @@ def get_task_permissions(self) -> Set[str]: """https://developers.facebook.com/docs/marketing-api/reference/ad-account/assigned_users/""" res = set() me = User(fbid="me", api=self._api.api) - business_users = me.get_business_users() - for business_user in business_users: + for business_user in me.get_business_users(): assigned_users = self._api.account.get_assigned_users(params={"business": business_user["business"].get_id()}) for assigned_user in assigned_users: if business_user.get_id() == assigned_user.get_id(): From a8728a819a1a287583c72c683d57398c0cad5f22 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 19 Jul 2022 15:20:22 +0000 Subject: [PATCH 9/9] auto-bump connector version [ci skip] --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 38ea9f444e62..f13da31cbdac 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -248,7 +248,7 @@ - name: Facebook Marketing sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c dockerRepository: airbyte/source-facebook-marketing - dockerImageTag: 0.2.54 + dockerImageTag: 0.2.55 documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing icon: facebook.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index edaac71d4402..d9b9d43b4393 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1827,7 +1827,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-facebook-marketing:0.2.54" +- dockerImage: "airbyte/source-facebook-marketing:0.2.55" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" @@ -2163,7 +2163,7 @@ title: "Maximum size of Batched Requests" description: "Maximum batch size used when sending batch requests to Facebook\ \ API. Most users do not need to set this field unless they specifically\ - \ need to tune the connector to address specific issues or use cases." + \ need to tune the connector to address specific issues or use cases." default: 50 order: 9 exclusiveMinimum: 0