diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 9ada498e36e9..86e17d43738c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -210,7 +210,7 @@ - name: Facebook Marketing sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c dockerRepository: airbyte/source-facebook-marketing - dockerImageTag: 0.2.45 + dockerImageTag: 0.2.46 documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing icon: facebook.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 3234c4165298..4feea13903b9 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1757,7 +1757,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-facebook-marketing:0.2.45" +- dockerImage: "airbyte/source-facebook-marketing:0.2.46" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" @@ -2062,6 +2062,16 @@ format: "date-time" required: - "name" + page_size: + title: "Page Size of Requests" + description: "Page size used when sending requests to Facebook API to specify\ + \ number of records per page when response has pagination. Most users\ + \ do not need to set this field unless they specifically need to tune\ + \ the connector to address specific issues or use cases." + default: 100 + order: 7 + exclusiveMinimum: 0 + type: "integer" required: - "account_id" - "start_date" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index dbf4bac16a10..c9fd858fe219 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.45 +LABEL io.airbyte.version=0.2.46 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 3e5ca7679f53..6113be0de55c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -293,6 +293,14 @@ }, "required": ["name"] } + }, + "page_size": { + "title": "Page Size of Requests", + "description": "Page size used when sending requests to Facebook API to specify number of records per page when response has pagination. Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases.", + "default": 100, + "order": 7, + "exclusiveMinimum": 0, + "type": "integer" } }, "required": ["account_id", "start_date", "access_token"] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index d3e079a561cb..d559e2af6c45 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -67,20 +67,56 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: ) streams = [ AdAccount(api=api), - AdSets(api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted), - Ads(api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted), - AdCreatives(api=api, fetch_thumbnail_images=config.fetch_thumbnail_images), - AdsInsights(**insights_args), - AdsInsightsAgeAndGender(**insights_args), - AdsInsightsCountry(**insights_args), - AdsInsightsRegion(**insights_args), - AdsInsightsDma(**insights_args), - AdsInsightsPlatformAndDevice(**insights_args), - AdsInsightsActionType(**insights_args), - Campaigns(api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted), - Images(api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted), - Videos(api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted), - Activities(api=api, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted), + AdSets( + api=api, + start_date=config.start_date, + end_date=config.end_date, + include_deleted=config.include_deleted, + page_size=config.page_size, + ), + Ads( + api=api, + start_date=config.start_date, + end_date=config.end_date, + include_deleted=config.include_deleted, + page_size=config.page_size, + ), + AdCreatives(api=api, fetch_thumbnail_images=config.fetch_thumbnail_images, page_size=config.page_size), + AdsInsights(page_size=config.page_size, **insights_args), + AdsInsightsAgeAndGender(page_size=config.page_size, **insights_args), + AdsInsightsCountry(page_size=config.page_size, **insights_args), + AdsInsightsRegion(page_size=config.page_size, **insights_args), + AdsInsightsDma(page_size=config.page_size, **insights_args), + AdsInsightsPlatformAndDevice(page_size=config.page_size, **insights_args), + AdsInsightsActionType(page_size=config.page_size, **insights_args), + Campaigns( + api=api, + start_date=config.start_date, + end_date=config.end_date, + include_deleted=config.include_deleted, + page_size=config.page_size, + ), + Images( + api=api, + start_date=config.start_date, + end_date=config.end_date, + include_deleted=config.include_deleted, + page_size=config.page_size, + ), + Videos( + api=api, + start_date=config.start_date, + end_date=config.end_date, + include_deleted=config.include_deleted, + page_size=config.page_size, + ), + Activities( + api=api, + start_date=config.start_date, + end_date=config.end_date, + include_deleted=config.include_deleted, + page_size=config.page_size, + ), ] return self._update_insights_streams(insights=config.custom_insights, default_args=insights_args, streams=streams) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index 6152b50ab119..56052299f180 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -147,3 +147,12 @@ class Config: "A list which contains insights entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns)" ), ) + + page_size: Optional[PositiveInt] = Field( + title="Page Size of Requests", + order=7, + default=100, + description=( + "Page size used when sending requests to Facebook API to specify number of records per page when response has pagination. Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases." + ), + ) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py index 58dd83f844cb..b0d7832c4f3c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py @@ -30,8 +30,6 @@ class FBMarketingStream(Stream, ABC): primary_key = "id" transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - # number of records per page when response has pagination - page_size = 100 # use batch API to retrieve details for each record in a stream use_batch = True # this flag will override `include_deleted` option for streams that does not support it @@ -39,9 +37,10 @@ class FBMarketingStream(Stream, ABC): # entity prefix for `include_deleted` filter, it usually matches singular version of stream name entity_prefix = None - def __init__(self, api: "API", include_deleted: bool = False, **kwargs): + def __init__(self, api: "API", include_deleted: bool = False, page_size: int = 100, **kwargs): super().__init__(**kwargs) self._api = api + self.page_size = page_size if page_size is not None else 100 self._include_deleted = include_deleted if self.enable_deleted else False @cached_property diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 98bd82243c0f..70cfd1846d62 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -108,6 +108,7 @@ For more information, see the [Facebook Insights API documentation.](https://dev | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.46 | 2022-04-22 | [12171](https://github.com/airbytehq/airbyte/pull/12171) | Allow configuration of page_size for requests | | 0.2.45 | 2022-05-03 | [12390](https://github.com/airbytehq/airbyte/pull/12390) | Better retry logic for split-up async jobs | | 0.2.44 | 2022-04-14 | [11751](https://github.com/airbytehq/airbyte/pull/11751) | Update API to a directly initialise an AdAccount with the given ID | | 0.2.43 | 2022-04-13 | [11801](https://github.com/airbytehq/airbyte/pull/11801) | Fix `user_tos_accepted` schema to be an object