From 0d7101ecc41d9b740c4957cabb31c7fcbadd5904 Mon Sep 17 00:00:00 2001 From: Serhii Chvaliuk Date: Fri, 17 Dec 2021 16:52:42 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Source=20Facebook=20Marketing:?= =?UTF-8?q?=20fix=20schema=20for=20'breakdowns'=20fields=20(#8742)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix schema for 'breakdowns' and 'action_breakdowns fields Signed-off-by: Sergey Chvalyuk --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../source-facebook-marketing/Dockerfile | 2 +- .../ads_insights_action_breakdowns.json | 14 ++++++++++ .../schemas/ads_insights_breakdowns.json | 10 ------- .../source_facebook_marketing/source.py | 13 ++++------ .../source_facebook_marketing/streams.py | 26 ++++--------------- .../sources/facebook-marketing.md | 1 + 8 files changed, 28 insertions(+), 42 deletions(-) create mode 100644 airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_action_breakdowns.json diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 14a03adbd581..58a1fcf51736 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -175,7 +175,7 @@ - name: Facebook Marketing sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c dockerRepository: airbyte/source-facebook-marketing - dockerImageTag: 0.2.27 + dockerImageTag: 0.2.28 documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing icon: facebook.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 6b06fdca9ee3..99e728e5a264 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1428,7 +1428,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-facebook-marketing:0.2.27" +- dockerImage: "airbyte/source-facebook-marketing:0.2.28" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index fdbebf36d142..b6989bcf6eb1 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.27 +LABEL io.airbyte.version=0.2.28 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_action_breakdowns.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_action_breakdowns.json new file mode 100644 index 000000000000..ac976913bc5d --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_action_breakdowns.json @@ -0,0 +1,14 @@ +{ + "properties": { + "action_device": { "type": ["null", "string"] }, + "action_canvas_component_name": { "type": ["null", "string"] }, + "action_carousel_card_id": { "type": ["null", "string"] }, + "action_carousel_card_name": { "type": ["null", "string"] }, + "action_destination": { "type": ["null", "string"] }, + "action_reaction": { "type": ["null", "string"] }, + "action_target_id": { "type": ["null", "string"] }, + "action_type": { "type": ["null", "string"] }, + "action_video_sound": { "type": ["null", "string"] }, + "action_video_type": { "type": ["null", "string"] } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json index 87d31b9854b9..cb8824fbf5e1 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json @@ -1,15 +1,5 @@ { "properties": { - "action_device": { "type": ["null", "string"] }, - "action_canvas_component_name": { "type": ["null", "string"] }, - "action_carousel_card_id": { "type": ["null", "string"] }, - "action_carousel_card_name": { "type": ["null", "string"] }, - "action_destination": { "type": ["null", "string"] }, - "action_reaction": { "type": ["null", "string"] }, - "action_target_id": { "type": ["null", "string"] }, - "action_type": { "type": ["null", "string"] }, - "action_video_sound": { "type": ["null", "string"] }, - "action_video_type": { "type": ["null", "string"] }, "ad_format_asset": { "type": ["null", "string"] }, "age": { "type": ["null", "string"] }, "app_id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 0150bd0ec475..bccebefc2c5d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -204,13 +204,10 @@ def _update_insights_streams(self, insights, args, streams) -> List[Type[Stream] def _check_custom_insights_entries(self, insights: List[Mapping[str, Any]]): - default_fields = list( - ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ads_insights").get("properties", {}).keys() - ) - default_breakdowns = list( - ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ads_insights_breakdowns").get("properties", {}).keys() - ) - default_actions_breakdowns = [e for e in default_breakdowns if "action_" in e] + loader = ResourceSchemaLoader(package_name_from_class(self.__class__)) + default_fields = list(loader.get_schema("ads_insights").get("properties", {}).keys()) + default_breakdowns = list(loader.get_schema("ads_insights_breakdowns").get("properties", {}).keys()) + default_action_breakdowns = list(loader.get_schema("ads_insights_action_breakdowns").get("properties", {}).keys()) for insight in insights: if insight.get("fields"): @@ -224,7 +221,7 @@ def _check_custom_insights_entries(self, insights: List[Mapping[str, Any]]): message = f"{value} is not a valid breakdown name" raise Exception("Config validation error: " + message) from None if insight.get("action_breakdowns"): - value_checked, value = self._check_values(default_actions_breakdowns, insight.get("action_breakdowns")) + value_checked, value = self._check_values(default_action_breakdowns, insight.get("action_breakdowns")) if not value_checked: message = f"{value} is not a valid action_breakdown name" raise Exception("Config validation error: " + message) from None diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py index c0d236a80e70..5930bf3bae0f 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py @@ -402,10 +402,13 @@ def get_json_schema(self) -> Mapping[str, Any]: """Add fields from breakdowns to the stream schema :return: A dict of the JSON schema representing this stream. """ - schema = ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ads_insights") + loader = ResourceSchemaLoader(package_name_from_class(self.__class__)) + schema = loader.get_schema("ads_insights") if self._fields: schema["properties"] = {k: v for k, v in schema["properties"].items() if k in self._fields} - schema["properties"].update(self._schema_for_breakdowns()) + if self.breakdowns: + breakdowns_properties = loader.get_schema("ads_insights_breakdowns")["properties"] + schema["properties"].update({prop: breakdowns_properties[prop] for prop in self.breakdowns}) return schema @cached_property @@ -416,25 +419,6 @@ def fields(self) -> List[str]: schema = ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ads_insights") return list(schema.get("properties", {}).keys()) - def _schema_for_breakdowns(self) -> Mapping[str, Any]: - """Breakdown fields and their type""" - schemas = { - "age": {"type": ["null", "integer", "string"]}, - "gender": {"type": ["null", "string"]}, - "country": {"type": ["null", "string"]}, - "dma": {"type": ["null", "string"]}, - "region": {"type": ["null", "string"]}, - "impression_device": {"type": ["null", "string"]}, - "placement": {"type": ["null", "string"]}, - "platform_position": {"type": ["null", "string"]}, - "publisher_platform": {"type": ["null", "string"]}, - } - breakdowns = self.breakdowns[:] - if "platform_position" in breakdowns: - breakdowns.append("placement") - - return {breakdown: schemas[breakdown] for breakdown in self.breakdowns} - def _date_ranges(self, stream_state: Mapping[str, Any]) -> Iterator[dict]: """Iterate over period between start_date/state and now diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index c1b3f7b585a0..ee984c55c114 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -96,6 +96,7 @@ As a summary, custom insights allows to replicate only some fields, resulting in | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.28 | 2021-12-13 | [8742](https://github.com/airbytehq/airbyte/pull/8742) | Fix for schema generation related to "breakdown" fields | | 0.2.27 | 2021-11-29 | [8257](https://github.com/airbytehq/airbyte/pull/8257) | Add fields to Campaign stream | | 0.2.26 | 2021-11-19 | [7855](https://github.com/airbytehq/airbyte/pull/7855) | Add Video stream | | 0.2.25 | 2021-11-12 | [7904](https://github.com/airbytehq/airbyte/pull/7904) | Implement retry logic for async jobs |