🎉 Source Facebook Marketing: fix schema for 'breakdowns' fields (#8742)

* fix schema for 'breakdowns' and 'action_breakdowns fields

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -175,7 +175,7 @@
 - name: Facebook Marketing
   sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
   dockerRepository: airbyte/source-facebook-marketing
-  dockerImageTag: 0.2.27
+  dockerImageTag: 0.2.28
   documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing
   icon: facebook.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -1428,7 +1428,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-facebook-marketing:0.2.27"
+- dockerImage: "airbyte/source-facebook-marketing:0.2.28"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"

airbyte-integrations/connectors/source-facebook-marketing/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.27
+LABEL io.airbyte.version=0.2.28
 LABEL io.airbyte.name=airbyte/source-facebook-marketing

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_action_breakdowns.json

@@ -0,0 +1,14 @@
+{
+  "properties": {
+    "action_device": { "type": ["null", "string"] },
+    "action_canvas_component_name": { "type": ["null", "string"] },
+    "action_carousel_card_id": { "type": ["null", "string"] },
+    "action_carousel_card_name": { "type": ["null", "string"] },
+    "action_destination": { "type": ["null", "string"] },
+    "action_reaction": { "type": ["null", "string"] },
+    "action_target_id": { "type": ["null", "string"] },
+    "action_type": { "type": ["null", "string"] },
+    "action_video_sound": { "type": ["null", "string"] },
+    "action_video_type": { "type": ["null", "string"] }
+  }
+}

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json

@@ -1,15 +1,5 @@
 {
   "properties": {
-    "action_device": { "type": ["null", "string"] },
-    "action_canvas_component_name": { "type": ["null", "string"] },
-    "action_carousel_card_id": { "type": ["null", "string"] },
-    "action_carousel_card_name": { "type": ["null", "string"] },
-    "action_destination": { "type": ["null", "string"] },
-    "action_reaction": { "type": ["null", "string"] },
-    "action_target_id": { "type": ["null", "string"] },
-    "action_type": { "type": ["null", "string"] },
-    "action_video_sound": { "type": ["null", "string"] },
-    "action_video_type": { "type": ["null", "string"] },
     "ad_format_asset": { "type": ["null", "string"] },
     "age": { "type": ["null", "string"] },
     "app_id": { "type": ["null", "string"] },

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py

@@ -204,13 +204,10 @@ def _update_insights_streams(self, insights, args, streams) -> List[Type[Stream]
 
     def _check_custom_insights_entries(self, insights: List[Mapping[str, Any]]):
 
-        default_fields = list(
-            ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ads_insights").get("properties", {}).keys()
-        )
-        default_breakdowns = list(
-            ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ads_insights_breakdowns").get("properties", {}).keys()
-        )
-        default_actions_breakdowns = [e for e in default_breakdowns if "action_" in e]
+        loader = ResourceSchemaLoader(package_name_from_class(self.__class__))
+        default_fields = list(loader.get_schema("ads_insights").get("properties", {}).keys())
+        default_breakdowns = list(loader.get_schema("ads_insights_breakdowns").get("properties", {}).keys())
+        default_action_breakdowns = list(loader.get_schema("ads_insights_action_breakdowns").get("properties", {}).keys())
 
         for insight in insights:
             if insight.get("fields"):
@@ -224,7 +221,7 @@ def _check_custom_insights_entries(self, insights: List[Mapping[str, Any]]):
                     message = f"{value} is not a valid breakdown name"
                     raise Exception("Config validation error: " + message) from None
             if insight.get("action_breakdowns"):
-                value_checked, value = self._check_values(default_actions_breakdowns, insight.get("action_breakdowns"))
+                value_checked, value = self._check_values(default_action_breakdowns, insight.get("action_breakdowns"))
                 if not value_checked:
                     message = f"{value} is not a valid action_breakdown name"
                     raise Exception("Config validation error: " + message) from None

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams.py

@@ -402,10 +402,13 @@ def get_json_schema(self) -> Mapping[str, Any]:
         """Add fields from breakdowns to the stream schema
         :return: A dict of the JSON schema representing this stream.
         """
-        schema = ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ads_insights")
+        loader = ResourceSchemaLoader(package_name_from_class(self.__class__))
+        schema = loader.get_schema("ads_insights")
         if self._fields:
             schema["properties"] = {k: v for k, v in schema["properties"].items() if k in self._fields}
-        schema["properties"].update(self._schema_for_breakdowns())
+        if self.breakdowns:
+            breakdowns_properties = loader.get_schema("ads_insights_breakdowns")["properties"]
+            schema["properties"].update({prop: breakdowns_properties[prop] for prop in self.breakdowns})
         return schema
 
     @cached_property
@@ -416,25 +419,6 @@ def fields(self) -> List[str]:
         schema = ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ads_insights")
         return list(schema.get("properties", {}).keys())
 
-    def _schema_for_breakdowns(self) -> Mapping[str, Any]:
-        """Breakdown fields and their type"""
-        schemas = {
-            "age": {"type": ["null", "integer", "string"]},
-            "gender": {"type": ["null", "string"]},
-            "country": {"type": ["null", "string"]},
-            "dma": {"type": ["null", "string"]},
-            "region": {"type": ["null", "string"]},
-            "impression_device": {"type": ["null", "string"]},
-            "placement": {"type": ["null", "string"]},
-            "platform_position": {"type": ["null", "string"]},
-            "publisher_platform": {"type": ["null", "string"]},
-        }
-        breakdowns = self.breakdowns[:]
-        if "platform_position" in breakdowns:
-            breakdowns.append("placement")
-
-        return {breakdown: schemas[breakdown] for breakdown in self.breakdowns}
-
     def _date_ranges(self, stream_state: Mapping[str, Any]) -> Iterator[dict]:
         """Iterate over period between start_date/state and now
 

docs/integrations/sources/facebook-marketing.md

@@ -96,6 +96,7 @@ As a summary, custom insights allows to replicate only some fields, resulting in
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.2.28  | 2021-12-13 | [8742](https://github.com/airbytehq/airbyte/pull/8742) | Fix for schema generation related to "breakdown" fields |
 | 0.2.27  | 2021-11-29 | [8257](https://github.com/airbytehq/airbyte/pull/8257) | Add fields to Campaign stream |
 | 0.2.26  | 2021-11-19 | [7855](https://github.com/airbytehq/airbyte/pull/7855) | Add Video stream |
 | 0.2.25  | 2021-11-12 | [7904](https://github.com/airbytehq/airbyte/pull/7904) | Implement retry logic for async jobs |