🐛 Source FB Marketing: fix execute_in_batch when batch is bigger th…

…an 50 (#10588) * fix execute_in_batch * add tests * fix pre-commit config Co-authored-by: Sherif A. Nada <snadalive@gmail.com> Co-authored-by: Eugene Kulak <kulak.eugene@gmail.com> Co-authored-by: Sherif A. Nada <snadalive@gmail.com>
airbytehq · Feb 23, 2022 · 6cd20e6 · 6cd20e6
1 parent 91933c2
commit 6cd20e6
Show file tree

Hide file tree

Showing 8 changed files with 356 additions and 86 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -13,7 +13,7 @@ repos:
       - id: black
         args: ["--config", "pyproject.toml"]
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.6.4
+    rev: 5.10.1
     hooks:
       - id: isort
         args:
@@ -38,14 +38,14 @@ repos:
           ).?$
 
   - repo: https://github.com/csachs/pyproject-flake8
-    rev: 0.0.1a2
+    rev: v0.0.1a2.post1
     hooks:
       - id: pyproject-flake8
         args: ["--config", "pyproject.toml"]
         additional_dependencies: ["mccabe"]
         alias: flake8
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 0.930
+    rev: v0.930
     hooks:
       - id: mypy
         args: ["--config-file", "pyproject.toml"]

diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -203,7 +203,7 @@
 - name: Facebook Marketing
   sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
   dockerRepository: airbyte/source-facebook-marketing
-  dockerImageTag: 0.2.35
+  dockerImageTag: 0.2.36
   documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing
   icon: facebook.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -1648,145 +1648,278 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-facebook-marketing:0.2.35"
+- dockerImage: "airbyte/source-facebook-marketing:0.2.36"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
     connectionSpecification:
       title: "Source Facebook Marketing"
       type: "object"
       properties:
-        account_id:
-          title: "Account Id"
-          description: "The Facebook Ad account ID to use when pulling data from the\
-            \ Facebook Marketing API."
-          type: "string"
-        access_token:
-          title: "Access Token"
-          description: "The value of the access token generated. See the <a href=\"\
-            https://docs.airbyte.io/integrations/sources/facebook-marketing\">docs</a>\
-            \ for more information"
-          airbyte_secret: true
-          type: "string"
         start_date:
           title: "Start Date"
-          description: "The date from which you'd like to replicate data for AdCreatives\
-            \ and AdInsights APIs, in the format YYYY-MM-DDT00:00:00Z. All data generated\
-            \ after this date will be replicated."
+          description: "The date from which you'd like to replicate data for all incremental\
+            \ streams, in the format YYYY-MM-DDT00:00:00Z. All data generated after\
+            \ this date will be replicated."
+          order: 0
           pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
           examples:
           - "2017-01-25T00:00:00Z"
           type: "string"
           format: "date-time"
         end_date:
           title: "End Date"
-          description: "The date until which you'd like to replicate data for AdCreatives\
-            \ and AdInsights APIs, in the format YYYY-MM-DDT00:00:00Z. All data generated\
+          description: "The date until which you'd like to replicate data for all\
+            \ incremental streams, in the format YYYY-MM-DDT00:00:00Z. All data generated\
             \ between start_date and this date will be replicated. Not setting this\
             \ option will result in always syncing the latest data."
+          order: 1
           pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
           examples:
           - "2017-01-26T00:00:00Z"
           type: "string"
           format: "date-time"
+        account_id:
+          title: "Account ID"
+          description: "The Facebook Ad account ID to use when pulling data from the\
+            \ Facebook Marketing API."
+          order: 2
+          examples:
+          - "111111111111111"
+          type: "string"
+        access_token:
+          title: "Access Token"
+          description: "The value of the access token generated. See the <a href=\"\
+            https://docs.airbyte.io/integrations/sources/facebook-marketing\">docs</a>\
+            \ for more information"
+          order: 3
+          airbyte_secret: true
+          type: "string"
+        include_deleted:
+          title: "Include Deleted"
+          description: "Include data from deleted Campaigns, Ads, and AdSets"
+          default: false
+          order: 4
+          type: "boolean"
         fetch_thumbnail_images:
           title: "Fetch Thumbnail Images"
           description: "In each Ad Creative, fetch the thumbnail_url and store the\
             \ result in thumbnail_data_url"
           default: false
+          order: 5
           type: "boolean"
-        include_deleted:
-          title: "Include Deleted"
-          description: "Include data from deleted campaigns, ads, and adsets"
-          default: false
-          type: "boolean"
-        insights_lookback_window:
-          title: "Insights Lookback Window"
-          description: "The attribution window for the actions"
-          default: 28
-          minimum: 0
-          maximum: 28
-          type: "integer"
-        insights_days_per_job:
-          title: "Insights Days Per Job"
-          description: "Number of days to sync in one job (the more data you have,\
-            \ the smaller this parameter should be)"
-          default: 7
-          minimum: 1
-          maximum: 30
-          type: "integer"
         custom_insights:
           title: "Custom Insights"
-          description: "A list wich contains insights entries, each entry must have\
+          description: "A list which contains insights entries, each entry must have\
             \ a name and can contains fields, breakdowns or action_breakdowns)"
+          order: 6
           type: "array"
           items:
             title: "InsightConfig"
+            description: "Config for custom insights"
             type: "object"
             properties:
               name:
                 title: "Name"
                 description: "The name value of insight"
                 type: "string"
               fields:
-                title: "Fields"
                 description: "A list of chosen fields for fields parameter"
                 default: []
                 type: "array"
                 items:
-                  type: "string"
+                  title: "ValidEnums"
+                  description: "Generic enumeration.\n\nDerive from this class to\
+                    \ define new enumerations."
+                  enum:
+                  - "account_currency"
+                  - "account_id"
+                  - "account_name"
+                  - "action_values"
+                  - "actions"
+                  - "ad_bid_value"
+                  - "ad_click_actions"
+                  - "ad_id"
+                  - "ad_impression_actions"
+                  - "ad_name"
+                  - "adset_bid_value"
+                  - "adset_end"
+                  - "adset_id"
+                  - "adset_name"
+                  - "adset_start"
+                  - "age_targeting"
+                  - "attribution_setting"
+                  - "auction_bid"
+                  - "auction_competitiveness"
+                  - "auction_max_competitor_bid"
+                  - "buying_type"
+                  - "campaign_id"
+                  - "campaign_name"
+                  - "canvas_avg_view_percent"
+                  - "canvas_avg_view_time"
+                  - "catalog_segment_actions"
+                  - "catalog_segment_value"
+                  - "catalog_segment_value_mobile_purchase_roas"
+                  - "catalog_segment_value_omni_purchase_roas"
+                  - "catalog_segment_value_website_purchase_roas"
+                  - "clicks"
+                  - "conversion_rate_ranking"
+                  - "conversion_values"
+                  - "conversions"
+                  - "converted_product_quantity"
+                  - "converted_product_value"
+                  - "cost_per_15_sec_video_view"
+                  - "cost_per_2_sec_continuous_video_view"
+                  - "cost_per_action_type"
+                  - "cost_per_ad_click"
+                  - "cost_per_conversion"
+                  - "cost_per_dda_countby_convs"
+                  - "cost_per_estimated_ad_recallers"
+                  - "cost_per_inline_link_click"
+                  - "cost_per_inline_post_engagement"
+                  - "cost_per_one_thousand_ad_impression"
+                  - "cost_per_outbound_click"
+                  - "cost_per_thruplay"
+                  - "cost_per_unique_action_type"
+                  - "cost_per_unique_click"
+                  - "cost_per_unique_conversion"
+                  - "cost_per_unique_inline_link_click"
+                  - "cost_per_unique_outbound_click"
+                  - "cpc"
+                  - "cpm"
+                  - "cpp"
+                  - "created_time"
+                  - "ctr"
+                  - "date_start"
+                  - "date_stop"
+                  - "dda_countby_convs"
+                  - "dda_results"
+                  - "engagement_rate_ranking"
+                  - "estimated_ad_recall_rate"
+                  - "estimated_ad_recall_rate_lower_bound"
+                  - "estimated_ad_recall_rate_upper_bound"
+                  - "estimated_ad_recallers"
+                  - "estimated_ad_recallers_lower_bound"
+                  - "estimated_ad_recallers_upper_bound"
+                  - "frequency"
+                  - "full_view_impressions"
+                  - "full_view_reach"
+                  - "gender_targeting"
+                  - "impressions"
+                  - "inline_link_click_ctr"
+                  - "inline_link_clicks"
+                  - "inline_post_engagement"
+                  - "instant_experience_clicks_to_open"
+                  - "instant_experience_clicks_to_start"
+                  - "instant_experience_outbound_clicks"
+                  - "interactive_component_tap"
+                  - "labels"
+                  - "location"
+                  - "mobile_app_purchase_roas"
+                  - "objective"
+                  - "optimization_goal"
+                  - "outbound_clicks"
+                  - "outbound_clicks_ctr"
+                  - "place_page_name"
+                  - "purchase_roas"
+                  - "qualifying_question_qualify_answer_rate"
+                  - "quality_ranking"
+                  - "quality_score_ectr"
+                  - "quality_score_ecvr"
+                  - "quality_score_organic"
+                  - "reach"
+                  - "social_spend"
+                  - "spend"
+                  - "total_postbacks"
+                  - "unique_actions"
+                  - "unique_clicks"
+                  - "unique_conversions"
+                  - "unique_ctr"
+                  - "unique_inline_link_click_ctr"
+                  - "unique_inline_link_clicks"
+                  - "unique_link_clicks_ctr"
+                  - "unique_outbound_clicks"
+                  - "unique_outbound_clicks_ctr"
+                  - "unique_video_continuous_2_sec_watched_actions"
+                  - "unique_video_view_15_sec"
+                  - "updated_time"
+                  - "video_15_sec_watched_actions"
+                  - "video_30_sec_watched_actions"
+                  - "video_avg_time_watched_actions"
+                  - "video_continuous_2_sec_watched_actions"
+                  - "video_p100_watched_actions"
+                  - "video_p25_watched_actions"
+                  - "video_p50_watched_actions"
+                  - "video_p75_watched_actions"
+                  - "video_p95_watched_actions"
+                  - "video_play_actions"
+                  - "video_play_curve_actions"
+                  - "video_play_retention_0_to_15s_actions"
+                  - "video_play_retention_20_to_60s_actions"
+                  - "video_play_retention_graph_actions"
+                  - "video_thruplay_watched_actions"
+                  - "video_time_watched_actions"
+                  - "website_ctr"
+                  - "website_purchase_roas"
+                  - "wish_bid"
               breakdowns:
-                title: "Breakdowns"
                 description: "A list of chosen breakdowns for breakdowns"
                 default: []
                 type: "array"
                 items:
-                  type: "string"
+                  title: "ValidBreakdowns"
+                  description: "Generic enumeration.\n\nDerive from this class to\
+                    \ define new enumerations."
+                  enum:
+                  - "ad_format_asset"
+                  - "age"
+                  - "app_id"
+                  - "body_asset"
+                  - "call_to_action_asset"
+                  - "country"
+                  - "description_asset"
+                  - "device_platform"
+                  - "dma"
+                  - "frequency_value"
+                  - "gender"
+                  - "hourly_stats_aggregated_by_advertiser_time_zone"
+                  - "hourly_stats_aggregated_by_audience_time_zone"
+                  - "image_asset"
+                  - "impression_device"
+                  - "link_url_asset"
+                  - "place_page_id"
+                  - "platform_position"
+                  - "product_id"
+                  - "publisher_platform"
+                  - "region"
+                  - "skan_conversion_id"
+                  - "title_asset"
+                  - "video_asset"
               action_breakdowns:
-                title: "Action Breakdowns"
                 description: "A list of chosen action_breakdowns for action_breakdowns"
                 default: []
                 type: "array"
                 items:
-                  type: "string"
+                  title: "ValidActionBreakdowns"
+                  description: "Generic enumeration.\n\nDerive from this class to\
+                    \ define new enumerations."
+                  enum:
+                  - "action_canvas_component_name"
+                  - "action_carousel_card_id"
+                  - "action_carousel_card_name"
+                  - "action_destination"
+                  - "action_device"
+                  - "action_reaction"
+                  - "action_target_id"
+                  - "action_type"
+                  - "action_video_sound"
+                  - "action_video_type"
             required:
             - "name"
       required:
+      - "start_date"
       - "account_id"
       - "access_token"
-      - "start_date"
-      definitions:
-        InsightConfig:
-          title: "InsightConfig"
-          type: "object"
-          properties:
-            name:
-              title: "Name"
-              description: "The name value of insight"
-              type: "string"
-            fields:
-              title: "Fields"
-              description: "A list of chosen fields for fields parameter"
-              default: []
-              type: "array"
-              items:
-                type: "string"
-            breakdowns:
-              title: "Breakdowns"
-              description: "A list of chosen breakdowns for breakdowns"
-              default: []
-              type: "array"
-              items:
-                type: "string"
-            action_breakdowns:
-              title: "Action Breakdowns"
-              description: "A list of chosen action_breakdowns for action_breakdowns"
-              default: []
-              type: "array"
-              items:
-                type: "string"
-          required:
-          - "name"
     supportsIncremental: true
     supportsNormalization: false
     supportsDBT: false

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile
@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=0.2.35
+LABEL io.airbyte.version=0.2.36
 LABEL io.airbyte.name=airbyte/source-facebook-marketing