🎉 Source Google Ads: Implement multiple customer ids for google ads (#…

…10150)

* Source Google Ads: Implement multiple Customer ID(s)

* Source Google Ads: update states

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -252,7 +252,7 @@
 - name: Google Ads
   sourceDefinitionId: 253487c0-2246-43ba-a21f-5116b20a2c50
   dockerRepository: airbyte/source-google-ads
-  dockerImageTag: 0.1.25
+  dockerImageTag: 0.1.26
   documentationUrl: https://docs.airbyte.io/integrations/sources/google-ads
   icon: google-adwords.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -2294,7 +2294,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-google-ads:0.1.25"
+- dockerImage: "airbyte/source-google-ads:0.1.26"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/google-ads"
     connectionSpecification:
@@ -2352,13 +2352,16 @@
                 >docs</a>"
               airbyte_secret: true
         customer_id:
-          title: "Customer ID"
+          title: "Customer ID(s)"
           type: "string"
-          description: "Customer ID must be specified as a 10-digit number without\
-            \ dashes. More instruction on how to find this value in our <a href=\"\
-            https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\"\
+          description: "Comma separated list of (client) customer IDs. Each customer\
+            \ ID must be specified as a 10-digit number without dashes. More instruction\
+            \ on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\"\
             >docs</a>. Metrics streams like AdGroupAdReport cannot be requested for\
             \ a manager account."
+          pattern: "^[0-9]{10}(,[0-9]{10})*$"
+          examples:
+          - "6783948572,5839201945"
           order: 1
         start_date:
           type: "string"
@@ -2407,6 +2410,9 @@
             \ the manager account (10-digit number without dashes). More information\
             \ about this field you can see <a href=\"https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid\"\
             >here</a>"
+          pattern: "^([0-9]{10})?$"
+          examples:
+          - "7349206847"
           order: 4
         conversion_window_days:
           title: "Conversion Window (Optional)"

airbyte-integrations/connectors/source-google-ads/Dockerfile

@@ -13,5 +13,5 @@ RUN pip install .
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.25
+LABEL io.airbyte.version=0.1.26
 LABEL io.airbyte.name=airbyte/source-google-ads

airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml

@@ -13,7 +13,16 @@ tests:
     - config_path: "secrets/config.json"
   basic_read:
     - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog_without_empty_streams.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      empty_streams:
+        [
+          "geographic_report",
+          "keyword_report",
+          "display_keyword_performance_report",
+          "display_topics_performance_report",
+          "shopping_performance_report",
+        ]
+      timeout_seconds: 600
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog_protobuf_msg.json"
       expect_records:

airbyte-integrations/connectors/source-google-ads/integration_tests/invalid_config.json

@@ -5,7 +5,7 @@
     "client_secret": "client_secret",
     "refresh_token": "refresh_token"
   },
-  "customer_id": "customer_id",
+  "customer_id": "4312523412",
   "start_date": "2021-06-01",
   "conversion_window_days": 14
 }

airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py

@@ -58,7 +58,10 @@ def test_incremental_sync(config):
     for record in records:
         if record and record.type == Type.STATE:
             print(record)
-            current_state = record.state.data["ad_group_ad_report"]["segments.date"]
+            temp_state = record.state.data["ad_group_ad_report"]
+            current_state = (
+                temp_state[config["customer_id"]]["segments.date"] if temp_state.get(config["customer_id"]) else temp_state["segments.date"]
+            )
         if record and record.type == Type.RECORD:
             assert record.record.data["segments.date"] >= current_state
 
@@ -71,7 +74,7 @@ def test_incremental_sync(config):
 
     for record in records:
         if record and record.type == Type.STATE:
-            current_state = record.state.data["ad_group_ad_report"]["segments.date"]
+            current_state = record.state.data["ad_group_ad_report"][config["customer_id"]]["segments.date"]
         if record and record.type == Type.RECORD:
             assert record.record.data["segments.date"] >= current_state
 
@@ -80,7 +83,6 @@ def test_incremental_sync(config):
     records = google_ads_client.read(
         AirbyteLogger(), config, ConfiguredAirbyteCatalog.parse_obj(SAMPLE_CATALOG), {"ad_group_ad_report": {"segments.date": state}}
     )
-    current_state = pendulum.parse(state).subtract(days=14).to_date_string()
 
     no_records = True
     for record in records:

airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py

@@ -4,7 +4,7 @@
 
 
 from enum import Enum
-from typing import Any, List, Mapping
+from typing import Any, Iterator, List, Mapping
 
 import pendulum
 from google.ads.googleads.client import GoogleAdsClient
@@ -32,18 +32,21 @@ class GoogleAds:
     DEFAULT_PAGE_SIZE = 1000
 
     def __init__(self, credentials: Mapping[str, Any], customer_id: str):
+        # `google-ads` library version `14.0.0` and higher requires an additional required parameter `use_proto_plus`.
+        # More details can be found here: https://developers.google.com/google-ads/api/docs/client-libs/python/protobuf-messages
+        credentials["use_proto_plus"] = True
+
         self.client = GoogleAdsClient.load_from_dict(credentials)
-        self.customer_id = customer_id
+        self.customer_ids = customer_id.split(",")
         self.ga_service = self.client.get_service("GoogleAdsService")
 
-    def send_request(self, query: str) -> SearchGoogleAdsResponse:
+    def send_request(self, query: str, customer_id: str) -> Iterator[SearchGoogleAdsResponse]:
         client = self.client
         search_request = client.get_type("SearchGoogleAdsRequest")
-        search_request.customer_id = self.customer_id
         search_request.query = query
         search_request.page_size = self.DEFAULT_PAGE_SIZE
-
-        return self.ga_service.search(search_request)
+        search_request.customer_id = customer_id
+        yield self.ga_service.search(search_request)
 
     def get_fields_metadata(self, fields: List[str]) -> Mapping[str, Any]:
         """

airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py

@@ -6,6 +6,7 @@
 from typing import Any, List, Mapping, Tuple, Union
 
 from airbyte_cdk import AirbyteLogger
+from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from google.ads.googleads.errors import GoogleAdsException
@@ -46,7 +47,9 @@ def get_credentials(config: Mapping[str, Any]) -> Mapping[str, Any]:
 
     @staticmethod
     def get_account_info(google_api) -> dict:
-        return next(Accounts(api=google_api).read_records(sync_mode=None), {})
+        accounts_streams = Accounts(api=google_api)
+        for stream_slice in accounts_streams.stream_slices(sync_mode=SyncMode.full_refresh):
+            return next(accounts_streams.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice), {})
 
     @staticmethod
     def get_time_zone(account: dict) -> Union[timezone, str]:
@@ -84,7 +87,8 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
                     raise Exception(f"Custom query should not contain {CustomQuery.cursor_field}")
 
                 req_q = CustomQuery.insert_segments_date_expr(query, "1980-01-01", "1980-01-01")
-                google_api.send_request(req_q)
+                for customer_id in google_api.customer_ids:
+                    google_api.send_request(req_q, customer_id=customer_id)
             return True, None
         except GoogleAdsException as error:
             return False, f"Unable to connect to Google Ads API with the provided credentials - {repr(error.failure)}"

airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json

@@ -50,9 +50,11 @@
         }
       },
       "customer_id": {
-        "title": "Customer ID",
+        "title": "Customer ID(s)",
         "type": "string",
-        "description": "Customer ID must be specified as a 10-digit number without dashes. More instruction on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\">docs</a>. Metrics streams like AdGroupAdReport cannot be requested for a manager account.",
+        "description": "Comma separated list of (client) customer IDs. Each customer ID must be specified as a 10-digit number without dashes. More instruction on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\">docs</a>. Metrics streams like AdGroupAdReport cannot be requested for a manager account.",
+        "pattern": "^[0-9]{10}(,[0-9]{10})*$",
+        "examples": ["6783948572,5839201945"],
         "order": 1
       },
       "start_date": {
@@ -98,6 +100,8 @@
         "type": "string",
         "title": "Login Customer ID for Managed Accounts (Optional)",
         "description": "If your access to the customer account is through a manager account, this field is required and must be set to the customer ID of the manager account (10-digit number without dashes). More information about this field you can see <a href=\"https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid\">here</a>",
+        "pattern": "^([0-9]{10})?$",
+        "examples": ["7349206847"],
         "order": 4
       },
       "conversion_window_days": {