From e74cb02b41e22ab5629280ca62ce6bcf369fc394 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Fri, 4 Feb 2022 17:39:51 +0200 Subject: [PATCH 01/10] Source Google Ads: Implement multiple Customer ID(s) --- .../integration_tests/configured_catalog.json | 142 ------------------ .../source_google_ads/google_ads.py | 11 +- .../source_google_ads/spec.json | 8 +- .../source_google_ads/streams.py | 5 +- .../unit_tests/test_google_ads.py | 10 +- 5 files changed, 20 insertions(+), 156 deletions(-) diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json index b99a68472e1e..eee3d6c2e6f9 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json @@ -11,148 +11,6 @@ "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"] - }, - { - "stream": { - "name": "account_performance_report", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["segments.date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["segments.date"] - }, - { - "stream": { - "name": "click_view", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["segments.date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["segments.date"] - }, - { - "stream": { - "name": "geographic_report", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["segments.date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["segments.date"] - }, - { - "stream": { - "name": "keyword_report", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["segments.date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["segments.date"] - }, - { - "stream": { - "name": "display_keyword_performance_report", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["segments.date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["segments.date"] - }, - { - "stream": { - "name": "display_topics_performance_report", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["segments.date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["segments.date"] - }, - { - "stream": { - "name": "shopping_performance_report", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["segments.date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["segments.date"] - }, - { - "stream": { - "name": "ad_group_ads", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["ad_group_ad.ad.id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "ad_groups", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["ad_group.id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "accounts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["customer.id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "campaigns", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["campaign.id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "happytable", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "unhappytable", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py index 86f3bad0d119..db86f6e3ca2c 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -4,7 +4,7 @@ from enum import Enum -from typing import Any, List, Mapping +from typing import Any, Iterator, List, Mapping import pendulum from google.ads.googleads.client import GoogleAdsClient @@ -33,17 +33,18 @@ class GoogleAds: def __init__(self, credentials: Mapping[str, Any], customer_id: str): self.client = GoogleAdsClient.load_from_dict(credentials) - self.customer_id = customer_id + self.customer_ids = customer_id.split(",") self.ga_service = self.client.get_service("GoogleAdsService") - def send_request(self, query: str) -> SearchGoogleAdsResponse: + def send_request(self, query: str) -> Iterator[SearchGoogleAdsResponse]: client = self.client search_request = client.get_type("SearchGoogleAdsRequest") - search_request.customer_id = self.customer_id search_request.query = query search_request.page_size = self.DEFAULT_PAGE_SIZE - return self.ga_service.search(search_request) + for customer_id in self.customer_ids: + search_request.customer_id = customer_id + yield self.ga_service.search(search_request) def get_fields_metadata(self, fields: List[str]) -> Mapping[str, Any]: """ diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json index 7620130f70b7..8a9304e1f8d8 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json @@ -50,9 +50,11 @@ } }, "customer_id": { - "title": "Customer ID", + "title": "Customer ID(s)", "type": "string", - "description": "Customer ID must be specified as a 10-digit number without dashes. More instruction on how to find this value in our docs. Metrics streams like AdGroupAdReport cannot be requested for a manager account.", + "description": "Comma separated list of (client) customer IDs. Each customer ID must be specified as a 10-digit number without dashes. More instruction on how to find this value in our docs. Metrics streams like AdGroupAdReport cannot be requested for a manager account.", + "pattern": "^[0-9]{10}(,[0-9]{10})*$", + "examples": ["6783948572,5839201945"], "order": 1 }, "start_date": { @@ -98,6 +100,8 @@ "type": "string", "title": "Login Customer ID for Managed Accounts (Optional)", "description": "If your access to the customer account is through a manager account, this field is required and must be set to the customer ID of the manager account (10-digit number without dashes). More information about this field you can see here", + "pattern": "^([0-9]{10})?$", + "examples": ["7349206847"], "order": 4 }, "conversion_window_days": { diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index 0e5648cad611..bfa635d4f3b4 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -61,8 +61,9 @@ def parse_response(self, response: SearchPager) -> Iterable[Mapping]: yield self.google_ads_client.parse_single_result(self.get_json_schema(), result) def read_records(self, sync_mode, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - response = self.google_ads_client.send_request(self.get_query(stream_slice)) - yield from self.parse_response(response) + account_responses = self.google_ads_client.send_request(self.get_query(stream_slice)) + for response in account_responses: + yield from self.parse_response(response) class IncrementalGoogleAdsStream(GoogleAdsStream, ABC): diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py index 100ca636cb78..122a5df38a21 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py @@ -77,7 +77,7 @@ def load_from_dict(config): def test_google_ads_init(mocker): google_client_mocker = mocker.patch("source_google_ads.google_ads.GoogleAdsClient", return_value=MockGoogleAdsClient) google_ads_client = GoogleAds(**SAMPLE_CONFIG) - assert google_ads_client.customer_id == SAMPLE_CONFIG["customer_id"] + assert google_ads_client.customer_ids == SAMPLE_CONFIG["customer_id"].split(",") assert google_client_mocker.load_from_dict.call_args[0][0] == EXPECTED_CRED @@ -87,11 +87,11 @@ def test_send_request(mocker): google_ads_client = GoogleAds(**SAMPLE_CONFIG) query = "Query" page_size = 1000 - response = google_ads_client.send_request(query) + response = list(google_ads_client.send_request(query)) - assert response.customer_id == SAMPLE_CONFIG["customer_id"] - assert response.query == query - assert response.page_size == page_size + assert response[0].customer_id == SAMPLE_CONFIG["customer_id"].split(",")[0] + assert response[0].query == query + assert response[0].page_size == page_size def test_get_fields_from_schema(): From 6677d98c9885dcec674162bf5add66854d5c3663 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Mon, 7 Feb 2022 17:52:32 +0200 Subject: [PATCH 02/10] Source Google Ads: update states --- .../acceptance-test-config.yml | 11 +- .../integration_tests/configured_catalog.json | 142 ++++++++++++++++++ ...figured_catalog_without_empty_streams.json | 102 ------------- .../integration_tests/invalid_config.json | 2 +- .../integration_tests/test_incremental.py | 5 +- .../source_google_ads/google_ads.py | 5 +- .../source_google_ads/streams.py | 55 ++++--- 7 files changed, 194 insertions(+), 128 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_without_empty_streams.json diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml index 861e01d01908..4a614f57685b 100644 --- a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml @@ -13,7 +13,16 @@ tests: - config_path: "secrets/config.json" basic_read: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_without_empty_streams.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + [ + "geographic_report", + "keyword_report", + "display_keyword_performance_report", + "display_topics_performance_report", + "shopping_performance_report", + ] + timeout_seconds: 600 - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_protobuf_msg.json" expect_records: diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json index eee3d6c2e6f9..b99a68472e1e 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json @@ -11,6 +11,148 @@ "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"] + }, + { + "stream": { + "name": "account_performance_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["segments.date"] + }, + { + "stream": { + "name": "click_view", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["segments.date"] + }, + { + "stream": { + "name": "geographic_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["segments.date"] + }, + { + "stream": { + "name": "keyword_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["segments.date"] + }, + { + "stream": { + "name": "display_keyword_performance_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["segments.date"] + }, + { + "stream": { + "name": "display_topics_performance_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["segments.date"] + }, + { + "stream": { + "name": "shopping_performance_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["segments.date"] + }, + { + "stream": { + "name": "ad_group_ads", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["ad_group_ad.ad.id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "ad_groups", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["ad_group.id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "accounts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["customer.id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["campaign.id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "happytable", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "unhappytable", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_without_empty_streams.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_without_empty_streams.json deleted file mode 100644 index 6dcce621fdba..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_without_empty_streams.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "ad_group_ad_report", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["segments.date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["segments.date"] - }, - { - "stream": { - "name": "account_performance_report", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["segments.date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["segments.date"] - }, - { - "stream": { - "name": "ad_group_ads", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["ad_group_ad.ad.id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "ad_groups", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["ad_group.id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "accounts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["customer.id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "click_view", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["segments.date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["segments.date"] - }, - { - "stream": { - "name": "campaigns", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["campaign.id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "happytable", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["campaign.start_date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["campaign.start_date"] - }, - { - "stream": { - "name": "unhappytable", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["customer.id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/invalid_config.json index 44e79cd86d6f..29b0cbce98e7 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/invalid_config.json @@ -5,7 +5,7 @@ "client_secret": "client_secret", "refresh_token": "refresh_token" }, - "customer_id": "customer_id", + "customer_id": "4312523412", "start_date": "2021-06-01", "conversion_window_days": 14 } diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py index bca63a71cced..76841efe0701 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py @@ -58,7 +58,7 @@ def test_incremental_sync(config): for record in records: if record and record.type == Type.STATE: print(record) - current_state = record.state.data["ad_group_ad_report"]["segments.date"] + current_state = record.state.data["ad_group_ad_report"][config["customer_id"]]["segments.date"] if record and record.type == Type.RECORD: assert record.record.data["segments.date"] >= current_state @@ -71,7 +71,7 @@ def test_incremental_sync(config): for record in records: if record and record.type == Type.STATE: - current_state = record.state.data["ad_group_ad_report"]["segments.date"] + current_state = record.state.data["ad_group_ad_report"][config["customer_id"]]["segments.date"] if record and record.type == Type.RECORD: assert record.record.data["segments.date"] >= current_state @@ -80,7 +80,6 @@ def test_incremental_sync(config): records = google_ads_client.read( AirbyteLogger(), config, ConfiguredAirbyteCatalog.parse_obj(SAMPLE_CATALOG), {"ad_group_ad_report": {"segments.date": state}} ) - current_state = pendulum.parse(state).subtract(days=14).to_date_string() no_records = True for record in records: diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py index db86f6e3ca2c..9863407ce2e4 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -36,13 +36,14 @@ def __init__(self, credentials: Mapping[str, Any], customer_id: str): self.customer_ids = customer_id.split(",") self.ga_service = self.client.get_service("GoogleAdsService") - def send_request(self, query: str) -> Iterator[SearchGoogleAdsResponse]: + def send_request(self, query: str, customer_id: str) -> Iterator[SearchGoogleAdsResponse]: client = self.client search_request = client.get_type("SearchGoogleAdsRequest") search_request.query = query search_request.page_size = self.DEFAULT_PAGE_SIZE - for customer_id in self.customer_ids: + customer_ids_list = [customer_id] if customer_id else self.customer_ids + for customer_id in customer_ids_list: search_request.customer_id = customer_id yield self.ga_service.search(search_request) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index bfa635d4f3b4..06862f47a926 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -51,6 +51,7 @@ def chunk_date_range( class GoogleAdsStream(Stream, ABC): def __init__(self, api: GoogleAds): self.google_ads_client = api + self._customer_id = None def get_query(self, stream_slice: Mapping[str, Any]) -> str: query = GoogleAds.convert_schema_into_query(schema=self.get_json_schema(), report_name=self.name) @@ -61,7 +62,8 @@ def parse_response(self, response: SearchPager) -> Iterable[Mapping]: yield self.google_ads_client.parse_single_result(self.get_json_schema(), result) def read_records(self, sync_mode, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - account_responses = self.google_ads_client.send_request(self.get_query(stream_slice)) + stream_slice = stream_slice or {} + account_responses = self.google_ads_client.send_request(self.get_query(stream_slice), customer_id=self._customer_id) for response in account_responses: yield from self.parse_response(response) @@ -80,18 +82,29 @@ def __init__(self, start_date: str, conversion_window_days: int, time_zone: [pen super().__init__(**kwargs) def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - stream_state = stream_state or {} - start_date = stream_state.get(self.cursor_field) or self._start_date - end_date = self._end_date - - return chunk_date_range( - start_date=start_date, - end_date=end_date, - conversion_window=self.conversion_window_days, - field=self.cursor_field, - days_of_data_storage=self.days_of_data_storage, - range_days=self.range_days, - ) + for customer_id in self.google_ads_client.customer_ids: + self.customer_id = customer_id + stream_state = stream_state or {} + if stream_state.get(customer_id): + start_date = stream_state[customer_id].get(self.cursor_field) or self._start_date + + # We should keep backward compatibility with the previous version + elif stream_state.get(self.cursor_field) and len(self.google_ads_client.customer_ids) == 1: + start_date = stream_state.get(self.cursor_field) or self._start_date + else: + start_date = self._start_date + + end_date = self._end_date + + for chunk in chunk_date_range( + start_date=start_date, + end_date=end_date, + conversion_window=self.conversion_window_days, + field=self.cursor_field, + days_of_data_storage=self.days_of_data_storage, + range_days=self.range_days, + ): + yield chunk def get_date_params(self, stream_slice: Mapping[str, Any], cursor_field: str, end_date: pendulum.datetime = None): """ @@ -116,16 +129,20 @@ def get_date_params(self, stream_slice: Mapping[str, Any], cursor_field: str, en def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: current_stream_state = current_stream_state or {} - # When state is none return date from latest record - if current_stream_state.get(self.cursor_field) is None: - current_stream_state[self.cursor_field] = latest_record[self.cursor_field] - + if current_stream_state.get(self.cursor_field): + current_stream = current_stream_state.pop(self.cursor_field) + elif current_stream_state.get(self.customer_id) and current_stream_state[self.customer_id].get(self.cursor_field): + current_stream = current_stream_state[self.customer_id][self.cursor_field] + else: + current_stream_state.update({self.customer_id: {self.cursor_field: latest_record[self.cursor_field]}}) return current_stream_state - date_in_current_stream = pendulum.parse(current_stream_state.get(self.cursor_field)) + date_in_current_stream = pendulum.parse(current_stream) date_in_latest_record = pendulum.parse(latest_record[self.cursor_field]) - current_stream_state[self.cursor_field] = (max(date_in_current_stream, date_in_latest_record)).to_date_string() + current_stream_state.update( + {self.customer_id: {self.cursor_field: (max(date_in_current_stream, date_in_latest_record)).to_date_string()}} + ) return current_stream_state From 0b43708e38bf9a2c1eae53f6f0a21202a5832fa3 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Mon, 7 Feb 2022 17:56:13 +0200 Subject: [PATCH 03/10] update naming --- .../source-google-ads/source_google_ads/streams.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index 06862f47a926..6ff1f7d5d0ae 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -130,14 +130,14 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late current_stream_state = current_stream_state or {} if current_stream_state.get(self.cursor_field): - current_stream = current_stream_state.pop(self.cursor_field) + stream_state = current_stream_state.pop(self.cursor_field) elif current_stream_state.get(self.customer_id) and current_stream_state[self.customer_id].get(self.cursor_field): - current_stream = current_stream_state[self.customer_id][self.cursor_field] + stream_state = current_stream_state[self.customer_id][self.cursor_field] else: current_stream_state.update({self.customer_id: {self.cursor_field: latest_record[self.cursor_field]}}) return current_stream_state - date_in_current_stream = pendulum.parse(current_stream) + date_in_current_stream = pendulum.parse(stream_state) date_in_latest_record = pendulum.parse(latest_record[self.cursor_field]) current_stream_state.update( From 38b6f268cdd152291d1a6c190f84837962eaba27 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Mon, 7 Feb 2022 18:28:10 +0200 Subject: [PATCH 04/10] fix test --- .../source-google-ads/integration_tests/test_incremental.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py index 76841efe0701..ac388e0b564a 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py @@ -58,7 +58,10 @@ def test_incremental_sync(config): for record in records: if record and record.type == Type.STATE: print(record) - current_state = record.state.data["ad_group_ad_report"][config["customer_id"]]["segments.date"] + temp_state = record.state.data["ad_group_ad_report"] + current_state = ( + temp_state[config["customer_id"]]["segments.date"] if temp_state.get(config["customer_id"]) else temp_state["segments.date"] + ) if record and record.type == Type.RECORD: assert record.record.data["segments.date"] >= current_state From 6ddbc1ad1dfed8d698d2c5e256479e5765790701 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Mon, 7 Feb 2022 20:27:57 +0200 Subject: [PATCH 05/10] update tests --- .../source_google_ads/google_ads.py | 4 ++++ .../source-google-ads/source_google_ads/streams.py | 10 +++++----- .../source-google-ads/unit_tests/test_google_ads.py | 6 ++++-- .../source-google-ads/unit_tests/test_source.py | 12 +++++++++--- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py index 9863407ce2e4..375f664b21a2 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -32,6 +32,10 @@ class GoogleAds: DEFAULT_PAGE_SIZE = 1000 def __init__(self, credentials: Mapping[str, Any], customer_id: str): + # `google-ads` library version `14.0.0` and higher requires an additional required parameter `use_proto_plus`. + # More details can be found here: https://developers.google.com/google-ads/api/docs/client-libs/python/protobuf-messages + credentials["use_proto_plus"] = True + self.client = GoogleAdsClient.load_from_dict(credentials) self.customer_ids = customer_id.split(",") self.ga_service = self.client.get_service("GoogleAdsService") diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index e51d232583d8..1bea83f2513b 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -116,7 +116,7 @@ def __init__(self, start_date: str, conversion_window_days: int, time_zone: [pen def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: for customer_id in self.google_ads_client.customer_ids: - self.customer_id = customer_id + self._customer_id = customer_id stream_state = stream_state or {} if stream_state.get(customer_id): start_date = stream_state[customer_id].get(self.cursor_field) or self._start_date @@ -185,17 +185,17 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late if current_stream_state.get(self.cursor_field): stream_state = current_stream_state.pop(self.cursor_field) - elif current_stream_state.get(self.customer_id) and current_stream_state[self.customer_id].get(self.cursor_field): - stream_state = current_stream_state[self.customer_id][self.cursor_field] + elif current_stream_state.get(self._customer_id) and current_stream_state[self._customer_id].get(self.cursor_field): + stream_state = current_stream_state[self._customer_id][self.cursor_field] else: - current_stream_state.update({self.customer_id: {self.cursor_field: latest_record[self.cursor_field]}}) + current_stream_state.update({self._customer_id: {self.cursor_field: latest_record[self.cursor_field]}}) return current_stream_state date_in_current_stream = pendulum.parse(stream_state) date_in_latest_record = pendulum.parse(latest_record[self.cursor_field]) current_stream_state.update( - {self.customer_id: {self.cursor_field: (max(date_in_current_stream, date_in_latest_record)).to_date_string()}} + {self._customer_id: {self.cursor_field: (max(date_in_current_stream, date_in_latest_record)).to_date_string()}} ) return current_stream_state diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py index c51a3071ea78..4a9150cc3fa1 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py @@ -46,6 +46,7 @@ def __getattr__(self, attr): "client_id": "client_id", "client_secret": "client_secret", "refresh_token": "refresh_token", + "use_proto_plus": True, } @@ -62,9 +63,10 @@ def test_send_request(mocker): google_ads_client = GoogleAds(**SAMPLE_CONFIG) query = "Query" page_size = 1000 - response = list(google_ads_client.send_request(query)) + customer_id = SAMPLE_CONFIG["customer_id"].split(",")[0] + response = list(google_ads_client.send_request(query, customer_id=customer_id)) - assert response[0].customer_id == SAMPLE_CONFIG["customer_id"].split(",")[0] + assert response[0].customer_id == customer_id assert response[0].query == query assert response[0].page_size == page_size diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py index 9159d1c5e6bd..eaaa5f3f6300 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py @@ -96,16 +96,22 @@ def test_get_updated_state(config): client = AdGroupAdReport( start_date=config["start_date"], api=google_api, conversion_window_days=config["conversion_window_days"], time_zone="local" ) + client._customer_id = "1234567890" + current_state_stream = {} latest_record = {"segments.date": "2020-01-01"} - new_stream_state = client.get_updated_state(current_state_stream, latest_record) - assert new_stream_state == {"segments.date": "2020-01-01"} + assert new_stream_state == {"1234567890": {"segments.date": "2020-01-01"}} current_state_stream = {"segments.date": "2020-01-01"} latest_record = {"segments.date": "2020-02-01"} new_stream_state = client.get_updated_state(current_state_stream, latest_record) - assert new_stream_state == {"segments.date": "2020-02-01"} + assert new_stream_state == {"1234567890": {"segments.date": "2020-02-01"}} + + current_state_stream = {"1234567890": {"segments.date": "2020-02-01"}} + latest_record = {"segments.date": "2021-03-03"} + new_stream_state = client.get_updated_state(current_state_stream, latest_record) + assert new_stream_state == {"1234567890": {"segments.date": "2021-03-03"}} def get_instance_from_config(config, query): From 10a524c3d8d736935225c8b640a2160291cee7fa Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Mon, 7 Feb 2022 20:44:42 +0200 Subject: [PATCH 06/10] update tests --- .../source_google_ads/streams.py | 4 ++-- .../source-google-ads/unit_tests/test_streams.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index 1bea83f2513b..58081dd71548 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -166,13 +166,13 @@ def read_records( # If range days is 1, no need in retry, because it's the minimum date range self.logger.error("Page token has expired.") raise exception - elif state.get(self.cursor_field) == stream_slice["start_date"]: + elif state.get(self._customer_id, {}).get(self.cursor_field) == stream_slice["start_date"]: # It couldn't read all the records within one day, it will enter an infinite loop, # so raise the error self.logger.error("Page token has expired.") raise exception # Retry reading records from where it crushed - stream_slice["start_date"] = state.get(self.cursor_field, stream_slice["start_date"]) + stream_slice["start_date"] = state.get(self._customer_id, {}).get(self.cursor_field, stream_slice["start_date"]) else: # raise caught error for other error statuses raise exception diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py index bf4cd9fd2b62..6099715203d5 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py @@ -48,7 +48,7 @@ def mock_ads_client(mocker): def mock_response_1(): - yield from [ + yield [ {"segments.date": "2021-01-01", "click_view.gclid": "1"}, {"segments.date": "2021-01-02", "click_view.gclid": "2"}, {"segments.date": "2021-01-03", "click_view.gclid": "3"}, @@ -58,7 +58,7 @@ def mock_response_1(): def mock_response_2(): - yield from [ + yield [ {"segments.date": "2021-01-03", "click_view.gclid": "3"}, {"segments.date": "2021-01-03", "click_view.gclid": "4"}, {"segments.date": "2021-01-03", "click_view.gclid": "5"}, @@ -73,7 +73,7 @@ class MockGoogleAds(GoogleAds): def parse_single_result(self, schema, result): return result - def send_request(self, query: str): + def send_request(self, query: str, customer_id: str): self.count += 1 if self.count == 1: return mock_response_1() @@ -109,7 +109,7 @@ def test_page_token_expired_retry_succeeds(mock_ads_client, test_config): def mock_response_fails_1(): - yield from [ + yield [ {"segments.date": "2021-01-01", "click_view.gclid": "1"}, {"segments.date": "2021-01-02", "click_view.gclid": "2"}, {"segments.date": "2021-01-03", "click_view.gclid": "3"}, @@ -120,7 +120,7 @@ def mock_response_fails_1(): def mock_response_fails_2(): - yield from [ + yield [ {"segments.date": "2021-01-03", "click_view.gclid": "3"}, {"segments.date": "2021-01-03", "click_view.gclid": "4"}, {"segments.date": "2021-01-03", "click_view.gclid": "5"}, @@ -131,7 +131,7 @@ def mock_response_fails_2(): class MockGoogleAdsFails(MockGoogleAds): - def send_request(self, query: str): + def send_request(self, query: str, customer_id: str): self.count += 1 if self.count == 1: return mock_response_fails_1() @@ -166,7 +166,7 @@ def test_page_token_expired_retry_fails(mock_ads_client, test_config): def mock_response_fails_one_date(): - yield from [ + yield [ {"segments.date": "2021-01-03", "click_view.gclid": "3"}, {"segments.date": "2021-01-03", "click_view.gclid": "4"}, {"segments.date": "2021-01-03", "click_view.gclid": "5"}, @@ -177,7 +177,7 @@ def mock_response_fails_one_date(): class MockGoogleAdsFailsOneDate(MockGoogleAds): - def send_request(self, query: str): + def send_request(self, query: str, customer_id: str): return mock_response_fails_one_date() From d26f5d6e5fb3f1753c61b6e3080f9b2a9b1b513a Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Mon, 7 Feb 2022 22:08:33 +0200 Subject: [PATCH 07/10] fix check_connection --- .../connectors/source-google-ads/source_google_ads/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py index bad864845cf1..3b0d22ba12d1 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py @@ -84,7 +84,7 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> raise Exception(f"Custom query should not contain {CustomQuery.cursor_field}") req_q = CustomQuery.insert_segments_date_expr(query, "1980-01-01", "1980-01-01") - google_api.send_request(req_q) + google_api.send_request(req_q, google_api.customer_ids[0]) return True, None except GoogleAdsException as error: return False, f"Unable to connect to Google Ads API with the provided credentials - {repr(error.failure)}" From 038926591d0701b2138eaabf83bf69cee1979983 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Tue, 8 Feb 2022 14:21:44 +0200 Subject: [PATCH 08/10] update version, add changelogs --- airbyte-integrations/connectors/source-google-ads/Dockerfile | 2 +- docs/integrations/sources/google-ads.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile index 6a607db3ebe3..e7b906a6b221 100644 --- a/airbyte-integrations/connectors/source-google-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-google-ads/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.25 +LABEL io.airbyte.version=0.1.26 LABEL io.airbyte.name=airbyte/source-google-ads diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md index 5f4316125bda..c96b56830f48 100644 --- a/docs/integrations/sources/google-ads.md +++ b/docs/integrations/sources/google-ads.md @@ -102,6 +102,7 @@ This source is constrained by whatever API limits are set for the Google Ads tha | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| `0.1.26` | 2022-02-09 | [10150](https://github.com/airbytehq/airbyte/pull/10150) | Add support for multiple `customer IDs` in specs. | | `0.1.25` | 2022-02-04 | [9812](https://github.com/airbytehq/airbyte/pull/9812) | Handle `EXPIRED_PAGE_TOKEN` exception and retry with updated state. | | `0.1.24` | 2022-02-04 | [9996](https://github.com/airbytehq/airbyte/pull/9996) | Use Google Ads API version V9. | | `0.1.23` | 2022-01-25 | [8669](https://github.com/airbytehq/airbyte/pull/8669) | Add end date parameter in spec. | From db568ea0a402ec607fd2f0900ab5a78c36cbba05 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Fri, 11 Feb 2022 01:14:02 +0200 Subject: [PATCH 09/10] update after review --- .../source-google-ads/source_google_ads/google_ads.py | 7 ++----- .../source-google-ads/source_google_ads/source.py | 8 ++++++-- .../source-google-ads/source_google_ads/streams.py | 6 +++++- docs/integrations/sources/google-ads.md | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py index 375f664b21a2..19ec8d02b5bc 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -45,11 +45,8 @@ def send_request(self, query: str, customer_id: str) -> Iterator[SearchGoogleAds search_request = client.get_type("SearchGoogleAdsRequest") search_request.query = query search_request.page_size = self.DEFAULT_PAGE_SIZE - - customer_ids_list = [customer_id] if customer_id else self.customer_ids - for customer_id in customer_ids_list: - search_request.customer_id = customer_id - yield self.ga_service.search(search_request) + search_request.customer_id = customer_id + yield self.ga_service.search(search_request) def get_fields_metadata(self, fields: List[str]) -> Mapping[str, Any]: """ diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py index 3b0d22ba12d1..9310096bbbf7 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py @@ -6,6 +6,7 @@ from typing import Any, List, Mapping, Tuple, Union from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from google.ads.googleads.errors import GoogleAdsException @@ -46,7 +47,9 @@ def get_credentials(config: Mapping[str, Any]) -> Mapping[str, Any]: @staticmethod def get_account_info(google_api) -> dict: - return next(Accounts(api=google_api).read_records(sync_mode=None), {}) + accounts_streams = Accounts(api=google_api) + for stream_slice in accounts_streams.stream_slices(sync_mode=SyncMode.full_refresh): + return next(accounts_streams.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice), {}) @staticmethod def get_time_zone(account: dict) -> Union[timezone, str]: @@ -84,7 +87,8 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> raise Exception(f"Custom query should not contain {CustomQuery.cursor_field}") req_q = CustomQuery.insert_segments_date_expr(query, "1980-01-01", "1980-01-01") - google_api.send_request(req_q, google_api.customer_ids[0]) + for customer_id in google_api.customer_ids: + google_api.send_request(req_q, customer_id=customer_id) return True, None except GoogleAdsException as error: return False, f"Unable to connect to Google Ads API with the provided credentials - {repr(error.failure)}" diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index 58081dd71548..15746b9716a0 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -94,8 +94,12 @@ def parse_response(self, response: SearchPager) -> Iterable[Mapping]: for result in response: yield self.google_ads_client.parse_single_result(self.get_json_schema(), result) + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + for customer_id in self.google_ads_client.customer_ids: + self._customer_id = customer_id + yield {} + def read_records(self, sync_mode, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - stream_slice = stream_slice or {} account_responses = self.google_ads_client.send_request(self.get_query(stream_slice), customer_id=self._customer_id) for response in account_responses: yield from self.parse_response(response) diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md index c96b56830f48..4bcdd845156d 100644 --- a/docs/integrations/sources/google-ads.md +++ b/docs/integrations/sources/google-ads.md @@ -102,7 +102,7 @@ This source is constrained by whatever API limits are set for the Google Ads tha | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| `0.1.26` | 2022-02-09 | [10150](https://github.com/airbytehq/airbyte/pull/10150) | Add support for multiple `customer IDs` in specs. | +| `0.1.26` | 2022-02-09 | [10150](https://github.com/airbytehq/airbyte/pull/10150) | Add support for multiple customer IDs. | | `0.1.25` | 2022-02-04 | [9812](https://github.com/airbytehq/airbyte/pull/9812) | Handle `EXPIRED_PAGE_TOKEN` exception and retry with updated state. | | `0.1.24` | 2022-02-04 | [9996](https://github.com/airbytehq/airbyte/pull/9996) | Use Google Ads API version V9. | | `0.1.23` | 2022-01-25 | [8669](https://github.com/airbytehq/airbyte/pull/8669) | Add end date parameter in spec. | From 7868b85adbf39d1a2567318421e6fe2342c2570f Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Fri, 11 Feb 2022 17:41:36 +0200 Subject: [PATCH 10/10] update version --- .../main/resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 16 +++++++++++----- docs/integrations/sources/google-ads.md | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e10d784c2250..e59481b93fd4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -252,7 +252,7 @@ - name: Google Ads sourceDefinitionId: 253487c0-2246-43ba-a21f-5116b20a2c50 dockerRepository: airbyte/source-google-ads - dockerImageTag: 0.1.25 + dockerImageTag: 0.1.26 documentationUrl: https://docs.airbyte.io/integrations/sources/google-ads icon: google-adwords.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 9ec51abce0b6..644512749d68 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2294,7 +2294,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-google-ads:0.1.25" +- dockerImage: "airbyte/source-google-ads:0.1.26" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/google-ads" connectionSpecification: @@ -2352,13 +2352,16 @@ >docs" airbyte_secret: true customer_id: - title: "Customer ID" + title: "Customer ID(s)" type: "string" - description: "Customer ID must be specified as a 10-digit number without\ - \ dashes. More instruction on how to find this value in our docs. Metrics streams like AdGroupAdReport cannot be requested for\ \ a manager account." + pattern: "^[0-9]{10}(,[0-9]{10})*$" + examples: + - "6783948572,5839201945" order: 1 start_date: type: "string" @@ -2407,6 +2410,9 @@ \ the manager account (10-digit number without dashes). More information\ \ about this field you can see here" + pattern: "^([0-9]{10})?$" + examples: + - "7349206847" order: 4 conversion_window_days: title: "Conversion Window (Optional)" diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md index 4bcdd845156d..9a75f2c72e24 100644 --- a/docs/integrations/sources/google-ads.md +++ b/docs/integrations/sources/google-ads.md @@ -102,7 +102,7 @@ This source is constrained by whatever API limits are set for the Google Ads tha | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| `0.1.26` | 2022-02-09 | [10150](https://github.com/airbytehq/airbyte/pull/10150) | Add support for multiple customer IDs. | +| `0.1.26` | 2022-02-11 | [10150](https://github.com/airbytehq/airbyte/pull/10150) | Add support for multiple customer IDs. | | `0.1.25` | 2022-02-04 | [9812](https://github.com/airbytehq/airbyte/pull/9812) | Handle `EXPIRED_PAGE_TOKEN` exception and retry with updated state. | | `0.1.24` | 2022-02-04 | [9996](https://github.com/airbytehq/airbyte/pull/9996) | Use Google Ads API version V9. | | `0.1.23` | 2022-01-25 | [8669](https://github.com/airbytehq/airbyte/pull/8669) | Add end date parameter in spec. |