🐛 Source google ads: remove "end_date" from config if empty value (#1…

…6344) Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>
airbytehq · Sep 6, 2022 · 1e5e37e · 1e5e37e
1 parent dd6baf9
commit 1e5e37e
Show file tree

Hide file tree

Showing 7 changed files with 19 additions and 11 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -347,7 +347,7 @@
 - name: Google Ads
   sourceDefinitionId: 253487c0-2246-43ba-a21f-5116b20a2c50
   dockerRepository: airbyte/source-google-ads
-  dockerImageTag: 0.1.44
+  dockerImageTag: 0.1.45
   documentationUrl: https://docs.airbyte.io/integrations/sources/google-ads
   icon: google-adwords.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -2904,7 +2904,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-google-ads:0.1.44"
+- dockerImage: "airbyte/source-google-ads:0.1.45"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/google-ads"
     connectionSpecification:
@@ -2993,7 +2993,7 @@
           title: "End Date (Optional)"
           description: "UTC date and time in the format 2017-01-25. Any data after\
             \ this date will not be replicated."
-          pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+          pattern: "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
           examples:
           - "2017-01-30"
           order: 6

diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile
@@ -13,5 +13,5 @@ COPY main.py ./
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.44
+LABEL io.airbyte.version=0.1.45
 LABEL io.airbyte.name=airbyte/source-google-ads
diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml
@@ -22,6 +22,7 @@ tests:
           "display_topics_performance_report",
           "shopping_performance_report",
           "unhappytable",
+          "click_view",
         ]
       timeout_seconds: 600
     - config_path: "secrets/config.json"

diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py
@@ -35,7 +35,7 @@ def configured_catalog():
 
 def test_incremental_sync(config, configured_catalog):
     today = pendulum.now().date()
-    start_date = today.subtract(months=2)
+    start_date = today.subtract(months=3)
     config["start_date"] = start_date.to_date_string()
 
     google_ads_client = SourceGoogleAds()

diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py
@@ -38,6 +38,12 @@
 
 
 class SourceGoogleAds(AbstractSource):
+    @staticmethod
+    def _validate_and_transform(config: Mapping[str, Any]):
+        if config.get("end_date") == "":
+            config.pop("end_date")
+        return config
+
     @staticmethod
     def get_credentials(config: Mapping[str, Any]) -> MutableMapping[str, Any]:
         credentials = config["credentials"]
@@ -52,16 +58,15 @@ def get_credentials(config: Mapping[str, Any]) -> MutableMapping[str, Any]:
 
     @staticmethod
     def get_incremental_stream_config(google_api: GoogleAds, config: Mapping[str, Any], customers: List[Customer]):
-        true_end_date = None
-        configured_end_date = config.get("end_date")
-        if configured_end_date is not None:
-            true_end_date = min(today(), parse(configured_end_date)).to_date_string()
+        end_date = config.get("end_date")
+        if end_date:
+            end_date = min(today(), parse(end_date)).to_date_string()
         incremental_stream_config = dict(
             api=google_api,
             customers=customers,
             conversion_window_days=config["conversion_window_days"],
             start_date=config["start_date"],
-            end_date=true_end_date,
+            end_date=end_date,
         )
         return incremental_stream_config
 
@@ -80,6 +85,7 @@ def is_metrics_in_custom_query(query: str) -> bool:
         return False
 
     def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]:
+        config = self._validate_and_transform(config)
         try:
             logger.info("Checking the config")
             google_api = GoogleAds(credentials=self.get_credentials(config))
@@ -109,6 +115,7 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) ->
             return False, f"Unable to connect to Google Ads API with the provided configuration - {error_messages}"
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
+        config = self._validate_and_transform(config)
         google_api = GoogleAds(credentials=self.get_credentials(config))
         accounts = self.get_account_info(google_api, config)
         customers = Customer.from_accounts(accounts)

diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json
@@ -75,7 +75,7 @@
         "type": "string",
         "title": "End Date (Optional)",
         "description": "UTC date and time in the format 2017-01-25. Any data after this date will not be replicated.",
-        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$",
+        "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}$",
         "examples": ["2017-01-30"],
         "order": 6
       },