🐛 Source Amazon Ads: Improve report streams date-range generation (#1…

…5031) Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>
airbytehq · Aug 5, 2022 · a4e52cc · a4e52cc
1 parent 2d60438
commit a4e52cc
Show file tree

Hide file tree

Showing 22 changed files with 386 additions and 258 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -17,7 +17,7 @@
 - name: Amazon Ads
   sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
   dockerRepository: airbyte/source-amazon-ads
-  dockerImageTag: 0.1.10
+  dockerImageTag: 0.1.11
   documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads
   icon: amazonads.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -87,7 +87,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-amazon-ads:0.1.10"
+- dockerImage: "airbyte/source-amazon-ads:0.1.11"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads"
     connectionSpecification:
@@ -123,14 +123,12 @@
           type: "string"
         region:
           title: "Region *"
-          description: "Region to pull data from (EU/NA/FE/SANDBOX). See <a href=\"\
-            https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints\"\
+          description: "Region to pull data from (EU/NA/FE). See <a href=\"https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints\"\
             >docs</a> for more details."
           enum:
           - "NA"
           - "EU"
           - "FE"
-          - "SANDBOX"
           type: "string"
           default: "NA"
           order: 4

diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.10
+LABEL io.airbyte.version=0.1.11
 LABEL io.airbyte.name=airbyte/source-amazon-ads
diff --git a/airbyte-integrations/connectors/source-amazon-ads/README.md b/airbyte-integrations/connectors/source-amazon-ads/README.md
@@ -8,7 +8,7 @@ For information about how to use this connector within Airbyte, see [the documen
 ### Prerequisites
 **To iterate on this connector, make sure to complete this prerequisites section.**
 
-#### Minimum Python version required `= 3.7.0`
+#### Minimum Python version required `= 3.9.0`
 
 #### Build & Activate Virtual Environment and install dependencies
 From this connector directory, create a virtual environment:

diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml
@@ -5,43 +5,35 @@ tests:
   spec:
     - spec_path: "integration_tests/spec.json"
   connection:
-    # THIS TEST IS COMMENTED OUT BECAUSE OF
-    # https://advertising.amazon.com/API/docs/en-us/info/release-notes#sandbox-deprecation-on-june-28-2022
-    # - config_path: "secrets/config.json"
-    #   status: "succeed"
-    - config_path: "secrets/config_test_account.json"
+    - config_path: "secrets/config.json"
       status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
   discovery:
-    # THIS TEST IS COMMENTED OUT BECAUSE OF LOST ACCESS TO SANDBOX
-    # - config_path: "secrets/config.json"
-    - config_path: "secrets/config_test_account.json"
+    - config_path: "secrets/config.json"
   basic_read:
-    - config_path: "secrets/config_test_account.json"
+    - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
-      empty_streams: ["sponsored_product_targetings"]
+      empty_streams: ["sponsored_brands_campaigns", "sponsored_brands_ad_groups", "sponsored_brands_keywords"]
       expect_records:
         path: "integration_tests/expected_records.txt"
         extra_fields: no
         exact_order: no
         extra_records: no
-      timeout_seconds: 3600
-    # THIS TEST IS COMMENTED OUT BECAUSE OF
-    # https://advertising.amazon.com/API/docs/en-us/info/release-notes#sandbox-deprecation-on-june-28-2022
-    # - config_path: "secrets/config.json"
-    #   configured_catalog_path: "integration_tests/configured_catalog_sponsored_display.json"
-    #   empty_streams: ["sponsored_display_targetings"]
-    #   expect_records:
-    #     path: "integration_tests/expected_records_sponsored_display.txt"
-    #     extra_fields: no
-    #     exact_order: no
-    #     extra_records: no
+    - config_path: "secrets/config_report.json"
+      configured_catalog_path: "integration_tests/configured_catalog_report.json"
+      timeout_seconds: 2400
+  incremental:
+    - config_path: "secrets/config_report.json"
+      configured_catalog_path: "integration_tests/configured_catalog_report.json"
+      future_state_path: "integration_tests/abnormal_state.json"
+      cursor_paths:
+        sponsored_products_report_stream: ["1861552880916640", "reportDate"]
   full_refresh:
-    - config_path: "secrets/config_test_account.json"
+    - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
+    - config_path: "secrets/config_report.json"
+      configured_catalog_path: "integration_tests/configured_catalog_report.json"
+      ignored_fields:
+        "sponsored_products_report_stream": ["updatedAt"]
       timeout_seconds: 3600
-    # THIS TEST IS COMMENTED OUT BECAUSE OF
-    # https://advertising.amazon.com/API/docs/en-us/info/release-notes#sandbox-deprecation-on-june-28-2022
-    # - config_path: "secrets/config.json"
-    #   configured_catalog_path: "integration_tests/configured_catalog_sponsored_display.json"
diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-docker.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env sh
 
 # Build latest connector image
-docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2)
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-)
 
 # Pull latest acctest image
 docker pull airbyte/source-acceptance-test:latest

diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/abnormal_state.json
@@ -0,0 +1,7 @@
+{
+  "sponsored_products_report_stream": {
+    "1861552880916640": {
+      "reportDate": "20990101"
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json
@@ -10,6 +10,46 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "sponsored_display_campaigns",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["campaignId"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "sponsored_display_ad_groups",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["adGroupId"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "sponsored_display_product_ads",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["adId"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "sponsored_display_targetings",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["targetId"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "sponsored_product_campaigns",
@@ -72,9 +112,30 @@
     },
     {
       "stream": {
-        "name": "sponsored_products_report_stream",
+        "name": "sponsored_brands_campaigns",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["campaignId"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "sponsored_brands_ad_groups",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["adGroupId"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "sponsored_brands_keywords",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"]
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["adGroupId"]]
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"

diff --git a/...ntegrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json b/...ntegrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json
@@ -0,0 +1,14 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "sponsored_products_report_stream",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_primary_key": [["profileId"], ["recordType"], ["reportDate"], ["updatedAt"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    }
+  ]
+}
diff --git a/.../connectors/source-amazon-ads/integration_tests/configured_catalog_sponsored_display.json b/.../connectors/source-amazon-ads/integration_tests/configured_catalog_sponsored_display.json