Remove pydantic spec from amazon ads and use YAML spec (#13988)

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py

@@ -6,13 +6,12 @@
 from typing import Any, List, Mapping, Tuple
 
 from airbyte_cdk.logger import AirbyteLogger
-from airbyte_cdk.models import ConnectorSpecification
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator
 
+from .constants import AmazonAdsRegion
 from .schemas import Profile
-from .spec import AmazonAdsConfig, advanced_auth
 from .streams import (
     Profiles,
     SponsoredBrandsAdGroups,
@@ -45,12 +44,12 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
         :param logger:  logger object
         :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise.
         """
-        config = AmazonAdsConfig(**config)
         # Check connection by sending list of profiles request. Its most simple
         # request, not require additional parameters and usually has few data
         # in response body.
         # It doesnt support pagination so there is no sense of reading single
         # record, it would fetch all the data anyway.
+        self._set_defaults(config)
         Profiles(config, authenticator=self._make_authenticator(config)).get_all_profiles()
         return True, None
 
@@ -59,7 +58,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         :param config: A Mapping of the user input configuration as defined in the connector spec.
         :return list of streams for current source
         """
-        config = AmazonAdsConfig(**config)
+        self._set_defaults(config)
         auth = self._make_authenticator(config)
         stream_args = {"config": config, "authenticator": auth}
         # All data for individual Amazon Ads stream divided into sets of data for
@@ -91,24 +90,21 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         ]
         return [profiles_stream, *[stream_class(**stream_args) for stream_class in non_profile_stream_classes]]
 
-    def spec(self, *args) -> ConnectorSpecification:
-        return ConnectorSpecification(
-            documentationUrl="https://docs.airbyte.com/integrations/sources/amazon-ads",
-            connectionSpecification=AmazonAdsConfig.schema(),
-            advanced_auth=advanced_auth,
-        )
-
     @staticmethod
-    def _make_authenticator(config: AmazonAdsConfig):
+    def _make_authenticator(config: Mapping[str, Any]):
         return Oauth2Authenticator(
             token_refresh_endpoint=TOKEN_URL,
-            client_id=config.client_id,
-            client_secret=config.client_secret,
-            refresh_token=config.refresh_token,
+            client_id=config["client_id"],
+            client_secret=config["client_secret"],
+            refresh_token=config["refresh_token"],
         )
 
     @staticmethod
-    def _choose_profiles(config: AmazonAdsConfig, profiles: List[Profile]):
-        if not config.profiles:
+    def _set_defaults(config: Mapping[str, Any]):
+        config["region"] = AmazonAdsRegion.NA
+
+    @staticmethod
+    def _choose_profiles(config: Mapping[str, Any], profiles: List[Profile]):
+        if not config.get("profiles"):
             return profiles
-        return list(filter(lambda profile: profile.profileId in config.profiles, profiles))
+        return list(filter(lambda profile: profile.profileId in config["profiles"], profiles))

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml

@@ -0,0 +1,128 @@
+---
+documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads
+connectionSpecification:
+  title: Amazon Ads Spec
+  type: object
+  properties:
+    auth_type:
+      title: Auth Type
+      const: oauth2.0
+      order: 0
+      type: string
+    client_id:
+      title: Client ID
+      description:
+        The client ID of your Amazon Ads developer application. See the
+        <a href="https://advertising.amazon.com/API/docs/en-us/get-started/generate-api-tokens#retrieve-your-client-id-and-client-secret">docs</a>
+        for more information.
+      order: 1
+      type: string
+    client_secret:
+      title: Client Secret
+      description:
+        The client secret of your Amazon Ads developer application. See
+        the <a href="https://advertising.amazon.com/API/docs/en-us/get-started/generate-api-tokens#retrieve-your-client-id-and-client-secret">docs</a>
+        for more information.
+      airbyte_secret: true
+      order: 2
+      type: string
+    refresh_token:
+      title: Refresh Token
+      description:
+        Amazon Ads refresh token. See the <a href="https://advertising.amazon.com/API/docs/en-us/get-started/generate-api-tokens">docs</a>
+        for more information on how to obtain this token.
+      airbyte_secret: true
+      order: 3
+      type: string
+    region:
+      title: Region *
+      description:
+        Region to pull data from (EU/NA/FE/SANDBOX). See <a href="https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints">docs</a>
+        for more details.
+      enum:
+        - NA
+        - EU
+        - FE
+        - SANDBOX
+      type: string
+      default: NA
+      order: 4
+    report_wait_timeout:
+      title: Report Wait Timeout *
+      description: Timeout duration in minutes for Reports. Default is 30 minutes.
+      default: 30
+      examples:
+        - 30
+        - 120
+      order: 5
+      type: integer
+    report_generation_max_retries:
+      title: Report Generation Maximum Retries *
+      description:
+        Maximum retries Airbyte will attempt for fetching report data.
+        Default is 5.
+      default: 5
+      examples:
+        - 5
+        - 10
+        - 15
+      order: 6
+      type: integer
+    start_date:
+      title: Start Date (Optional)
+      description:
+        The Start date for collecting reports, should not be more than
+        60 days in the past. In YYYY-MM-DD format
+      examples:
+        - "2022-10-10"
+        - "2022-10-22"
+      order: 7
+      type: string
+    profiles:
+      title: Profile IDs (Optional)
+      description:
+        Profile IDs you want to fetch data for. See <a href="https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles">docs</a>
+        for more details.
+      order: 8
+      type: array
+      items:
+        type: integer
+  required:
+    - client_id
+    - client_secret
+    - refresh_token
+  additionalProperties: true
+advanced_auth:
+  auth_flow_type: oauth2.0
+  predicate_key:
+    - auth_type
+  predicate_value: oauth2.0
+  oauth_config_specification:
+    complete_oauth_output_specification:
+      type: object
+      additionalProperties: false
+      properties:
+        refresh_token:
+          type: string
+          path_in_connector_config:
+            - refresh_token
+    complete_oauth_server_input_specification:
+      type: object
+      additionalProperties: false
+      properties:
+        client_id:
+          type: string
+        client_secret:
+          type: string
+    complete_oauth_server_output_specification:
+      type: object
+      additionalProperties: false
+      properties:
+        client_id:
+          type: string
+          path_in_connector_config:
+            - client_id
+        client_secret:
+          type: string
+          path_in_connector_config:
+            - client_secret

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py

@@ -14,7 +14,6 @@
 from source_amazon_ads.constants import URL_MAPPING
 from source_amazon_ads.schemas import CatalogModel
 from source_amazon_ads.schemas.profile import Profile
-from source_amazon_ads.spec import AmazonAdsConfig
 
 """
 This class hierarchy may seem overcomplicated so here is a visualization of
@@ -74,10 +73,10 @@ class BasicAmazonAdsStream(Stream, ABC):
     Base class for all Amazon Ads streams.
     """
 
-    def __init__(self, config: AmazonAdsConfig, profiles: List[Profile] = None):
+    def __init__(self, config: Mapping[str, Any], profiles: List[Profile] = None):
         self._profiles = profiles or []
-        self._client_id = config.client_id
-        self._url = URL_MAPPING[config.region]
+        self._client_id = config["client_id"]
+        self._url = URL_MAPPING[config["region"]]
 
     @property
     @abstractmethod
@@ -98,7 +97,7 @@ class AmazonAdsStream(HttpStream, BasicAmazonAdsStream):
     Class for getting data from streams that based on single http request.
     """
 
-    def __init__(self, config: AmazonAdsConfig, *args, profiles: List[Profile] = None, **kwargs):
+    def __init__(self, config: Mapping[str, Any], *args, profiles: List[Profile] = None, **kwargs):
         # Each AmazonAdsStream instance are dependant on list of profiles.
         BasicAmazonAdsStream.__init__(self, config, profiles=profiles)
         HttpStream.__init__(self, *args, **kwargs)

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py

@@ -22,7 +22,6 @@
 from pendulum import DateTime
 from pydantic import BaseModel
 from source_amazon_ads.schemas import CatalogModel, MetricsReport, Profile
-from source_amazon_ads.spec import AmazonAdsConfig
 from source_amazon_ads.streams.common import BasicAmazonAdsStream
 
 logger = AirbyteLogger()
@@ -101,14 +100,14 @@ class ReportStream(BasicAmazonAdsStream, ABC):
     REPORT_DATE_FORMAT = "YYYYMMDD"
     cursor_field = "reportDate"
 
-    def __init__(self, config: AmazonAdsConfig, profiles: List[Profile], authenticator: Oauth2Authenticator):
+    def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authenticator: Oauth2Authenticator):
         self._authenticator = authenticator
         self._session = requests.Session()
         self._model = self._generate_model()
-        self.report_wait_timeout = timedelta(minutes=config.report_wait_timeout).total_seconds
-        self.report_generation_maximum_retries = config.report_generation_max_retries
+        self.report_wait_timeout = timedelta(minutes=config.get("report_wait_timeout", 30)).total_seconds
+        self.report_generation_maximum_retries = config.get("report_generation_max_retries", 5)
         # Set start date from config file, should be in UTC timezone.
-        self._start_date = pendulum.parse(config.start_date).set(tz="UTC") if config.start_date else None
+        self._start_date = pendulum.parse(config.get("start_date")).set(tz="UTC") if config.get("start_date") else None
         super().__init__(config, profiles)
 
     @property

airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py

@@ -6,12 +6,13 @@
 
 
 @fixture
-def test_config():
+def config():
     return {
         "client_id": "test_client_id",
         "client_secret": "test_client_secret",
         "scope": "test_scope",
         "refresh_token": "test_refresh",
+        "region": "NA",
     }