Source Klaviyo: unit test coverage 96% (#14872)

* #14793 Source Klaviyo: unit test coverage 96%

* #14793 source klaviyo: upd changelog

* #14793 source klaviyo: fix unit test

* auto-bump connector version [ci skip]

Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -470,7 +470,7 @@
 - name: Klaviyo
   sourceDefinitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde
   dockerRepository: airbyte/source-klaviyo
-  dockerImageTag: 0.1.5
+  dockerImageTag: 0.1.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/klaviyo
   icon: klaviyo.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -4382,11 +4382,12 @@
     supported_destination_sync_modes: []
     supported_source_sync_modes:
     - "append"
-- dockerImage: "airbyte/source-klaviyo:0.1.5"
+- dockerImage: "airbyte/source-klaviyo:0.1.6"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/klaviyo"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/klaviyo"
     connectionSpecification:
+      $schema: "http://json-schema.org/draft-07/schema#"
       title: "Klaviyo Spec"
       type: "object"
       properties:
@@ -4407,11 +4408,9 @@
       required:
       - "api_key"
       - "start_date"
-    supportsIncremental: true
     supportsNormalization: false
     supportsDBT: false
-    supported_destination_sync_modes:
-    - "append"
+    supported_destination_sync_modes: []
 - dockerImage: "airbyte/source-kyriba:0.1.0"
   spec:
     documentationUrl: "https://docsurl.com"

airbyte-integrations/connectors/source-klaviyo/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.5
+LABEL io.airbyte.version=0.1.6
 LABEL io.airbyte.name=airbyte/source-klaviyo

airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml

@@ -3,7 +3,7 @@
 connector_image: airbyte/source-klaviyo:dev
 tests:
   spec:
-    - spec_path: "integration_tests/spec.json"
+    - spec_path: "source_klaviyo/spec.json"
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"
@@ -14,8 +14,13 @@ tests:
   basic_read:
     - config_path: "secrets/config.json"
       empty_streams: ['flows']
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      expect_records:
+        path: "integration_tests/expected_records.txt"
   incremental:
     - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
       future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
     - config_path: "secrets/config.json"
+    - configured_catalog_path: "integration_tests/configured_catalog.json"

airbyte-integrations/connectors/source-klaviyo/integration_tests/configured_catalog.json

@@ -13,7 +13,7 @@
       "sync_mode": "full_refresh",
       "cursor_field": null,
       "destination_sync_mode": "append",
-      "primary_key": null
+      "primary_key": [["id"]]
     },
     {
       "stream": {
@@ -28,7 +28,7 @@
       "sync_mode": "full_refresh",
       "cursor_field": ["timestamp"],
       "destination_sync_mode": "append",
-      "primary_key": null
+      "primary_key": [["id"]]
     },
     {
       "stream": {
@@ -43,7 +43,7 @@
       "sync_mode": "full_refresh",
       "cursor_field": null,
       "destination_sync_mode": "append",
-      "primary_key": null
+      "primary_key": [["email"]]
     },
     {
       "stream": {
@@ -58,7 +58,7 @@
       "sync_mode": "full_refresh",
       "cursor_field": null,
       "destination_sync_mode": "append",
-      "primary_key": null
+      "primary_key": [["id"]]
     },
     {
       "stream": {
@@ -73,7 +73,7 @@
       "sync_mode": "full_refresh",
       "cursor_field": null,
       "destination_sync_mode": "append",
-      "primary_key": null
+      "primary_key": [["id"]]
     }
   ]
 }

airbyte-integrations/connectors/source-klaviyo/setup.py

@@ -7,11 +7,7 @@
 
 MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"]
 
-TEST_REQUIREMENTS = [
-    "pytest~=6.1",
-    "pytest-mock",
-    "source-acceptance-test",
-]
+TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock", "source-acceptance-test", "requests_mock~=1.8"]
 
 setup(
     name="source_klaviyo",

airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py

@@ -2,77 +2,40 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+from typing import Any, List, Mapping, Tuple
 
-from typing import Any, List, Mapping, Tuple, Type
-
-from airbyte_cdk.models import ConnectorSpecification, DestinationSyncMode, SyncMode
+from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
-from pydantic import Field
-from pydantic.main import BaseModel
 from source_klaviyo.streams import Campaigns, Events, Flows, GlobalExclusions, Lists, Metrics
 
 
-class ConnectorConfig(BaseModel):
-    class Config:
-        title = "Klaviyo Spec"
-
-    api_key: str = Field(
-        description='Klaviyo API Key. See our <a href="https://docs.airbyte.io/integrations/sources/klaviyo">docs</a> if you need help finding this key.',
-        airbyte_secret=True,
-    )
-    start_date: str = Field(
-        description="UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.",
-        pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
-        examples=["2017-01-25T00:00:00Z"],
-    )
-
-
 class SourceKlaviyo(AbstractSource):
     def check_connection(self, logger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
         """Connection check to validate that the user-provided config can be used to connect to the underlying API
-
         :param config:  the user-input config object conforming to the connector's spec.json
         :param logger:  logger object
         :return Tuple[bool, Any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise.
         """
-        ok = False
-        error_msg = None
-        config = ConnectorConfig.parse_obj(config)  # FIXME: this will be not need after we fix CDK
-
         try:
-            # we use metrics endpoint because it never return an error
-            _ = list(Metrics(api_key=config.api_key).read_records(sync_mode=SyncMode.full_refresh))
-            ok = True
+            # we use metrics endpoint because it never returns an error
+            _ = list(Metrics(api_key=config["api_key"]).read_records(sync_mode=SyncMode.full_refresh))
         except Exception as e:
-            error_msg = repr(e)
-
-        return ok, error_msg
-
-    def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]:
-        """Discovery method, returns available streams
+            return False, repr(e)
+        return True, None
 
+    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
+        """
+        Discovery method, returns available streams
         :param config: A Mapping of the user input configuration as defined in the connector spec.
         """
-        config = ConnectorConfig.parse_obj(config)  # FIXME: this will be not need after we fix CDK
+        api_key = config["api_key"]
+        start_date = config["start_date"]
         return [
-            Campaigns(api_key=config.api_key),
-            Events(api_key=config.api_key, start_date=config.start_date),
-            GlobalExclusions(api_key=config.api_key, start_date=config.start_date),
-            Lists(api_key=config.api_key),
-            Metrics(api_key=config.api_key),
-            Flows(api_key=config.api_key, start_date=config.start_date),
+            Campaigns(api_key=api_key),
+            Events(api_key=api_key, start_date=start_date),
+            GlobalExclusions(api_key=api_key, start_date=start_date),
+            Lists(api_key=api_key),
+            Metrics(api_key=api_key),
+            Flows(api_key=api_key, start_date=start_date),
         ]
-
-    def spec(self, *args, **kwargs) -> ConnectorSpecification:
-        """
-        Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password)
-        required to run this integration.
-        """
-        return ConnectorSpecification(
-            documentationUrl="https://docs.airbyte.io/integrations/sources/klaviyo",
-            changelogUrl="https://docs.airbyte.io/integrations/sources/klaviyo",
-            supportsIncremental=True,
-            supported_destination_sync_modes=[DestinationSyncMode.append],
-            connectionSpecification=ConnectorConfig.schema(),
-        )

airbyte-integrations/connectors/source-klaviyo/integration_tests/spec.json → airbyte-integrations/connectors/source-klaviyo/source_klaviyo/spec.json

@@ -2,6 +2,7 @@
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/klaviyo",
   "changelogUrl": "https://docs.airbyte.io/integrations/sources/klaviyo",
   "connectionSpecification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Klaviyo Spec",
     "type": "object",
     "properties": {
@@ -20,7 +21,5 @@
       }
     },
     "required": ["api_key", "start_date"]
-  },
-  "supportsIncremental": true,
-  "supported_destination_sync_modes": ["append"]
+  }
 }