fix: integration tests

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4.json

@@ -2,7 +2,7 @@
   "sourceDefinitionId": "40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4",
   "name": "Zendesk Chat",
   "dockerRepository": "airbyte/source-zendesk-chat",
-  "dockerImageTag": "0.1.2",
+  "dockerImageTag": "0.1.3",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-chat",
   "icon": "zendesk.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -554,7 +554,7 @@
 - name: Zendesk Chat
   sourceDefinitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4
   dockerRepository: airbyte/source-zendesk-chat
-  dockerImageTag: 0.1.2
+  dockerImageTag: 0.1.3
   documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-chat
   icon: zendesk.svg
   sourceType: api

airbyte-integrations/connectors/source-zendesk-chat/.dockerignore

@@ -1,6 +1,7 @@
 *
 !Dockerfile
 !Dockerfile.test
+!main_dev.py
 !source_zendesk_chat
 !setup.py
 !secrets

airbyte-integrations/connectors/source-zendesk-chat/Dockerfile

@@ -1,18 +1,20 @@
-FROM airbyte/integration-base-python:0.1.6
+FROM python:3.7-slim
 
 # Bash is installed for more convenient debugging.
 RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
 
 ENV CODE_PATH="source_zendesk_chat"
 ENV AIRBYTE_IMPL_MODULE="source_zendesk_chat"
 ENV AIRBYTE_IMPL_PATH="SourceZendeskChat"
+ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main_dev.py"
 
 WORKDIR /airbyte/integration_code
 COPY $CODE_PATH ./$CODE_PATH
+COPY main_dev.py ./
 COPY setup.py ./
 RUN pip install .
 
-ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
+ENTRYPOINT ["python", "/airbyte/integration_code/main_dev.py"]
 
 LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/source-zendesk-chat

airbyte-integrations/connectors/source-zendesk-chat/integration_tests/invalid_config.json

@@ -1,6 +1,4 @@
 {
-  "credentials": {
-    "access_token": "wrongkey-access-token",
-    "start_date": "2020-12-12T00:00:00Z"
-  }
+  "access_token": "wrongkey-access-token",
+  "start_date": "2020-12-12T00:00:00Z"
 }

airbyte-integrations/connectors/source-zendesk-chat/sample_files/sample_config.json

@@ -1,4 +1,4 @@
 {
   "access_token": "<your_access_token>",
-  "start_date": "2020-11-01T00:00:00"
+  "start_date": "2020-11-01T00:00:00Z"
 }

airbyte-integrations/connectors/source-zendesk-chat/setup.py

@@ -11,7 +11,7 @@
     "requests==2.25.1",
 ]
 
-TEST_REQUIREMENTS = ["pytest==6.1.2", "source-acceptance-test"]
+TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock"]
 
 setup(
     name="source_zendesk_chat",

airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py

@@ -10,7 +10,6 @@
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
 
-
 from .streams import Accounts, Agents, AgentTimelines, Bans, Chats, Departments, Goals, Roles, RoutingSettings, Shortcuts, Skills, Triggers
 
 

airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py

@@ -5,15 +5,13 @@
 
 from abc import ABC, abstractmethod
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
+from urllib.parse import parse_qs, urlparse
 
 import pendulum
 import requests
-from urllib.parse import urlparse
-from urllib.parse import parse_qs
-
-
 from airbyte_cdk.sources.streams.http import HttpStream
 
+
 class Stream(HttpStream, ABC):
     url_base = "https://www.zopim.com/api/v2/"
     primary_key = "id"
@@ -32,11 +30,11 @@ def path(self, **kwargs) -> str:
 
     def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
         response_data = response.json()
-        
+
         if "next_url" in response_data:
             next_url = response_data["next_url"]
-            cursor = parse_qs(urlparse(next_url).query)['cursor']     
-            return { "cursor": cursor }
+            cursor = parse_qs(urlparse(next_url).query)["cursor"]
+            return {"cursor": cursor}
 
     def request_params(
         self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs
@@ -58,12 +56,15 @@ def get_stream_data(self, response_data: Any) -> List[dict]:
             response_data = response_data.get(self.data_field, [])
 
         if isinstance(response_data, list):
-            return response_data
+            return list(map(self.parse_response_obj, response_data))
         elif isinstance(response_data, dict):
-            return [response_data]
+            return [self.parse_response_obj(response_data)]
         else:
             raise Exception(f"Unsupported type of response data for stream {self.name}")
 
+    def parse_response_obj(self, response_obj: dict) -> dict:
+        return response_obj
+
 
 class BaseIncrementalStream(Stream, ABC):
     @property
@@ -105,8 +106,9 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str,
     def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]:
         latest_benchmark = self._field_to_datetime(latest_record[self.cursor_field])
         if current_stream_state.get(self.cursor_field):
-            return {self.cursor_field: str(max(latest_benchmark, self._field_to_datetime(current_stream_state[self.cursor_field])))}
-        return {self.cursor_field: str(latest_benchmark)}
+            state = max(latest_benchmark, self._field_to_datetime(current_stream_state[self.cursor_field]))
+            return {self.cursor_field: state.strftime("%Y-%m-%dT%H:%M:%SZ")}
+        return {self.cursor_field: latest_benchmark.strftime("%Y-%m-%dT%H:%M:%SZ")}
 
     def request_params(
         self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs
@@ -127,6 +129,10 @@ def request_params(
     def path(self, **kwargs) -> str:
         return f"incremental/{self.name}"
 
+    def parse_response_obj(self, response_obj: dict) -> dict:
+        response_obj[self.cursor_field] = pendulum.parse(response_obj[self.cursor_field]).strftime("%Y-%m-%dT%H:%M:%SZ")
+        return response_obj
+
 
 class IdIncrementalStream(BaseIncrementalStream):
     cursor_field = "id"
@@ -177,6 +183,20 @@ def request_params(self, **kwargs) -> MutableMapping[str, Any]:
             params["start_time"] = params["start_time"] * 1000000
         return params
 
+    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
+        response_data = response.json()
+        stream_data = self.get_stream_data(response_data)
+
+        def generate_key(record):
+            record.update({"id": "|".join((str(record.get("agent_id", "")), str(record.get("start_time", ""))))})
+            return record
+
+        # associate the surrogate key
+        yield from map(
+            generate_key,
+            stream_data,
+        )
+
 
 class Accounts(Stream):
     """
@@ -250,7 +270,6 @@ class RoutingSettings(Stream):
     name = "routing_settings"
     data_field = "data"
 
-
     def path(
         self,
         stream_state: Mapping[str, Any] = None,