fix: chats stream is only pulling for first page

airbytehq · Oct 21, 2021 · 06f0ad3 · 06f0ad3
1 parent e3be3d2
commit 06f0ad3
Show file tree

Hide file tree

Showing 7 changed files with 47 additions and 14 deletions.
diff --git a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile
@@ -14,5 +14,5 @@ RUN pip install .
 
 ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
 
-LABEL io.airbyte.version=0.1.2
+LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/source-zendesk-chat
diff --git a/...yte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog.json b/...yte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog.json
@@ -0,0 +1,13 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "chats",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    }
+  ]
+}
diff --git a/airbyte-integrations/connectors/source-zendesk-chat/main_dev.py b/airbyte-integrations/connectors/source-zendesk-chat/main_dev.py
@@ -5,7 +5,7 @@
 
 import sys
 
-from base_python.entrypoint import launch
+from airbyte_cdk.entrypoint import launch
 from source_zendesk_chat import SourceZendeskChat
 
 if __name__ == "__main__":

diff --git a/airbyte-integrations/connectors/source-zendesk-chat/setup.py b/airbyte-integrations/connectors/source-zendesk-chat/setup.py
@@ -6,8 +6,7 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
-    "airbyte-protocol",
-    "base-python",
+    "airbyte-cdk~=0.1",
     "pendulum==1.2.0",
     "requests==2.25.1",
 ]

diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py
@@ -5,23 +5,27 @@
 
 from typing import Any, List, Mapping, Tuple
 
-from airbyte_protocol import SyncMode
-from base_python import AbstractSource, Stream, TokenAuthenticator
+from airbyte_cdk.models import SyncMode
+from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
 
-from .api import Accounts, Agents, AgentTimelines, Bans, Chats, Departments, Goals, Roles, RoutingSettings, Shortcuts, Skills, Triggers
+
+from .streams import Accounts, Agents, AgentTimelines, Bans, Chats, Departments, Goals, Roles, RoutingSettings, Shortcuts, Skills, Triggers
 
 
 class SourceZendeskChat(AbstractSource):
     def check_connection(self, logger, config) -> Tuple[bool, any]:
         try:
-            authenticator = TokenAuthenticator(token=config["access_token"])
-            list(RoutingSettings(authenticator=authenticator).read_records(SyncMode.full_refresh))
+            authenticator = TokenAuthenticator(config["access_token"])
+            records = RoutingSettings(authenticator=authenticator).read_records(sync_mode=SyncMode.full_refresh)
+            next(records)
             return True, None
         except Exception as error:
             return False, f"Unable to connect to Zendesk Chat API with the provided credentials - {error}"
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        authenticator = TokenAuthenticator(token=config["access_token"])
+        authenticator = TokenAuthenticator(config["access_token"])
         return [
             Agents(authenticator=authenticator),
             AgentTimelines(authenticator=authenticator, start_date=config["start_date"]),

diff --git a/...e-zendesk-chat/source_zendesk_chat/api.py → ...ndesk-chat/source_zendesk_chat/streams.py b/...e-zendesk-chat/source_zendesk_chat/api.py → ...ndesk-chat/source_zendesk_chat/streams.py
@@ -8,13 +8,18 @@
 
 import pendulum
 import requests
-from base_python import HttpStream
+from urllib.parse import urlparse
+from urllib.parse import parse_qs
 
 
-class Stream(HttpStream):
+from airbyte_cdk.sources.streams.http import HttpStream
+
+class Stream(HttpStream, ABC):
     url_base = "https://www.zopim.com/api/v2/"
+    primary_key = "id"
 
     data_field = None
+
     limit = 100
 
     def backoff_time(self, response: requests.Response) -> Optional[float]:
@@ -26,7 +31,12 @@ def path(self, **kwargs) -> str:
         return self.name
 
     def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
-        return {}
+        response_data = response.json()
+
+        if "next_url" in response_data:
+            next_url = response_data["next_url"]
+            cursor = parse_qs(urlparse(next_url).query)['cursor']     
+            return { "cursor": cursor }
 
     def request_params(
         self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs
@@ -240,5 +250,11 @@ class RoutingSettings(Stream):
     name = "routing_settings"
     data_field = "data"
 
-    def path(self, **kwargs) -> str:
+
+    def path(
+        self,
+        stream_state: Mapping[str, Any] = None,
+        stream_slice: Mapping[str, Any] = None,
+        next_page_token: Mapping[str, Any] = None,
+    ) -> str:
         return "routing_settings/account"
diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md
@@ -62,6 +62,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.3 | 2021-10-21 | [7210](https://github.com/airbytehq/airbyte/pull/7210) | Chats stream is only getting data from first page |
 | 0.1.2 | 2021-08-17 | [5476](https://github.com/airbytehq/airbyte/pull/5476) | Correct field unread to boolean type |
 | 0.1.1 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support |
 | 0.1.0 | 2021-05-03 | [3088](https://github.com/airbytehq/airbyte/pull/3088) | Initial release |