From c69f60982b32816ff8a55ac1bedf1ff217e33abe Mon Sep 17 00:00:00 2001 From: harshithmullapudi Date: Wed, 20 Oct 2021 15:16:29 +0530 Subject: [PATCH 1/5] fix: chats stream is only pulling for first page --- .../connectors/source-zendesk-chat/Dockerfile | 2 +- .../integration_tests/configured_catalog.json | 13 ++++++++++ .../source-zendesk-chat/main_dev.py | 2 +- .../connectors/source-zendesk-chat/setup.py | 3 +-- .../source_zendesk_chat/source.py | 16 ++++++++----- .../{api.py => streams.py} | 24 +++++++++++++++---- docs/integrations/sources/zendesk-chat.md | 1 + 7 files changed, 47 insertions(+), 14 deletions(-) create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog.json rename airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/{api.py => streams.py} (92%) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile index 958d43448ce5..67d80d64a020 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile @@ -14,5 +14,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh" -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-zendesk-chat diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..d22ffc702c98 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "chats", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/main_dev.py b/airbyte-integrations/connectors/source-zendesk-chat/main_dev.py index 0135fc536761..ae5be85e4adf 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/main_dev.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/main_dev.py @@ -5,7 +5,7 @@ import sys -from base_python.entrypoint import launch +from airbyte_cdk.entrypoint import launch from source_zendesk_chat import SourceZendeskChat if __name__ == "__main__": diff --git a/airbyte-integrations/connectors/source-zendesk-chat/setup.py b/airbyte-integrations/connectors/source-zendesk-chat/setup.py index 1ec3ae9425f4..e654c05df9e0 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/setup.py @@ -6,8 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-protocol", - "base-python", + "airbyte-cdk~=0.1", "pendulum==1.2.0", "requests==2.25.1", ] diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py index 94946dc2ec92..278f059114a7 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py @@ -5,23 +5,27 @@ from typing import Any, List, Mapping, Tuple -from airbyte_protocol import SyncMode -from base_python import AbstractSource, Stream, TokenAuthenticator +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from .api import Accounts, Agents, AgentTimelines, Bans, Chats, Departments, Goals, Roles, RoutingSettings, Shortcuts, Skills, Triggers + +from .streams import Accounts, Agents, AgentTimelines, Bans, Chats, Departments, Goals, Roles, RoutingSettings, Shortcuts, Skills, Triggers class SourceZendeskChat(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: try: - authenticator = TokenAuthenticator(token=config["access_token"]) - list(RoutingSettings(authenticator=authenticator).read_records(SyncMode.full_refresh)) + authenticator = TokenAuthenticator(config["access_token"]) + records = RoutingSettings(authenticator=authenticator).read_records(sync_mode=SyncMode.full_refresh) + next(records) return True, None except Exception as error: return False, f"Unable to connect to Zendesk Chat API with the provided credentials - {error}" def streams(self, config: Mapping[str, Any]) -> List[Stream]: - authenticator = TokenAuthenticator(token=config["access_token"]) + authenticator = TokenAuthenticator(config["access_token"]) return [ Agents(authenticator=authenticator), AgentTimelines(authenticator=authenticator, start_date=config["start_date"]), diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/api.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py similarity index 92% rename from airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/api.py rename to airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py index e3b6ccac2621..5e8ee0ffb56e 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/api.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py @@ -8,13 +8,18 @@ import pendulum import requests -from base_python import HttpStream +from urllib.parse import urlparse +from urllib.parse import parse_qs -class Stream(HttpStream): +from airbyte_cdk.sources.streams.http import HttpStream + +class Stream(HttpStream, ABC): url_base = "https://www.zopim.com/api/v2/" + primary_key = "id" data_field = None + limit = 100 def backoff_time(self, response: requests.Response) -> Optional[float]: @@ -26,7 +31,12 @@ def path(self, **kwargs) -> str: return self.name def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return {} + response_data = response.json() + + if "next_url" in response_data: + next_url = response_data["next_url"] + cursor = parse_qs(urlparse(next_url).query)['cursor'] + return { "cursor": cursor } def request_params( self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs @@ -240,5 +250,11 @@ class RoutingSettings(Stream): name = "routing_settings" data_field = "data" - def path(self, **kwargs) -> str: + + def path( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: return "routing_settings/account" diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md index 91e56231b6b8..ba0629792372 100644 --- a/docs/integrations/sources/zendesk-chat.md +++ b/docs/integrations/sources/zendesk-chat.md @@ -62,6 +62,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.3 | 2021-10-21 | [7210](https://github.com/airbytehq/airbyte/pull/7210) | Chats stream is only getting data from first page | | 0.1.2 | 2021-08-17 | [5476](https://github.com/airbytehq/airbyte/pull/5476) | Correct field unread to boolean type | | 0.1.1 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | | 0.1.0 | 2021-05-03 | [3088](https://github.com/airbytehq/airbyte/pull/3088) | Initial release | From 9ff57deb7b4ef5514caa0e0cc2a8bc3359a27593 Mon Sep 17 00:00:00 2001 From: harshithmullapudi Date: Wed, 27 Oct 2021 08:58:32 +0530 Subject: [PATCH 2/5] fix: integration tests --- .../40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../source-zendesk-chat/.dockerignore | 1 + .../connectors/source-zendesk-chat/Dockerfile | 6 ++- .../integration_tests/invalid_config.json | 6 +-- .../sample_files/sample_config.json | 2 +- .../connectors/source-zendesk-chat/setup.py | 2 +- .../source_zendesk_chat/source.py | 1 - .../source_zendesk_chat/streams.py | 43 +++++++++++++------ 9 files changed, 42 insertions(+), 23 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4.json index 017ddd500244..61cd0192bfaa 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4", "name": "Zendesk Chat", "dockerRepository": "airbyte/source-zendesk-chat", - "dockerImageTag": "0.1.2", + "dockerImageTag": "0.1.3", "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-chat", "icon": "zendesk.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 959ca13ebb06..541a18c32bb0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -554,7 +554,7 @@ - name: Zendesk Chat sourceDefinitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 dockerRepository: airbyte/source-zendesk-chat - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-chat icon: zendesk.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-zendesk-chat/.dockerignore b/airbyte-integrations/connectors/source-zendesk-chat/.dockerignore index dc3649354c32..39caaa9872cc 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/.dockerignore +++ b/airbyte-integrations/connectors/source-zendesk-chat/.dockerignore @@ -1,6 +1,7 @@ * !Dockerfile !Dockerfile.test +!main_dev.py !source_zendesk_chat !setup.py !secrets diff --git a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile index 67d80d64a020..3d61adfccd49 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile @@ -1,4 +1,4 @@ -FROM airbyte/integration-base-python:0.1.6 +FROM python:3.7-slim # Bash is installed for more convenient debugging. RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* @@ -6,13 +6,15 @@ RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* ENV CODE_PATH="source_zendesk_chat" ENV AIRBYTE_IMPL_MODULE="source_zendesk_chat" ENV AIRBYTE_IMPL_PATH="SourceZendeskChat" +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main_dev.py" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH +COPY main_dev.py ./ COPY setup.py ./ RUN pip install . -ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh" +ENTRYPOINT ["python", "/airbyte/integration_code/main_dev.py"] LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-zendesk-chat diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/invalid_config.json index 10ca0ef76aad..7cf01b4084b4 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/invalid_config.json @@ -1,6 +1,4 @@ { - "credentials": { - "access_token": "wrongkey-access-token", - "start_date": "2020-12-12T00:00:00Z" - } + "access_token": "wrongkey-access-token", + "start_date": "2020-12-12T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-zendesk-chat/sample_files/sample_config.json b/airbyte-integrations/connectors/source-zendesk-chat/sample_files/sample_config.json index 6d9448d65706..b5e68067b85d 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/sample_files/sample_config.json +++ b/airbyte-integrations/connectors/source-zendesk-chat/sample_files/sample_config.json @@ -1,4 +1,4 @@ { "access_token": "", - "start_date": "2020-11-01T00:00:00" + "start_date": "2020-11-01T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-zendesk-chat/setup.py b/airbyte-integrations/connectors/source-zendesk-chat/setup.py index e654c05df9e0..6cc0e89c3936 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/setup.py @@ -11,7 +11,7 @@ "requests==2.25.1", ] -TEST_REQUIREMENTS = ["pytest==6.1.2", "source-acceptance-test"] +TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock"] setup( name="source_zendesk_chat", diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py index 278f059114a7..4d3f8a255c30 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py @@ -10,7 +10,6 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator - from .streams import Accounts, Agents, AgentTimelines, Bans, Chats, Departments, Goals, Roles, RoutingSettings, Shortcuts, Skills, Triggers diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py index 5e8ee0ffb56e..22dda84e4ba6 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py @@ -5,15 +5,13 @@ from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union +from urllib.parse import parse_qs, urlparse import pendulum import requests -from urllib.parse import urlparse -from urllib.parse import parse_qs - - from airbyte_cdk.sources.streams.http import HttpStream + class Stream(HttpStream, ABC): url_base = "https://www.zopim.com/api/v2/" primary_key = "id" @@ -32,11 +30,11 @@ def path(self, **kwargs) -> str: def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: response_data = response.json() - + if "next_url" in response_data: next_url = response_data["next_url"] - cursor = parse_qs(urlparse(next_url).query)['cursor'] - return { "cursor": cursor } + cursor = parse_qs(urlparse(next_url).query)["cursor"] + return {"cursor": cursor} def request_params( self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs @@ -58,12 +56,15 @@ def get_stream_data(self, response_data: Any) -> List[dict]: response_data = response_data.get(self.data_field, []) if isinstance(response_data, list): - return response_data + return list(map(self.parse_response_obj, response_data)) elif isinstance(response_data, dict): - return [response_data] + return [self.parse_response_obj(response_data)] else: raise Exception(f"Unsupported type of response data for stream {self.name}") + def parse_response_obj(self, response_obj: dict) -> dict: + return response_obj + class BaseIncrementalStream(Stream, ABC): @property @@ -105,8 +106,9 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: latest_benchmark = self._field_to_datetime(latest_record[self.cursor_field]) if current_stream_state.get(self.cursor_field): - return {self.cursor_field: str(max(latest_benchmark, self._field_to_datetime(current_stream_state[self.cursor_field])))} - return {self.cursor_field: str(latest_benchmark)} + state = max(latest_benchmark, self._field_to_datetime(current_stream_state[self.cursor_field])) + return {self.cursor_field: state.strftime("%Y-%m-%dT%H:%M:%SZ")} + return {self.cursor_field: latest_benchmark.strftime("%Y-%m-%dT%H:%M:%SZ")} def request_params( self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs @@ -127,6 +129,10 @@ def request_params( def path(self, **kwargs) -> str: return f"incremental/{self.name}" + def parse_response_obj(self, response_obj: dict) -> dict: + response_obj[self.cursor_field] = pendulum.parse(response_obj[self.cursor_field]).strftime("%Y-%m-%dT%H:%M:%SZ") + return response_obj + class IdIncrementalStream(BaseIncrementalStream): cursor_field = "id" @@ -177,6 +183,20 @@ def request_params(self, **kwargs) -> MutableMapping[str, Any]: params["start_time"] = params["start_time"] * 1000000 return params + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_data = response.json() + stream_data = self.get_stream_data(response_data) + + def generate_key(record): + record.update({"id": "|".join((str(record.get("agent_id", "")), str(record.get("start_time", ""))))}) + return record + + # associate the surrogate key + yield from map( + generate_key, + stream_data, + ) + class Accounts(Stream): """ @@ -250,7 +270,6 @@ class RoutingSettings(Stream): name = "routing_settings" data_field = "data" - def path( self, stream_state: Mapping[str, Any] = None, From bf530c9d3b1f53cb5f331d69793ff3366104931f Mon Sep 17 00:00:00 2001 From: harshithmullapudi Date: Wed, 27 Oct 2021 09:22:17 +0530 Subject: [PATCH 3/5] fix: remove cred --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 541a18c32bb0..959ca13ebb06 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -554,7 +554,7 @@ - name: Zendesk Chat sourceDefinitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 dockerRepository: airbyte/source-zendesk-chat - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-chat icon: zendesk.svg sourceType: api From fa10366acdcce6767b4a54eaeb98efc8dfb0444f Mon Sep 17 00:00:00 2001 From: harshithmullapudi Date: Mon, 1 Nov 2021 17:00:24 +0530 Subject: [PATCH 4/5] bump version in source definition --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 959ca13ebb06..541a18c32bb0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -554,7 +554,7 @@ - name: Zendesk Chat sourceDefinitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 dockerRepository: airbyte/source-zendesk-chat - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-chat icon: zendesk.svg sourceType: api From 4e7db6cc4747ef97c7e049d222970d8e7b785401 Mon Sep 17 00:00:00 2001 From: harshithmullapudi Date: Mon, 1 Nov 2021 17:54:19 +0530 Subject: [PATCH 5/5] Fixed: integration tests --- .../acceptance-test-config.yml | 2 +- .../configured_catalog_incremental.json | 40 +++++++++++++++++ .../sample_files/configured_catalog.json | 45 ------------------- .../source_zendesk_chat/streams.py | 4 ++ 4 files changed, 45 insertions(+), 46 deletions(-) create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog_incremental.json diff --git a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml index f9c942e0b1e5..f6fbd69197e5 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml @@ -14,7 +14,7 @@ tests: configured_catalog_path: "sample_files/configured_catalog.json" incremental: - config_path: "secrets/config.json" - configured_catalog_path: "sample_files/configured_catalog.json" + configured_catalog_path: "integration_tests/configured_catalog_incremental.json" # Unable to use 'state_path' because Zendesk Chat API returns an error when specifying a date in the future. # future_state_path: "integration_tests/abnormal_state.json" cursor_paths: diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog_incremental.json new file mode 100644 index 000000000000..e350c92fb6cd --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/configured_catalog_incremental.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "stream": { + "name": "agents", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["id"] + }, + { + "stream": { + "name": "agent_timeline", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["start_time"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["start_time"] + }, + { + "stream": { + "name": "bans", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["id"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-zendesk-chat/sample_files/configured_catalog.json index 5ca4f7c01f55..4cde9b055ac5 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-zendesk-chat/sample_files/configured_catalog.json @@ -1,29 +1,5 @@ { "streams": [ - { - "stream": { - "name": "agents", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["id"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["id"] - }, - { - "stream": { - "name": "agent_timeline", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["start_time"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["start_time"] - }, { "stream": { "name": "accounts", @@ -60,18 +36,6 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, - { - "stream": { - "name": "bans", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["id"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["id"] - }, { "stream": { "name": "departments", @@ -107,15 +71,6 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "routing_settings", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py index 22dda84e4ba6..6caa79ad81b4 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py @@ -203,6 +203,8 @@ class Accounts(Stream): Accounts Stream: https://developer.zendesk.com/rest_api/docs/chat/accounts#show-account """ + primary_key = "account_key" + def path(self, **kwargs) -> str: return "account" @@ -267,6 +269,8 @@ class RoutingSettings(Stream): Routing Settings Stream: https://developer.zendesk.com/rest_api/docs/chat/routing_settings#show-account-routing-settings """ + primary_key = "" + name = "routing_settings" data_field = "data"